connect1.f

Go to the documentation of this file.

c-----------------------------------------------------------------------
      subroutine setup_topo
C
C     Parallel compatible routine to find 
C     connectivity of element structure.
C
C     On Processor 0:
C
C     .Verify right-handedness of elements.
C     .Verify element-to-element reciprocity of BC's
C     .Verify correlation between E-E BC's and physical coincidence
C     .Set rotations
C     .Determine multiplicity
C     .Set up direct stiffness summation arrays.
C
C     All Processors:
C
C     .Disperse/Receive BC and MULT temporary data read from preprocessor.
C
C
      include 'SIZE'
      include 'TOTAL'
      include 'NONCON'
      include 'SCRCT'
c
      COMMON /scruz/ xm3(lx3,ly3,lz3,lelt)
     $ ,             ym3(lx3,ly3,lz3,lelt)
     $ ,             zm3(lx3,ly3,lz3,lelt)
C
      common /c_is1/ glo_num(1*lx1*ly1*lz1*lelv)
      integer*8 glo_num
      common /ivrtx/ vertex((2**ldim)*lelt)
      integer vertex
 
      if(nio.eq.0) write(6,*) 'setup mesh topology'
C
C     Initialize key arrays for Direct Stiffness SUM.
C
      nxl=3
      nyl=3
      nzl=1+2*(ldim-2)
 
      call initds
      call dsset (nxl,nyl,nzl)
      call setedge
C
C=================================================
C     Establish (global) domain topology
C=================================================
C
C     .Generate topologically correct mesh data.
C     .Set up element centers, face centers, etc. 
C     .Check  right handedness of elements.
C     .Check  element boundary conditions.
C     .Establish Element-Element rotations
C     .Construct the element to processor map and
 
      call genxyzl
      call setside
      call verify
 
      CALL setcdof 
      IF (ifaxis            ) CALL setrzer
      IF (ifmvbd            ) CALL cbcmesh
      CALL chkaxcb
C
C========================================================================
C     Set up element-processor mapping and establish global numbering
C========================================================================
C
      mfield=2
      if (ifflow) mfield=1
      if (ifmvbd) mfield=0
 
      ncrnr = 2**ldim
 
      if (nelgv.eq.nelgt) then
         call get_vert
         call setupds(gsh_fld(1),lx1,ly1,lz1,nelv,nelgv,vertex,glo_num)
         gsh_fld(2)=gsh_fld(1)
 
c        call gs_counter(glo_num,gsh_fld(1))
 
      else
 
c
c        For conjugate heat transfer, it is assumed that fluid
c        elements are listed both globally and locally with lower
c        element numbers than the solid elements.
c
c        We currently assume that there is at least one fluid elem.
c        per processor.
c
 
         call get_vert
c        call outmati(vertex,4,nelv,'vrtx V')
         call setupds(gsh_fld(1),lx1,ly1,lz1,nelv,nelgv,vertex,glo_num)
 
c        call get_vert  (vertex, ncrnr, nelgt, '.mp2')  !  LATER !
c        call outmati(vertex,4,nelt,'vrtx T')
         call setupds(gsh_fld(2),lx1,ly1,lz1,nelt,nelgt,vertex,glo_num)
 
c
c        Feb 20, 2012:  It appears that we do not need this restriction: (pff)
c
c        check if there is a least one fluid element on each processor
c        do iel = 1,nelt
c           ieg = lglel(iel)
c           if (ieg.le.nelgv) goto 101 
c        enddo
c        if(nio.eq.0) write(6,*) 
c    &     'ERROR: each domain must contain at least one fluid element!'
c        call exitt
c 101   continue
 
      endif
 
 
c     if (ifmvbd) call setup_mesh_dssum ! Set up dssum for mesh
 
C========================================================================
C     Set up multiplicity and direct stiffness arrays for each IFIELD
C========================================================================
 
      ntotv = lx1*ly1*lz1*nelv
      ntott = lx1*ly1*lz1*nelt
 
 
 
      if (ifflow) then
         ifield = 1
         call rone    (vmult,ntotv)
         call dssum   (vmult,lx1,ly1,lz1)
         vmltmax=glmax(vmult,ntotv)
         ivmltmax=vmltmax
         if (nio.eq.0) write(6,*) ivmltmax,' max multiplicity'
         call invcol1 (vmult,ntotv)
      endif
      if (ifheat) then
         ifield = 2
         call rone    (tmult,ntott)
         call dssum   (tmult,lx1,ly1,lz1)
         call invcol1 (tmult,ntott)
      endif
      if (.not.ifflow) call copy(vmult,tmult,ntott)
      if (ifmvbd)  call copy (wmult,vmult,ntott)
      do ifield=3,nfield                  ! Additional pass. scalrs.
         if (nelg(ifield).eq.nelgv) then
            gsh_fld(ifield) = gsh_fld(1)
            call copy (tmult(1,1,1,1,ifield-1),vmult,ntotv)
         else
            gsh_fld(ifield) = gsh_fld(2)
            call copy (tmult(1,1,1,1,ifield-1),tmult,ntott)
         endif
      enddo
 
      ifgsh_fld_same = .true.
      do ifield=2,nfield
         if (gsh_fld(ifield).ne.gsh_fld(1)) then
            ifgsh_fld_same = .false.
            goto 100
         endif
      enddo
 100  continue
 
      if(nio.eq.0) then
        write(6,*) 'done :: setup mesh topology'
        write(6,*) ' '
      endif
 
      return
      end
c-----------------------------------------------------------------------
      subroutine initds
C
C          -- Direct Stiffness Initialization Routine --
C
C     Set up required data for packing data on faces of spectral cubes.
C
      include 'SIZE'
      include 'TOPOL'
C
C     Nominal ordering for direct stiffness summation of faces
C
      j=0
      DO 5 idim=1,ldim
      DO 5 iface=1,2
        j=j+1
         nomlis(iface,idim)=j
    5 CONTINUE
C
C     Assign Ed's numbering scheme to PF's scheme.
C
      eface(1)=4
      eface(2)=2
      eface(3)=1
      eface(4)=3
      eface(5)=5
      eface(6)=6
C
C     Assign inverse of Ed's numbering scheme to PF's scheme.
C
      eface1(1)=3
      eface1(2)=2
      eface1(3)=4
      eface1(4)=1
      eface1(5)=5
      eface1(6)=6
C
C     Assign group designation to each face to determine ordering of indices.
C
      group(1)=0
      group(2)=1
      group(3)=1
      group(4)=0
      group(5)=0
      group(6)=1
C
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine setedge
C
C     .Initialize EDGE arrays for face and edge specific tasks.
C
C     .NOTE: Sevaral arrays in common are initialized via 
C            BLOCKDATA EDGEC
C
C     Computed arrays: 
C
C     IEDGE  -  Minimal list of wire frame nodes.
C               Used to search for all physical
C               coincidences.
C
C
C     IEDGEF - .Ordered list of wire frame nodes 
C               associated with faces 1 through 6.
C              .Each of 4 sides of square frame stored 
C               individually so that rotations are 
C               readily handled.
C              .Two types of node orderings stored -
C               (0) is clockwise marching
C               (1) is counter-clockwise marching 
C                   for image face.
C                               
C
C     IFACE         - indicates the face number.  Two notations
C                     are currently in use:
C
C                     i) Preprocessor notation:
C                
C                                       +--------+     ^ S
C                                      /        /|     |
C                                     /    3   / |     |
C                               4--> /        /  |     |
C                                   +--------+ 2 +     +----> R
C                                   |        |  /     /
C                                   |    6   | /     /
C                                   |        |/     /
C                                   +--------+     T
C                                       1
C
C                    ii) Symmetric notation:
C                     
C                                       +--------+     ^ S
C                                      /        /|     |
C                                     /    4   / |     |
C                               1--> /        /  |     |
C                                   +--------+ 2 +     +----> R
C                                   |        |  /     /
C                                   |    6   | /     /
C                                   |        |/     /
C                                   +--------+     T
C                                       3
C
C     EFACE(IFACE)  -   Given face number IFACE in symmetric notation, 
C                       returns preprocessor notation face number.
C
C     EFACE1(IFACE) -   Given face number IFACE in preprocessor notation, 
C                       returns symmetric notation face number.
C
C  The following variables all take the symmetric 
C  notation of IFACE as arguments:
C
C     ICFACE(i,IFACE) -   Gives the 4 vertices which reside on face IFACE
C                         as depicted below, e.g. ICFACE(i,2)=2,4,6,8.
C
C                        3+-----+4    ^ Y
C                        /  2  /|     |
C     Edge 1 extends    /     / |     |
C       from vertex   7+-----+8 +2    +----> X
C       1 to 2.        |  4  | /     /
C                      |     |/     /
C                     5+-----+6    Z
C                         3
C
C     IEDGFC(i,IFACE) -   Gives the 4 edges which border the face IFACE
C                         Edge numbering is as follows:
C                            Edge = 1,2,3,4     run in +r direction
C                            Edge = 5,6,7,8     run in +s direction
C                            Edge = 9,10,11,12  run in +t direction
C
C                         Ordering of each edge is such that a monotonically
C                         increasing sequence of vertices is associated with
C                         the start point of a corresponding set of 
C                         monotonically increasing edge numbers, e.g.,
C
C     ICEDG(i,IEDGE)  -   Gives 3 variables for determining the stride along
C                         a given edge, IEDGE;  i=1 gives the starting vertex
C                                               i=2 gives the stopping vertex
C                                               i=3 gives the stride size.
C
      include 'SIZE'
      include 'TOPOL'
C
      COMMON /ctmp0/ itmp(3,3,3)
      INTEGER ORDER
C
      nxl=3
      nyl=3
      nzl=1+2*(ldim-2)
      nxy   =nxl*nyl
      nxyz  =nxl*nyl*nzl
      nfaces=2*ldim
C
C----------------------------------------------------------------------
C     Set up edge arrays (temporary - required only for defining DS)
C----------------------------------------------------------------------
C
C     Fill corners - 1 through 8.
C
      i3d=1
      IF (ldim.EQ.2) i3d=0
C
      i=0
      DO 10 i3=0,i3d
         iz=1+(nzl-1)*i3
         DO 10 i2=0,1
            iy=1+(nyl-1)*i2
            DO 10 i1=0,1
               ix=1+(nxl-1)*i1
               i=i+1
               iedge(i)=ix+nxl*(iy-1)+nxy*(iz-1)
   10 CONTINUE
C
C     Fill X-direction edges.
C
      DO 20 i3=0,i3d
         iz=1+(nzl-1)*i3
         DO 20 i2=0,1
            iy=1+(nyl-1)*i2
            DO 20 ix=2,nxl-1
               i=i+1
               iedge(i)=ix+nxl*(iy-1)+nxy*(iz-1)
   20 CONTINUE
C
C     Fill Y-direction edges.
C
      DO 30 i3=0,i3d
         iz=1+(nzl-1)*i3
         DO 30 i1=0,1
            ix=1+(nxl-1)*i1
            DO 30 iy=2,nyl-1
               i=i+1
               iedge(i)=ix+nxl*(iy-1)+nxy*(iz-1)
   30 CONTINUE
C
C     Fill Z-direction edges.
C
      IF (ldim.EQ.3) THEN
         DO 40 i2=0,1
            iy=1+(nyl-1)*i2
            DO 40 i1=0,1
               ix=1+(nxl-1)*i1
               DO 40 iz=2,nzl-1
                  i=i+1
                  iedge(i)=ix+nxl*(iy-1)+nxy*(iz-1)
   40    CONTINUE
      ENDIF
C
      CALL izero(invedg,27)
      DO 44 ii=1,20
         ix=iedge(ii)
         invedg(ix)=ii
   44 CONTINUE
C
C
C     GENERAL FACE, GENERAL ROTATION EDGE NUMBERS.
C
      IF (ldim.EQ.3) THEN
C
C        Pack 3-D edge numbering:
C   
C        Fill temporary array with local index numbers:
C
         DO 50 ix=1,nxyz
            itmp(ix,1,1)=ix
   50    CONTINUE
C
C        Two sets are required, the base cube and the image cube 
C        which is being summed with it.
C
         DO 1000 image=0,1
C
C        Pack edges for each face, no rotation.
C
         DO 500 iface=1,nfaces
            js1    = skpdat(1,iface)
            jf1    = skpdat(2,iface)
            jskip1 = skpdat(3,iface)
            js2    = skpdat(4,iface)
            jf2    = skpdat(5,iface)
            jskip2 = skpdat(6,iface)
C
C           Choose proper indexing order according to face type and image.
C
            order = (-1)**(group(iface)+image)
            IF (order.EQ.1) THEN
C
C              Forward ordering:
C
C            +-------------+    ^ v1
C            | --------->| |    |
C            | ^    2    | |    +-->
C            | |         | |      v2
C            | |1       3| |
C            | |    4    V |
C            | |<--------- |
C            F-------------I     F is fiducial node.
C
C                                I is location of fiducial node for
C                                     image face.
C
C           Load edge 1:
C
            j=0
            j2=js2
            DO 100 j1=js1,jf1-jskip1,jskip1
               j=j+1
               iedgef(j,1,iface,image)=itmp(j1,j2,1)
  100       CONTINUE
C
C           Load edge 2:
C
            j=0
            j1=jf1
            DO 200 j2=js2,jf2-jskip2,jskip2
               j=j+1
               iedgef(j,2,iface,image)=itmp(j1,j2,1)
  200       CONTINUE
C
C
C           Load edge 3:
C
            j=0
            j2=jf2
            DO 300 j1=jf1,js1+jskip1,-jskip1
               j=j+1
               iedgef(j,3,iface,image)=itmp(j1,j2,1)
  300       CONTINUE
C
C           Load edge 4:
C
            j=0
            j1=js1
            DO 400 j2=jf2,js2+jskip2,-jskip2
               j=j+1
               iedgef(j,4,iface,image)=itmp(j1,j2,1)
  400       CONTINUE
C
            ELSE
C
C           Reverse ordering:
C
C            +-------------+
C            | |<--------- |       ^ v2
C            | |    2    ^ |       |
C            | |         | |    <--+
C            | |3       1| |     v1
C            | V    4    | |
C            | --------->| |
C            I-------------F     F is fiducial node.
C
C                                I is location of fiducial node for
C                                     image face.
C
C           Load edge 1:
C   
            j=0
            j1=js1
            DO 105 j2=js2,jf2-jskip2,jskip2
               j=j+1
               iedgef(j,1,iface,image)=itmp(j1,j2,1)
  105       CONTINUE
C
C           Load edge 2:
C
            j=0
            j2=jf2
            DO 205 j1=js1,jf1-jskip1,jskip1
               j=j+1
               iedgef(j,2,iface,image)=itmp(j1,j2,1)
  205       CONTINUE
C
C           Load edge 3:
C
            j=0
            j1=jf1
            DO 305 j2=jf2,js2+jskip2,-jskip2
                j=j+1
               iedgef(j,3,iface,image)=itmp(j1,j2,1)
  305       CONTINUE
C
C           Load edge 4:
C
            j=0
            j2=js2
            DO 405 j1=jf1,js1+jskip1,-jskip1
               j=j+1
              iedgef(j,4,iface,image)=itmp(j1,j2,1)
  405       CONTINUE
            ENDIF
C
  500    CONTINUE
 1000    CONTINUE
      ELSE
C
C        Load edge information for 2-D case
C
         iedgef(1,1,1,0) = nxy - nxl + 1
         iedgef(1,2,1,0) = 1
         iedgef(1,1,2,0) = nxl
         iedgef(1,2,2,0) = nxy
         iedgef(1,1,3,0) = 1
         iedgef(1,2,3,0) = nxl
         iedgef(1,1,4,0) = nxy
         iedgef(1,2,4,0) = nxy - nxl + 1
C
         iedgef(1,1,1,1) = 1
         iedgef(1,2,1,1) = nxy - nxl + 1
         iedgef(1,1,2,1) = nxy
         iedgef(1,2,2,1) = nxl
         iedgef(1,1,3,1) = nxl
         iedgef(1,2,3,1) = 1
         iedgef(1,1,4,1) = nxy - nxl + 1
         iedgef(1,2,4,1) = nxy
      ENDIF
C
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine dsset(nx,ny,nz)
C
C     Set up arrays IXCN,ESKIP,SKPDAT,NEDG,NOFFST for new NX,NY,NZ
C
      include 'SIZE'
      include 'INPUT'
      include 'TOPOL'
      INTEGER NXO,NYO,NZO
      SAVE    nxo,nyo,nzo
      DATA    nxo,nyo,nzo /3*0/
C
C     Check if element surface counters are already set from last call...
C
      IF (nxo.EQ.nx.AND.nyo.EQ.ny.AND.nzo.EQ.nz) RETURN
C
C     else, proceed....
C
      nxo = nx
      nyo = ny
      nzo = nz
C
C     Establish corner to elemental node number mappings
C
      ic=0
      DO 10 icz=0,1
      DO 10 icy=0,1
      DO 10 icx=0,1
C       Supress vectorization to 
c        IF(ICX.EQ.0)DUMMY=0        
c        IF(ICX.EQ.1)DUMMY=1
c        DUMMY2=DUMMY2+DUMMY
        ic=ic+1
        ixcn(ic)= 1 + (nx-1)*icx + nx*(ny-1)*icy + nx*ny*(nz-1)*icz
   10   CONTINUE
C
C     Assign indices for direct stiffness summation of arbitrary faces.
C
C
C     Y-Z Planes (Faces 1 and 2)
C
      skpdat(1,1)=1
      skpdat(2,1)=nx*(ny-1)+1
      skpdat(3,1)=nx
      skpdat(4,1)=1
      skpdat(5,1)=ny*(nz-1)+1
      skpdat(6,1)=ny
C
      skpdat(1,2)=1             + (nx-1)
      skpdat(2,2)=nx*(ny-1)+1   + (nx-1)
      skpdat(3,2)=nx
      skpdat(4,2)=1
      skpdat(5,2)=ny*(nz-1)+1
      skpdat(6,2)=ny
C
C     X-Z Planes (Faces 3 and 4)
C
      skpdat(1,3)=1
      skpdat(2,3)=nx
      skpdat(3,3)=1
      skpdat(4,3)=1
      skpdat(5,3)=ny*(nz-1)+1
      skpdat(6,3)=ny
C
      skpdat(1,4)=1           + nx*(ny-1)
      skpdat(2,4)=nx          + nx*(ny-1)
      skpdat(3,4)=1
      skpdat(4,4)=1
      skpdat(5,4)=ny*(nz-1)+1
      skpdat(6,4)=ny
C
C     X-Y Planes (Faces 5 and 6)
C
      skpdat(1,5)=1
      skpdat(2,5)=nx
      skpdat(3,5)=1
      skpdat(4,5)=1
      skpdat(5,5)=ny
      skpdat(6,5)=1
C
      skpdat(1,6)=1           + nx*ny*(nz-1)
      skpdat(2,6)=nx          + nx*ny*(nz-1)
      skpdat(3,6)=1
      skpdat(4,6)=1
      skpdat(5,6)=ny
      skpdat(6,6)=1
C
C     Set up skip indices for each of the 12 edges 
C
C         Note that NXY = NX*NY even for 2-D since
C         this branch does not apply to the 2D case anyway.
C
C     ESKIP(*,1) = start location
C     ESKIP(*,2) = end 
C     ESKIP(*,3) = stride
C
      nxy=nx*ny
      eskip( 1,1) = ixcn(1) + 1
      eskip( 1,2) = ixcn(2) - 1
      eskip( 1,3) = 1
      eskip( 2,1) = ixcn(3) + 1
      eskip( 2,2) = ixcn(4) - 1
      eskip( 2,3) = 1
      eskip( 3,1) = ixcn(5) + 1
      eskip( 3,2) = ixcn(6) - 1
      eskip( 3,3) = 1
      eskip( 4,1) = ixcn(7) + 1
      eskip( 4,2) = ixcn(8) - 1
      eskip( 4,3) = 1
      eskip( 5,1) = ixcn(1) + nx
      eskip( 5,2) = ixcn(3) - nx
      eskip( 5,3) = nx
      eskip( 6,1) = ixcn(2) + nx
      eskip( 6,2) = ixcn(4) - nx
      eskip( 6,3) = nx
      eskip( 7,1) = ixcn(5) + nx
      eskip( 7,2) = ixcn(7) - nx
      eskip( 7,3) = nx
      eskip( 8,1) = ixcn(6) + nx
      eskip( 8,2) = ixcn(8) - nx
      eskip( 8,3) = nx
      eskip( 9,1) = ixcn(1) + nxy
      eskip( 9,2) = ixcn(5) - nxy
      eskip( 9,3) = nxy
      eskip(10,1) = ixcn(2) + nxy
      eskip(10,2) = ixcn(6) - nxy
      eskip(10,3) = nxy
      eskip(11,1) = ixcn(3) + nxy
      eskip(11,2) = ixcn(7) - nxy
      eskip(11,3) = nxy
      eskip(12,1) = ixcn(4) + nxy
      eskip(12,2) = ixcn(8) - nxy
      eskip(12,3) = nxy
C
C     Load reverse direction edge arrays for reverse mappings...
C
      DO 20 ied=1,12
      iedm=-ied
      eskip(iedm,1) =  eskip(ied,2)
      eskip(iedm,2) =  eskip(ied,1)
      eskip(iedm,3) = -eskip(ied,3)
   20 CONTINUE
C
C     Compute offset for global edge vector given current element
C     dimensions.....
C
C     NGSPED(ITE,ICMP) = number of global (ie, distinct) special edges
C                        of type ITE (1,2, or 3)  for field ICMP.
C
C                        ITE = 1 implies an "X" edge
C                        ITE = 2 implies an "Y" edge
C                        ITE = 3 implies an "Z" edge
C
C     Set up number of nodes along each of the 3 types of edges
C     (endpoints excluded).
C
      nedg(1)=nx-2
      nedg(2)=ny-2
      nedg(3)=nz-2
C
C
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine genxyzl
C
C     Generate xyz coordinates 
C
      include 'SIZE'
      include 'INPUT'
      include 'SCRCT'
      COMMON /ctmp0/ xcb(2,2,2),ycb(2,2,2),zcb(2,2,2),h(3,3,2),indx(8)
C
      nxl=3
      nyl=3
      nzl=1+2*(ldim-2)
      ntot3=nxl*nyl*nzl*nelt
C
C   Preprocessor Corner notation:      Symmetric Corner notation:
C                
C           4+-----+3    ^ s                    3+-----+4    ^ s        
C           /     /|     |                      /     /|     |          
C          /     / |     |                     /     / |     |          
C        8+-----+7 +2    +----> r            7+-----+8 +2    +----> r   
C         |     | /     /                     |     | /     /           
C         |     |/     /                      |     |/     /            
C        5+-----+6    t                      5+-----+6    t             
C
      DO 10 ix=1,nxl
         h(ix,1,1)=0.5*float(3-ix)
         h(ix,1,2)=0.5*float(ix-1)
   10 CONTINUE
      DO 20 iy=1,nyl
         h(iy,2,1)=0.5*float(3-iy)
         h(iy,2,2)=0.5*float(iy-1)
   20 CONTINUE
      DO 30 iz=1,nzl
         h(iz,3,1)=0.5*float(3-iz)
         h(iz,3,2)=0.5*float(iz-1)
   30 CONTINUE
C
      indx(1)=1
      indx(2)=2
      indx(3)=4
      indx(4)=3
      indx(5)=5
      indx(6)=6
      indx(7)=8
      indx(8)=7
C
      CALL rzero(xml,ntot3)
      CALL rzero(yml,ntot3)
      CALL rzero(zml,ntot3)
      CALL rzero(xcb,8)
      CALL rzero(ycb,8)
      CALL rzero(zcb,8)
C
      DO 5000 ie=1,nelt
C
         ldim2 = 2**ldim
         DO 50 ix=1,ldim2
            i=indx(ix)
            xcb(ix,1,1)=xc(i,ie)
            ycb(ix,1,1)=yc(i,ie)
            zcb(ix,1,1)=zc(i,ie)
   50    CONTINUE
C
C        Map R-S-T space into physical X-Y-Z space.
C
         DO 100 izt=1,ldim-1
         DO 100 iyt=1,2
         DO 100 ixt=1,2
C
         DO 100 iz=1,nzl
         DO 100 iy=1,nyl
         DO 100 ix=1,nxl
            xml(ix,iy,iz,ie)=xml(ix,iy,iz,ie)+
     $        h(ix,1,ixt)*h(iy,2,iyt)*h(iz,3,izt)*xcb(ixt,iyt,izt)
            yml(ix,iy,iz,ie)=yml(ix,iy,iz,ie)+
     $        h(ix,1,ixt)*h(iy,2,iyt)*h(iz,3,izt)*ycb(ixt,iyt,izt)
            zml(ix,iy,iz,ie)=zml(ix,iy,iz,ie)+
     $        h(ix,1,ixt)*h(iy,2,iyt)*h(iz,3,izt)*zcb(ixt,iyt,izt)
  100    CONTINUE
C
 5000 CONTINUE
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine verify
C
C     .Verify right-handedness of elements.
C     .Verify element-to-element reciprocity of BC's
C     .Verify correlation between E-E BC's and physical coincidence
C
      include 'SIZE'
      include 'PARALLEL'
      include 'INPUT'
      include 'SCRCT'
 
      call verrhe
 
      return
      end
c-----------------------------------------------------------------------
      subroutine setside
      include 'SIZE'
      include 'INPUT'
      include 'TOPOL'
      include 'SCRCT'
C
C     SIDE(i,IFACE,IE) -  Physical (xyz) location of element side midpoint.
C                         i=1,2,3 gives x,y,z value, respectively.
C                         i=4  gives average dimension of face for setting
C                              tolerances.
C
      indx(1)=1
      indx(2)=2
      indx(3)=4
      indx(4)=3
      indx(5)=5
      indx(6)=6
      indx(7)=8
      indx(8)=7
C
C     Flip vertex array structure
C
c     write(6,*) nelv,nelt,if3d
      call rzero(xyz,24*nelt)
      if (if3d) then
         do ie=1,nelt
         do j=1,8
            ivtx = indx(j)
            xyz(1,ivtx,ie) = xc(j,ie)
            xyz(2,ivtx,ie) = yc(j,ie)
            xyz(3,ivtx,ie) = zc(j,ie)
c           write(6,1) ie,j,ivtx,xc(j,ie),yc(j,ie),zc(j,ie),' xcz'
c           write(6,1) ie,j,ivtx,(xyz(k,ivtx,ie),k=1,3),' vtx'
c   1       format(3i5,1p3e12.4,a4)
         enddo
         enddo
      else
         do ie=1,nelt
         do j=1,4
            ivtx = indx(j)
            xyz(1,ivtx,ie) = xc(j,ie)
            xyz(2,ivtx,ie) = yc(j,ie)
            xyz(3,ivtx,ie) = 0.0
         enddo
         enddo
      endif
C
C     Compute location of center and "diameter" of each element side.
C
      nfaces=ldim*2
      ncrnr =2**(ldim-1)
      CALL rzero(side,24*nelt)
      DO 500 icrn=1,ncrnr
      DO 500 ifac=1,nfaces
         ivtx = icface(icrn,ifac)
         icr1 = ncrnr+(icrn-1)
         icr1 = mod1(icr1,ncrnr)
         ivt1 = icface(icr1,ifac)
         DO 400 ie=1,nelt
            DO 300 idim=1,ldim
               side(idim,ifac,ie)=side(idim,ifac,ie)+xyz(idim,ivtx,ie)
               side(   4,ifac,ie)=side(   4,ifac,ie)+
     $                     ( xyz(idim,ivtx,ie)-xyz(idim,ivt1,ie) )**2
  300       CONTINUE
            side(4,ifac,ie)=sqrt( side(4,ifac,ie) )
  400    CONTINUE
  500 CONTINUE
      avwght=1.0/float(ncrnr)
      CALL cmult(side,avwght,24*nelt)
C
c     call exitt
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine verrhe
C
C     8 Mar 1989 21:58:26   PFF
C     Verify right-handedness of given elements. 
C
      include 'SIZE'
      include 'INPUT'
      include 'PARALLEL'
      include 'SCRCT'
      include 'TOPOL'
      LOGICAL IFYES,IFCSTT
C
      ifcstt=.true.
      IF (.NOT.if3d) THEN
      DO 1000 ie=1,nelt
C
C        CRSS2D(A,B,O) = (A-O) X (B-O)
C
         c1=crss2d(xyz(1,2,ie),xyz(1,3,ie),xyz(1,1,ie))
         c2=crss2d(xyz(1,4,ie),xyz(1,1,ie),xyz(1,2,ie))
         c3=crss2d(xyz(1,1,ie),xyz(1,4,ie),xyz(1,3,ie))
         c4=crss2d(xyz(1,3,ie),xyz(1,2,ie),xyz(1,4,ie))
C
         IF (c1.LE.0.0.OR.c2.LE.0.0.OR.
     $       c3.LE.0.0.OR.c4.LE.0.0 ) THEN
C
            ieg=lglel(ie)
            WRITE(6,800) ieg,c1,c2,c3,c4
            call exitt
  800       FORMAT(/,2x,'WARNINGa: Detected non-right-handed element.',
     $      /,2x,'Number',i8,'  C1-4:',4e12.4)
            ifcstt=.false.
C           CALL QUERY(IFYES,'Proceed                                 ')
C           IF (.NOT.IFYES) GOTO 9000
         ENDIF
 1000 CONTINUE
C
C     Else 3-D:
C
      ELSE      
      DO 2000 ie=1,nelt
C
C        VOLUM0(A,B,C,O) = (A-O)X(B-O).(C-O)
C
         v1= volum0(xyz(1,2,ie),xyz(1,3,ie),xyz(1,5,ie),xyz(1,1,ie))
         v2= volum0(xyz(1,4,ie),xyz(1,1,ie),xyz(1,6,ie),xyz(1,2,ie))
         v3= volum0(xyz(1,1,ie),xyz(1,4,ie),xyz(1,7,ie),xyz(1,3,ie))
         v4= volum0(xyz(1,3,ie),xyz(1,2,ie),xyz(1,8,ie),xyz(1,4,ie))
         v5=-volum0(xyz(1,6,ie),xyz(1,7,ie),xyz(1,1,ie),xyz(1,5,ie))
         v6=-volum0(xyz(1,8,ie),xyz(1,5,ie),xyz(1,2,ie),xyz(1,6,ie))
         v7=-volum0(xyz(1,5,ie),xyz(1,8,ie),xyz(1,3,ie),xyz(1,7,ie))
         v8=-volum0(xyz(1,7,ie),xyz(1,6,ie),xyz(1,4,ie),xyz(1,8,ie))
C
         IF (v1.LE.0.0.OR.v2.LE.0.0.OR.
     $       v3.LE.0.0.OR.v4.LE.0.0.OR.
     $       v5.LE.0.0.OR.v6.LE.0.0.OR.
     $       v7.LE.0.0.OR.v8.LE.0.0    ) THEN
C
            ieg=lglel(ie)
            WRITE(6,1800) ieg,v1,v2,v3,v4,v5,v6,v7,v8
            call exitt
 1800       FORMAT(/,2x,'WARNINGb: Detected non-right-handed element.',
     $      /,2x,'Number',i8,'  V1-8:',4e12.4
     $      /,2x,'      ',4x,'       ',4e12.4)
            ifcstt=.false.
         ENDIF
 2000 CONTINUE
      ENDIF
C
 9000 CONTINUE
C
C     Print out results from right-handed check
C
      IF (.NOT.ifcstt) WRITE(6,2001)
C
C     Check consistency accross all processors.
C
      CALL gllog(ifcstt,.false.)
C
      IF (.NOT.ifcstt) THEN
         IF (nid.EQ.0) WRITE(6,2003) nelgt
         call exitt
      ELSE
         IF (nio.EQ.0) WRITE(6,2002) nelgt
      ENDIF
C
 2001 FORMAT(//,'  Elemental geometry not right-handed, ABORTING'
     $      ,' in routine VERRHE.')
 2002 FORMAT('   Right-handed check complete for',i12,' elements. OK.')
 2003 FORMAT('   Right-handed check failed for',i12,' elements.'
     $      ,'   Exiting in routine VERRHE.')
      RETURN
      END
c-----------------------------------------------------------------------
      FUNCTION volum0(P1,P2,P3,P0)
C
C                           3
C     Given four points in R , (P1,P2,P3,P0), VOLUM0 returns
C     the volume enclosed by the parallelagram defined by the
C     vectors { (P1-P0),(P2-P0),(P3-P0) }.  This routine has
C     the nice feature that if the 3 vectors so defined are
C     not right-handed then the volume returned is negative.
C
      REAL p1(3),p2(3),p3(3),p0(3)
C
         u1=p1(1)-p0(1)
         u2=p1(2)-p0(2)
         u3=p1(3)-p0(3)
C
         v1=p2(1)-p0(1)
         v2=p2(2)-p0(2)
         v3=p2(3)-p0(3)
C
         w1=p3(1)-p0(1)
         w2=p3(2)-p0(2)
         w3=p3(3)-p0(3)
C
         cross1 = u2*v3-u3*v2
         cross2 = u3*v1-u1*v3
         cross3 = u1*v2-u2*v1
C
         volum0  = w1*cross1 + w2*cross2 + w3*cross3
         
      RETURN
      END
c-----------------------------------------------------------------------
      FUNCTION crss2d(XY1,XY2,XY0)
      REAL xy1(2),xy2(2),xy0(2)
C
         v1x=xy1(1)-xy0(1)
         v2x=xy2(1)-xy0(1)
         v1y=xy1(2)-xy0(2)
         v2y=xy2(2)-xy0(2)
         crss2d = v1x*v2y - v1y*v2x
C
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine facind (kx1,kx2,ky1,ky2,kz1,kz2,nx,ny,nz,iface)
c      ifcase in preprocessor notation
       kx1=1
       ky1=1
       kz1=1
       kx2=nx
       ky2=ny
       kz2=nz
       IF (iface.EQ.1) ky2=1
       IF (iface.EQ.2) kx1=nx
       IF (iface.EQ.3) ky1=ny
       IF (iface.EQ.4) kx2=1
       IF (iface.EQ.5) kz2=1
       IF (iface.EQ.6) kz1=nz
      return
      end
c-----------------------------------------------------------------------
      subroutine facindr (kx1,kx2,ky1,ky2,kz1,kz2,nx,ny,nz,iface)
 
c     restricted index set, iface in preprocessor notation
 
      kx1=2
      ky1=2
      kz1=2
      kx2=nx-1
      ky2=ny-1
      kz2=nz-1
 
      if (iface.eq.1) ky1=1
      if (iface.eq.1) ky2=1
 
      if (iface.eq.2) kx1=nx
      if (iface.eq.2) kx2=nx
 
      if (iface.eq.3) ky1=ny
      if (iface.eq.3) ky2=ny
 
      if (iface.eq.4) kx1=1
      if (iface.eq.4) kx2=1
 
      if (iface.eq.5) kz1=1
      if (iface.eq.5) kz2=1
 
      if (iface.eq.6) kz1=nz
      if (iface.eq.6) kz2=nz
 
      return
      end
c-----------------------------------------------------------------------
      subroutine facev(a,ie,iface,val,nx,ny,nz)
C
C     Assign the value VAL to face(IFACE,IE) of array A.
C     IFACE is the input in the pre-processor ordering scheme.
C
      include 'SIZE'
      dimension a(nx,ny,nz,lelt)
      CALL facind (kx1,kx2,ky1,ky2,kz1,kz2,nx,ny,nz,iface)
      DO 100 iz=kz1,kz2
      DO 100 iy=ky1,ky2
      DO 100 ix=kx1,kx2
         a(ix,iy,iz,ie)=val
  100 CONTINUE
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine ifacev(a,ie,iface,val,nx,ny,nz)
C
C     Assign the value VAL to face(IFACE,IE) of array A.
C     IFACE is the input in the pre-processor ordering scheme.
C
      include 'SIZE'
      integer a(nx,ny,nz,lelt),val
      call facind (kx1,kx2,ky1,ky2,kz1,kz2,nx,ny,nz,iface)
      do 100 iz=kz1,kz2
      do 100 iy=ky1,ky2
      do 100 ix=kx1,kx2
         a(ix,iy,iz,ie)=val
  100 continue
      return
      end
c-----------------------------------------------------------------------
      subroutine facec(a,b,ie,iface,nx,ny,nz,nel)
C
C     Copy the face (IFACE) of B to A.
C     IFACE is the input in the pre-processor ordering scheme.
C
      dimension a(nx,ny,nz,nel)
      dimension b(nx,ny,nz,nel)
      CALL facind (kx1,kx2,ky1,ky2,kz1,kz2,nx,ny,nz,iface)
      DO 100 iz=kz1,kz2
      DO 100 iy=ky1,ky2
      DO 100 ix=kx1,kx2
         a(ix,iy,iz,ie)=b(ix,iy,iz,ie)
  100 CONTINUE
      RETURN
      END
c-----------------------------------------------------------------------
      subroutine combin2(glnm1,glnm2,nglob)
c
      write(6,*) 'Hey, who called combin2???  ABORT'
      call exitt
c
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldio (x,txt10)
      include 'SIZE'
      integer x(lx1,ly1,lz1,lelt)
      character*10 txt10
C
      do ie=1,nelv,2
         do iz=1,lz1,1
            if (iz.eq.1) write(6,106) txt10,iz,ie
            if (iz.gt.1) write(6,107) 
            i1 = ie+1
            do j=ly1,1,-1
               write(6,105) (x(i,j,iz,ie),i=1,lx1)
     $                    , (x(i,j,iz,i1),i=1,lx1)
            enddo
         enddo
      enddo
C
  107 FORMAT(' ')
  105 FORMAT(4i6,20x,4i6)
  106 FORMAT(  /,5x,'     ^              ',/,
     $           5x,'   Y |              ',/,
     $           5x,'     |              ',a10,/,
     $           5x,'     +---->         ','Plane = ',i2,'/',i2,/,
     $           5x,'       X            ')
C
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldi (x,txt10)
      include 'SIZE'
      integer x(lx1,ly1,lz1,lelt)
      character*10 txt10
c
      character*6 s(20,20)
c
      if (lx1.ne.4 .or. nelv.gt.3) return
c
      write(6,106) txt10,ie,ie
  106 FORMAT(  /,5x,'     ^              ',/,
     $           5x,'   Y |              ',/,
     $           5x,'     |              ',a10,/,
     $           5x,'     +---->         ','elem. = ',i2,'/',i2,/,
     $           5x,'       X            ')
C
C
      call blank(s,6*20*20)
      do ie=1,3
         if (ie.eq.1) then
            jstart  = 1
            istart  = 1
            istride = 3
         else
            jstart  = 2 + lx1
            istart  = 1
            istride = 1
            if (ie.eq.2) istart = 7
         endif
c
         i=istart
         do iy=ly1,1,-1
            j=jstart
            do ix=1,lx1
               write(s(i,j),6) x(ix,iy,1,ie)
               j=j+1
            enddo
            i=i+istride
         enddo
    6    format(20i6)
      enddo
c
      do i=1,10
         write(6,7) (s(i,l),l=1,j-1)
      enddo
    7 format(20a6)
c
      write(6,*)
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldr (x,txt10)
      include 'SIZE'
      real x(lx1,ly1,lz1,lelt)
      character*10 txt10
c
      character*6 s(20,20)
c
      if (lx1.ne.4 .or. nelv.gt.3) return
      write(6,106) txt10,ie,ie
  106 FORMAT(  /,5x,'     ^              ',/,
     $           5x,'   Y |              ',/,
     $           5x,'     |              ',a10,/,
     $           5x,'     +---->         ','elem. = ',i2,'/',i2,/,
     $           5x,'       X            ')
      call blank(s,6*20*20)
C
C
      do ie=1,3
         if (ie.eq.1) then
            jstart  = 1
            istart  = 1
            istride = 3
         else
            jstart  = 2 + lx1
            istart  = 1
            istride = 1
            if (ie.eq.2) istart = 7
         endif
c
         i=istart
         do iy=ly1,1,-1
            j=jstart
            do ix=1,lx1
               write(s(i,j),6) x(ix,iy,1,ie)
               j=j+1
            enddo
            i=i+istride
         enddo
    6    format(f6.2)
      enddo
c
      do i=1,10
         write(6,7) (s(i,l),l=1,j-1)
      enddo
    7 format(20a6)
c
      write(6,*)
c
      return
      end
c-----------------------------------------------------------------------
      subroutine checkit(idum)
      write(6,*) 'continue?'
      read (5,*) idum
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldro (x,txt10,ichk)
      include 'SIZE'
      include 'TSTEP'
      real x(lx1,ly1,lz1,lelt)
      character*10 txt10
c
      integer idum,e
      save idum
      data idum /3/
      if (idum.lt.0) return
c
C
      mtot = lx1*ly1*lz1*nelv
      if (lx1.gt.8.or.nelv.gt.16) return
      xmin = glmin(x,mtot)
      xmax = glmax(x,mtot)
 
      do ie=1,1
         ne = 1
         do k=1,1
            write(6,116) txt10,k,ie,xmin,xmax,istep,time
            write(6,117) 
            do j=ly1,1,-1
              if (lx1.eq.2) write(6,102) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.3) write(6,103) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.4) write(6,104) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.5) write(6,105) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.6) write(6,106) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.7) write(6,107) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
              if (lx1.eq.8) write(6,118) ((x(i,j,k,e+1),i=1,lx1),e=0,ne)
            enddo
         enddo
      enddo
 
  102 FORMAT(4(2f9.5,2x))
  103 FORMAT(4(3f9.5,2x))
  104 FORMAT(4(4f7.3,2x))
  105 FORMAT(5f9.5,10x,5f9.5)
  106 FORMAT(6f9.5,5x,6f9.5)
  107 FORMAT(7f8.4,5x,7f8.4)
  108 FORMAT(8f8.4,4x,8f8.4)
  118 FORMAT(8f12.9)
c
  116 FORMAT(  /,5x,'     ^              ',/,
     $    5x,'   Y |              ',/,
     $    5x,'     |              ',a10,/,
     $    5x,'     +---->         ','Plane = ',i2,'/',i2,2x,2e12.4,/,
     $    5x,'       X            ','Step  =',i9,f15.5)
  117 FORMAT(' ')
c
      if (ichk.eq.1.and.idum.gt.0) call checkit(idum)
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldrv (x,txt10,ichk) ! writes to unit=40+ifield
      include 'SIZE'
      include 'TSTEP'
      real x(lx1,ly1,lz1,lelt)
      character*10 txt10
c
      integer idum,e
      save idum
      data idum /3/
      if (idum.lt.0) return
      m = 40 + ifield
c
C
      mtot = lx1*ly1*lz1*nelv
      if (lx1.gt.7.or.nelv.gt.16) return
      xmin = glmin(x,mtot)
      xmax = glmax(x,mtot)
c
      rnel = nelv
      snel = sqrt(rnel)+.1
      ne   = snel
      ne1  = nelv-ne+1
      do ie=ne1,1,-ne
         l=ie-1
         do k=1,1
            if (ie.eq.ne1) write(m,116) txt10,k,ie,xmin,xmax,istep,time
            write(m,117) 
            do j=ly1,1,-1
              if (lx1.eq.2) write(m,102) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.3) write(m,103) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.4) write(m,104) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.5) write(m,105) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.6) write(m,106) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.7) write(m,107) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.8) write(m,108) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
            enddo
         enddo
      enddo
 
C
  102 FORMAT(4(2f9.5,2x))
  103 FORMAT(4(3f9.5,2x))
  104 FORMAT(4(4f7.3,2x))
  105 FORMAT(5f9.5,10x,5f9.5)
  106 FORMAT(6f9.5,5x,6f9.5)
  107 FORMAT(7f8.4,5x,7f8.4)
  108 FORMAT(8f8.4,4x,8f8.4)
c
  116 FORMAT(  /,5x,'     ^              ',/,
     $    5x,'   Y |              ',/,
     $    5x,'     |              ',a10,/,
     $    5x,'     +---->         ','Plane = ',i2,'/',i2,2x,2e12.4,/,
     $    5x,'       X            ','Step  =',i9,f15.5)
  117 FORMAT(' ')
c
      if (ichk.eq.1.and.idum.gt.0) call checkit(idum)
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldrv0 (x,txt10,ichk)
      include 'SIZE'
      include 'TSTEP'
      real x(lx1,ly1,lz1,lelt)
      character*10 txt10
c
      integer idum,e
      save idum
      data idum /3/
      if (idum.lt.0) return
c
C
      mtot = lx1*ly1*lz1*nelv
      if (lx1.gt.7.or.nelv.gt.16) return
      xmin = glmin(x,mtot)
      xmax = glmax(x,mtot)
c
      rnel = nelv
      snel = sqrt(rnel)+.1
      ne   = snel
      ne1  = nelv-ne+1
      do ie=ne1,1,-ne
         l=ie-1
         do k=1,1
            if (ie.eq.ne1) write(6,116) txt10,k,ie,xmin,xmax,istep,time
            write(6,117) 
            do j=ly1,1,-1
              if (lx1.eq.2) write(6,102) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.3) write(6,103) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.4) write(6,104) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.5) write(6,105) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.6) write(6,106) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.7) write(6,107) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
              if (lx1.eq.8) write(6,108) ((x(i,j,k,e+l),i=1,lx1),e=1,ne)
            enddo
         enddo
      enddo
 
C
  102 FORMAT(4(2f9.5,2x))
  103 FORMAT(4(3f9.5,2x))
  104 FORMAT(4(4f7.3,2x))
  105 FORMAT(5f9.5,10x,5f9.5)
  106 FORMAT(6f9.5,5x,6f9.5)
  107 FORMAT(7f8.4,5x,7f8.4)
  108 FORMAT(8f8.4,4x,8f8.4)
c
  116 FORMAT(  /,5x,'     ^              ',/,
     $    5x,'   Y |              ',/,
     $    5x,'     |              ',a10,/,
     $    5x,'     +---->         ','Plane = ',i2,'/',i2,2x,2e12.4,/,
     $    5x,'       X            ','Step  =',i9,f15.5)
  117 FORMAT(' ')
c
      if (ichk.eq.1.and.idum.gt.0) call checkit(idum)
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldrp0 (x,txt10,ichk)
      include 'SIZE'
      include 'TSTEP'
      real x(lx2,ly2,lz2,lelt)
      character*10 txt10
c
      integer idum,e
      save idum
      data idum /3/
      if (idum.lt.0) return
c
C
      mtot = lx2*ly2*lz2*nelv
      if (lx2.gt.7.or.nelv.gt.16) return
      xmin = glmin(x,mtot)
      xmax = glmax(x,mtot)
c
      rnel = nelv
      snel = sqrt(rnel)+.1
      ne   = snel
      ne1  = nelv-ne+1
      do ie=ne1,1,-ne
         l=ie-1
         do k=1,1
            if (ie.eq.ne1) write(6,116) txt10,k,ie,xmin,xmax,istep,time
            write(6,117) 
            do j=ly2,1,-1
              if (lx2.eq.2) write(6,102) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.3) write(6,103) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.4) write(6,104) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.5) write(6,105) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.6) write(6,106) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.7) write(6,107) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.8) write(6,108) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
            enddo
         enddo
      enddo
 
C
  102 FORMAT(4(2f9.5,2x))
  103 FORMAT(4(3f9.5,2x))
  104 FORMAT(4(4f7.3,2x))
  105 FORMAT(5f9.5,10x,5f9.5)
  106 FORMAT(6f9.5,5x,6f9.5)
  107 FORMAT(7f8.4,5x,7f8.4)
  108 FORMAT(8f8.4,4x,8f8.4)
c
  116 FORMAT(  /,5x,'     ^              ',/,
     $    5x,'   Y |              ',/,
     $    5x,'     |              ',a10,/,
     $    5x,'     +---->         ','Plane = ',i2,'/',i2,2x,2e12.4,/,
     $    5x,'       X            ','Step  =',i9,f15.5)
  117 FORMAT(' ')
c
      if (ichk.eq.1.and.idum.gt.0) call checkit(idum)
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldrp (x,txt10,ichk) ! writes out into unit = 40+ifield 
      include 'SIZE'
      include 'TSTEP'
      real x(lx2,ly2,lz2,lelt)
      character*10 txt10
c
      integer idum,e
      save    idum
      data    idum /3/
      if (idum.lt.0)   return
      m = 40 + ifield             ! unit #
c
c
C
      mtot = lx2*ly2*lz2*nelv
      if (lx2.gt.7.or.nelv.gt.16) return
      xmin = glmin(x,mtot)
      xmax = glmax(x,mtot)
c
      rnel = nelv
      snel = sqrt(rnel)+.1
      ne   = snel
      ne1  = nelv-ne+1
      do ie=ne1,1,-ne
         l=ie-1
         do k=1,1
            if (ie.eq.ne1) write(m,116) txt10,k,ie,xmin,xmax,istep,time
            write(6,117) 
            do j=ly2,1,-1
              if (lx2.eq.2) write(m,102) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.3) write(m,103) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.4) write(m,104) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.5) write(m,105) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.6) write(m,106) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.7) write(m,107) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
              if (lx2.eq.8) write(m,108) ((x(i,j,k,e+l),i=1,lx2),e=1,ne)
            enddo
         enddo
      enddo
 
C
  102 FORMAT(4(2f9.5,2x))
  103 FORMAT(4(3f9.5,2x))
  104 FORMAT(4(4f7.3,2x))
  105 FORMAT(5f9.5,10x,5f9.5)
  106 FORMAT(6f9.5,5x,6f9.5)
  107 FORMAT(7f8.4,5x,7f8.4)
  108 FORMAT(8f8.4,4x,8f8.4)
c
  116 FORMAT(  /,5x,'     ^              ',/,
     $    5x,'   Y |              ',/,
     $    5x,'     |              ',a10,/,
     $    5x,'     +---->         ','Plane = ',i2,'/',i2,2x,2e12.4,/,
     $    5x,'       X            ','Step  =',i9,f15.5)
  117 FORMAT(' ')
c
      if (ichk.eq.1.and.idum.gt.0) call checkit(idum)
      return
      end
c-----------------------------------------------------------------------
      subroutine outmatp(a,m,n,name6,ie)
      include 'SIZE'
      include 'PARALLEL'
      real a(m,n)
      character*6 name6
c
      if (nelv.gt.1) return
      do jid=0,np-1
        imax = iglmax(jid,1)
        if (jid.eq.nid) then
           write(6,*) 
           write(6,*) nid,ie,' matrix: ',name6,m,n
           n12 = min(n,12)
           do i=1,m
              write(6,6) ie,name6,(a(i,j),j=1,n12)
           enddo
    6      format(i3,1x,a6,12f9.5)
           write(6,*) 
           call flush_io()
         endif
        imax = iglmax(jid,1)
      enddo
      return
      end
c-----------------------------------------------------------------------
      subroutine gs_chkr(glo_num)
      include 'SIZE'
      include 'TOTAL'
 
      integer glo_num(lx1,ly1,lz1,lelt)
      integer e,eg
 
      do ipass = 1,2
         iquick = 0
         if (nid.eq.0) iquick=glo_num(ipass,1,1,1)
         iquick = iglmax(iquick,1)
 
         do e=1,nelv
         do k=1,lz1
         do j=1,ly1
         do i=1,lx1
           if (glo_num(i,j,k,e).eq.iquick) then
            eg = lglel(e)
            write(6,1) nid,i,j,k,e,eg,iquick,ipass
     $      ,xm1(i,j,k,e),ym1(i,j,k,e),zm1(i,j,k,e)
  1         format(i12,3i4,2i12,i12,i2,1p3e12.4,' iquick')
          endif
         enddo
         enddo
         enddo
         enddo
      enddo
 
      return
      end
c-----------------------------------------------------------------------
      subroutine setup_mesh_dssum ! Set up dssum for mesh
 
      include 'SIZE'
      include 'TOTAL'
      include 'NONCON'
 
      common /c_is1/ glo_num(1*lx1*ly1*lz1*lelv)
      integer*8 glo_num
      common /ivrtx/ vertex((2**ldim)*lelt)
      integer vertex
 
      parameter(lxyz=lx1*ly1*lz1)
      common /scrns/ enum(lxyz,lelt)
     $             ,  rnx(lxyz,lelt) , rny(lxyz,lelt) , rnz(lxyz,lelt)
     $             ,  tnx(lxyz,lelt) , tny(lxyz,lelt) , tnz(lxyz,lelt)
      common /scruz/  snx(lxz) , sny(lxz) , snz(lxz) ,  efc(lxz)
      common /scrsf/  jvrtex((2**ldim),lelt)
 
      integer e,f,eg
 
 
      gsh_fld(0)=gsh_fld(1)
      if (iftmsh(0)) gsh_fld(0)=gsh_fld(2)
 
      ifield = 0
      nel    = nelfld(0)
      nxyz   = lx1*ly1*lz1
      nxz    = lx1*lz1
      n      = nel*nxyz
      nface  = 2*ldim
 
 
      iflag=0
      do e=1,nel
      do f=1,nface
         if (cbc(f,e,1).eq.'msi'.or.cbc(f,e,1).eq.'MSI') iflag=1
      enddo
      enddo
      iflag = iglmax(iflag,1)
      if (iflag.eq.0) return
 
 
c     We need to differentiate elements according to unit normal on msi face.
 
c     NOTE that this code assumes we do not have two adjacent faces on a given
c     element that both have cbc = msi!
 
      call rzero(rnx,n)
      call rzero(rny,n)
      call rzero(rnz,n)
      call rzero(tnx,n)
      call rzero(tny,n)
      call rzero(tnz,n)
 
      do e=1,nel
         re = lglel(e)
         call cfill(enum(1,e),re,nxyz)
      enddo
 
      call dsop(enum,'min')
 
 
      do e=1,nel
         eg = lglel(e)
         do f=1,nface
           if (cbc(f,e,1).eq.'msi'.or.cbc(f,e,1).eq.'MSI') then
             call facexs (efc,enum(1,e),f,0) ! enum-->efc
             do i=1,nxz
               jg = efc(i)+0.1
               if (jg.eq.eg) then         ! this is the controlling face
                  snx(i) = unx(i,1,f,e)
                  sny(i) = uny(i,1,f,e)
                  snz(i) = unz(i,1,f,e)
               endif
             enddo
             call facexv (snx,sny,snz,rnx(1,e),rny(1,e),rnz(1,e),f,1) ! s-->r
             call facexv (unx(1,1,f,e),uny(1,1,f,e),unz(1,1,f,e) ! Control
     $                   ,tnx(1,e),tny(1,e),tnz(1,e),f,1)        ! u-->r
           endif
         enddo
      enddo
 
      call opdssum(rnx,rny,rnz)
 
      nv = nel*(2**ldim)
      call icopy(jvrtex,vertex,nv)  ! Save vertex
      mvertx=iglmax(jvrtex,nv)
 
 
c     Now, check to see if normal is aligned with incoming normal
 
      nsx=lx1-1
      nsy=ly1-1
      nsz=max(1,lz1-1)
 
      do e=1,nel
         l=0
         do k=1,lz1,nsz
         do j=1,ly1,nsy
         do i=1,lx1,nsx
            l=l+1
            m=i + lx1*(j-1) + lx1*ly1*(k-1)
            dot = tnx(m,e)*rnx(m,e)+tny(m,e)*rny(m,e)+tnz(m,e)*rnz(m,e)
            if (dot.lt.-.5) jvrtex(l,e)=jvrtex(l,e)+mvertx
         enddo
         enddo
         enddo
      enddo
 
 
      call setupds(gsh_fld(0),lx1,ly1,lz1,nelv,nelgv,jvrtex,glo_num)
 
      return
      end
c-----------------------------------------------------------------------
      subroutine outfldnx(x,txt10,nx,ny)
      include 'SIZE'
      real x(nx,ny,4)
      character*10 txt10
      integer e,g
 
      write(6,106) txt10,nel,nx
  106 FORMAT(  /,5x,'     ^              ',/,
     $           5x,'   Y |              ',/,
     $           5x,'     |              ',a10,/,
     $           5x,'     +---->         ','elem. = ',i2,'/',i2,/,
     $           5x,'       X            ')
 
      do e=3,1,-2
         g=e+1
         i=istart
         do j=ny,1,-1
            if (nx.eq.3) write(6,3) (x(i,j,e),i=1,nx),(x(i,j,g),i=1,nx)
            if (nx.eq.4) write(6,4) (x(i,j,e),i=1,nx),(x(i,j,g),i=1,nx)
            if (nx.eq.5) write(6,5) (x(i,j,e),i=1,nx),(x(i,j,g),i=1,nx)
            if (nx.eq.6) write(6,6) (x(i,j,e),i=1,nx),(x(i,j,g),i=1,nx)
            if (nx.eq.7) write(6,7) (x(i,j,e),i=1,nx),(x(i,j,g),i=1,nx)
   3        format(3f8.4,3x,3f8.4)
   4        format(4f8.4,3x,4f8.4)
   5        format(5f8.4,3x,5f8.4)
   6        format(6f8.4,3x,6f8.4)
   7        format(7f8.4,3x,7f8.4)
         enddo
         write(6,*)
      enddo
      write(6,*)
 
      return
      end
c-----------------------------------------------------------------------