9 use mpi !< message passing interface (mpi) module
16 use m_mpi_proxy !< message passing interface (mpi) module proxy
42 integer,
intent(in) :: t_step
78 character(len=*),
intent(in) :: t_step_dir
79 character(len=1),
intent(in) :: direction
80 real(wp),
dimension(-1:),
intent(out) :: cb_array
81 real(wp),
dimension(0:),
intent(out) :: d_array
82 real(wp),
dimension(0:),
intent(out) :: cc_array
83 integer,
intent(in) :: size_dim
85 character(LEN=len_trim(t_step_dir) + 10) :: file_loc
89 file_loc = trim(t_step_dir)//
'/'//direction//
'_cb.dat'
90 inquire (file=trim(file_loc), exist=file_check)
94 open (1, file=trim(file_loc), form=
'unformatted', &
95 status=
'old', action=
'read')
96 read (1) cb_array(-1:size_dim)
99 call s_mpi_abort(
'File '//direction//
'_cb.dat is missing in '// &
100 trim(t_step_dir)//
'. Exiting.')
104 d_array(0:size_dim) = cb_array(0:size_dim) - cb_array(-1:size_dim - 1)
107 cc_array(0:size_dim) = cb_array(-1:size_dim - 1) + d_array(0:size_dim)/2._wp
118 integer,
intent(out) :: data_size
119 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: m_mok, n_mok, p_mok
120 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: wp_mok, mok, str_mok, nvars_mok
130 data_size = (
m + 1)*(
n + 1)*(
p + 1)
133 m_mok = int(
m_glb + 1, mpi_offset_kind)
134 n_mok = int(
n_glb + 1, mpi_offset_kind)
135 p_mok = int(
p_glb + 1, mpi_offset_kind)
136 wp_mok = int(8._wp, mpi_offset_kind)
137 mok = int(1._wp, mpi_offset_kind)
138 str_mok = int(name_len, mpi_offset_kind)
139 nvars_mok = int(
sys_size, mpi_offset_kind)
149 character(len=*),
intent(in) :: file_loc_base
150 integer,
intent(in),
optional :: t_step
152 character(LEN=len_trim(file_loc_base) + 20) :: file_loc
153 logical :: file_exist
154 integer :: ifile, ierr, data_size, var_mok
157 integer,
dimension(MPI_STATUS_SIZE) :: status
158 integer(KIND=MPI_OFFSET_KIND) :: disp
159 integer :: m_mok, n_mok, p_mok, mok, wp_mok, save_index
165 write (file_loc,
'(A)') trim(file_loc_base)//
'ib.dat'
167 write (file_loc,
'(A)') trim(file_loc_base)//
'/ib.dat'
169 inquire (file=trim(file_loc), exist=file_exist)
174 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
176 m_mok = int(
m_glb + 1, mpi_offset_kind)
177 n_mok = int(
n_glb + 1, mpi_offset_kind)
178 p_mok = int(
p_glb + 1, mpi_offset_kind)
179 mok = int(1._wp, mpi_offset_kind)
180 wp_mok = int(8._wp, mpi_offset_kind)
183 data_size = (
m + 1)*(
n + 1)*(
p + 1)
184 var_mok = int(
sys_size + 1, mpi_offset_kind)
185 if (t_step == 0)
then
188 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1 + int(save_index, mpi_offset_kind))
191 call mpi_file_set_view(ifile, disp, mpi_integer,
mpi_io_ib_data%view, &
194 mpi_integer, status, ierr)
196 call mpi_file_close(ifile, ierr)
199 open (2, file=trim(file_loc), &
200 form=
'unformatted', &
207 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
219 integer,
intent(in) :: local_start_idx, end_x, end_y, end_z
223 allocate (
q_cons_vf(i)%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
224 allocate (
q_prim_vf(i)%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
228 allocate (
ib_markers%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
232 allocate (
q_t_sf%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
244 integer,
intent(in) :: t_step
246 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_dir
249 character(LEN=len_trim(case_dir) + 3*name_len) :: file_loc
253 int(floor(log10(real(
sys_size, wp)))) + 1) :: file_num
257 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_ib_dir
263 logical :: file_check
269 write (t_step_dir,
'(A,I0,A,I0)')
'/p_all/p',
proc_rank,
'/', t_step
270 t_step_dir = trim(
case_dir)//trim(t_step_dir)
273 file_loc = trim(t_step_dir)//
'/.'
277 if (dir_check .neqv. .true.)
then
278 call s_mpi_abort(
'Time-step folder '//trim(t_step_dir)// &
279 ' is missing. Exiting.')
304 write (file_num,
'(I0)') i
305 file_loc = trim(t_step_dir)//
'/q_cons_vf'// &
306 trim(file_num)//
'.dat'
307 inquire (file=trim(file_loc), exist=file_check)
311 open (1, file=trim(file_loc), form=
'unformatted', &
312 status=
'old', action=
'read')
317 call s_mpi_abort(
'File q_cons_vf'//trim(file_num)// &
318 '.dat is missing in '//trim(t_step_dir)// &
336 integer,
intent(in) :: t_step
340 real(wp),
allocatable,
dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb
342 integer :: ifile, ierr, data_size, filetype, stride
343 integer,
dimension(MPI_STATUS_SIZE) :: status
345 integer(KIND=MPI_OFFSET_KIND) :: disp
346 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok
347 integer(KIND=MPI_OFFSET_KIND) :: wp_mok, var_mok, str_mok
348 integer(KIND=MPI_OFFSET_KIND) :: nvars_mok
349 integer(KIND=MPI_OFFSET_KIND) :: mok
350 integer(kind=MPI_OFFSET_KIND) :: offset
351 real(wp) :: delx, dely, delz
353 character(LEN=path_len + 2*name_len) :: file_loc
354 logical :: file_exist
356 character(len=10) :: t_step_string
360 allocate (x_cb_glb(-1:
m_glb))
361 allocate (y_cb_glb(-1:
n_glb))
362 allocate (z_cb_glb(-1:
p_glb))
372 inquire (file=trim(file_loc), exist=file_exist)
375 data_size =
m_glb + 2
376 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
378 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
379 call mpi_type_commit(filetype, ierr)
382 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
384 call mpi_file_read(ifile, x_cb_glb, data_size, mpi_p, status, ierr)
385 call mpi_file_close(ifile, ierr)
387 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
400 inquire (file=trim(file_loc), exist=file_exist)
403 data_size =
n_glb + 2
404 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
406 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
407 call mpi_type_commit(filetype, ierr)
410 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
412 call mpi_file_read(ifile, y_cb_glb, data_size, mpi_p, status, ierr)
413 call mpi_file_close(ifile, ierr)
415 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
428 inquire (file=trim(file_loc), exist=file_exist)
431 data_size =
p_glb + 2
432 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
434 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
435 call mpi_type_commit(filetype, ierr)
438 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
440 call mpi_file_read(ifile, z_cb_glb, data_size, mpi_p, status, ierr)
441 call mpi_file_close(ifile, ierr)
443 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
457 deallocate (x_cb_glb, y_cb_glb, z_cb_glb)
476 integer,
intent(in) :: t_step
477 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: m_mok, n_mok, p_mok
478 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: wp_mok, mok, str_mok, nvars_mok
480 integer :: ifile, ierr, data_size
481 integer,
dimension(MPI_STATUS_SIZE) :: status
482 integer(KIND=MPI_OFFSET_KIND) :: disp, var_mok
483 character(LEN=path_len + 2*name_len) :: file_loc
484 logical :: file_exist
485 character(len=10) :: t_step_string
491 write (file_loc,
'(I0,A1,I7.7,A)') t_step,
'_',
proc_rank,
'.dat'
492 file_loc = trim(
case_dir)//
'/restart_data/lustre_'//trim(t_step_string)//trim(
mpiiofs)//trim(file_loc)
493 inquire (file=trim(file_loc), exist=file_exist)
496 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
511 data_size = (
m + 3)*(
n + 3)*(
p + 3)
514 data_size = (
m + 1)*(
n + 1)*(
p + 1)
518 m_mok = int(
m_glb + 1, mpi_offset_kind)
519 n_mok = int(
n_glb + 1, mpi_offset_kind)
520 p_mok = int(
p_glb + 1, mpi_offset_kind)
521 wp_mok = int(8._wp, mpi_offset_kind)
522 mok = int(1._wp, mpi_offset_kind)
523 str_mok = int(name_len, mpi_offset_kind)
524 nvars_mok = int(
sys_size, mpi_offset_kind)
529 var_mok = int(i, mpi_offset_kind)
530 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
531 mpi_io_p, status, ierr)
535 var_mok = int(i, mpi_offset_kind)
536 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
537 mpi_io_p, status, ierr)
542 call mpi_file_close(ifile, ierr)
552 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
556 write (file_loc,
'(I0,A)') t_step,
'.dat'
557 file_loc = trim(
case_dir)//
'/restart_data'//trim(
mpiiofs)//trim(file_loc)
558 inquire (file=trim(file_loc), exist=file_exist)
561 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
567 var_mok = int(i, mpi_offset_kind)
570 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1)
572 call mpi_file_set_view(ifile, disp, mpi_p,
mpi_io_data%view(i), &
574 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
575 mpi_io_p, status, ierr)
579 call mpi_file_close(ifile, ierr)
583 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
628 allocate (
bc_type(1, 1)%sf(0:0, 0:
n, 0:
p))
629 allocate (
bc_type(1, 2)%sf(0:0, 0:
n, 0:
p))
Noncharacteristic and processor boundary condition application for ghost cells and buffer regions.
subroutine, public s_read_parallel_boundary_condition_files(bc_type)
Reads boundary condition type and buffer data from per-rank parallel files using MPI I/O.
subroutine, public s_read_serial_boundary_condition_files(step_dirpath, bc_type)
Reads boundary condition type and buffer data from serial (unformatted) restart files.
subroutine, public s_assign_default_bc_type(bc_type)
Initializes the per-cell boundary condition type arrays with the global default BC values.
Platform-specific file and directory operations: create, delete, inquire, getcwd, and basename.
impure subroutine my_inquire(fileloc, dircheck)
Inquires on the existence of a directory.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the post-process: domain geometry, equation of state, and output database setti...
logical, parameter chemistry
Chemistry modeling.
real(wp), dimension(:), allocatable y_cc
integer proc_rank
Rank of the local processor.
type(mpi_io_ib_var), public mpi_io_ib_data
real(wp), dimension(:), allocatable y_cb
character(len=name_len) mpiiofs
integer, dimension(:), allocatable start_idx
Starting cell-center index of local processor in global grid.
integer sys_size
Number of unknowns in the system of equations.
real(wp), dimension(:), allocatable dz
integer buff_size
Number of cells in buffer region. For the variables which feature a buffer region,...
real(wp), dimension(:), allocatable z_cb
integer num_dims
Number of spatial dimensions.
real(wp), dimension(:), allocatable x_cc
real(wp), dimension(:), allocatable x_cb
real(wp), dimension(:), allocatable dy
integer t_step_save
Interval between consecutive time-step directory.
real(wp), dimension(:), allocatable z_cc
character(len=path_len) case_dir
Case folder location.
logical mhd
Magnetohydrodynamics.
logical parallel_io
Format of the data files.
logical down_sample
down sampling of the database file(s)
logical file_per_process
output format
logical elasticity
elasticity modeling, true for hyper or hypo
type(mpi_io_var), public mpi_io_data
real(wp), dimension(:), allocatable dx
Cell-width distributions in the x-, y- and z-coordinate directions.
Utility routines for bubble model setup, coordinate transforms, array sampling, and special functions...
elemental subroutine, public s_int_to_str(i, res)
Converts an integer to its trimmed string representation.
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_barrier
Halts all processes until all have reached barrier.
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, beta)
subroutine s_initialize_mpi_data_ds(q_cons_vf)
MPI gather and scatter operations for distributing post-process grid and flow-variable data.
Derived type annexing an integer scalar field (SF).
Derived type annexing a scalar field (SF).