9 use mpi !< message passing interface (mpi) module
16 use m_mpi_proxy !< message passing interface (mpi) module proxy
42 integer,
intent(in) :: t_step
78 character(len=*),
intent(in) :: t_step_dir
79 character(len=1),
intent(in) :: direction
80 real(wp),
dimension(-1:),
intent(out) :: cb_array
81 real(wp),
dimension(0:),
intent(out) :: d_array
82 real(wp),
dimension(0:),
intent(out) :: cc_array
83 integer,
intent(in) :: size_dim
85 character(LEN=len_trim(t_step_dir) + 10) :: file_loc
89 file_loc = trim(t_step_dir)//
'/'//direction//
'_cb.dat'
90 inquire (file=trim(file_loc), exist=file_check)
94 open (1, file=trim(file_loc), form=
'unformatted', &
95 status=
'old', action=
'read')
96 read (1) cb_array(-1:size_dim)
99 call s_mpi_abort(
'File '//direction//
'_cb.dat is missing in '// &
100 trim(t_step_dir)//
'. Exiting.')
104 d_array(0:size_dim) = cb_array(0:size_dim) - cb_array(-1:size_dim - 1)
107 cc_array(0:size_dim) = cb_array(-1:size_dim - 1) + d_array(0:size_dim)/2._wp
118 integer,
intent(out) :: data_size
119 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: m_mok, n_mok, p_mok
120 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: wp_mok, mok, str_mok, nvars_mok
130 data_size = (
m + 1)*(
n + 1)*(
p + 1)
133 m_mok = int(
m_glb + 1, mpi_offset_kind)
134 n_mok = int(
n_glb + 1, mpi_offset_kind)
135 p_mok = int(
p_glb + 1, mpi_offset_kind)
136 wp_mok = int(8._wp, mpi_offset_kind)
137 mok = int(1._wp, mpi_offset_kind)
138 str_mok = int(name_len, mpi_offset_kind)
139 nvars_mok = int(
sys_size, mpi_offset_kind)
149 character(len=*),
intent(in) :: file_loc_base
150 integer,
intent(in),
optional :: t_step
152 character(LEN=len_trim(file_loc_base) + 20) :: file_loc
153 logical :: file_exist
154 integer :: ifile, ierr, data_size, var_mok
157 integer,
dimension(MPI_STATUS_SIZE) :: status
158 integer(KIND=MPI_OFFSET_KIND) :: disp
159 integer :: m_mok, n_mok, p_mok, mok, wp_mok, save_index
165 write (file_loc,
'(A)') trim(file_loc_base)//
'ib.dat'
167 write (file_loc,
'(A)') trim(file_loc_base)//
'/ib.dat'
169 inquire (file=trim(file_loc), exist=file_exist)
174 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
176 m_mok = int(
m_glb + 1, mpi_offset_kind)
177 n_mok = int(
n_glb + 1, mpi_offset_kind)
178 p_mok = int(
p_glb + 1, mpi_offset_kind)
179 mok = int(1._wp, mpi_offset_kind)
180 wp_mok = int(8._wp, mpi_offset_kind)
183 data_size = (
m + 1)*(
n + 1)*(
p + 1)
184 var_mok = int(
sys_size + 1, mpi_offset_kind)
185 if (t_step == 0)
then
188 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1 + int(save_index, mpi_offset_kind))
191 call mpi_file_set_view(ifile, disp, mpi_integer,
mpi_io_ib_data%view, &
194 mpi_integer, status, ierr)
196 call mpi_file_close(ifile, ierr)
199 open (2, file=trim(file_loc), &
200 form=
'unformatted', &
207 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
219 integer,
intent(in) :: local_start_idx, end_x, end_y, end_z
223 allocate (
q_cons_vf(i)%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
224 allocate (
q_prim_vf(i)%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
228 allocate (
ib_markers%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
232 allocate (
q_t_sf%sf(local_start_idx:end_x, local_start_idx:end_y, local_start_idx:end_z))
244 integer,
intent(in) :: t_step
246 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_dir
249 character(LEN=len_trim(case_dir) + 3*name_len) :: file_loc
253 int(floor(log10(real(
sys_size, wp)))) + 1) :: file_num
257 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_ib_dir
263 logical :: file_check
269 write (t_step_dir,
'(A,I0,A,I0)')
'/p_all/p',
proc_rank,
'/', t_step
270 t_step_dir = trim(
case_dir)//trim(t_step_dir)
273 file_loc = trim(t_step_dir)//
'/.'
277 if (dir_check .neqv. .true.)
then
278 call s_mpi_abort(
'Time-step folder '//trim(t_step_dir)// &
279 ' is missing. Exiting.')
304 write (file_num,
'(I0)') i
305 file_loc = trim(t_step_dir)//
'/q_cons_vf'// &
306 trim(file_num)//
'.dat'
307 inquire (file=trim(file_loc), exist=file_check)
311 open (1, file=trim(file_loc), form=
'unformatted', &
312 status=
'old', action=
'read')
320 call s_mpi_abort(
'File q_cons_vf'//trim(file_num)// &
321 '.dat is missing in '//trim(t_step_dir)// &
339 integer,
intent(in) :: t_step
343 real(wp),
allocatable,
dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb
345 integer :: ifile, ierr, data_size, filetype, stride
346 integer,
dimension(MPI_STATUS_SIZE) :: status
348 integer(KIND=MPI_OFFSET_KIND) :: disp
349 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok
350 integer(KIND=MPI_OFFSET_KIND) :: wp_mok, var_mok, str_mok
351 integer(KIND=MPI_OFFSET_KIND) :: nvars_mok
352 integer(KIND=MPI_OFFSET_KIND) :: mok
353 integer(kind=MPI_OFFSET_KIND) :: offset
354 real(wp) :: delx, dely, delz
356 character(LEN=path_len + 2*name_len) :: file_loc
357 logical :: file_exist
359 character(len=10) :: t_step_string
363 allocate (x_cb_glb(-1:
m_glb))
364 allocate (y_cb_glb(-1:
n_glb))
365 allocate (z_cb_glb(-1:
p_glb))
375 inquire (file=trim(file_loc), exist=file_exist)
378 data_size =
m_glb + 2
379 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
381 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
382 call mpi_type_commit(filetype, ierr)
385 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
387 call mpi_file_read(ifile, x_cb_glb, data_size, mpi_p, status, ierr)
388 call mpi_file_close(ifile, ierr)
390 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
403 inquire (file=trim(file_loc), exist=file_exist)
406 data_size =
n_glb + 2
407 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
409 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
410 call mpi_type_commit(filetype, ierr)
413 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
415 call mpi_file_read(ifile, y_cb_glb, data_size, mpi_p, status, ierr)
416 call mpi_file_close(ifile, ierr)
418 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
431 inquire (file=trim(file_loc), exist=file_exist)
434 data_size =
p_glb + 2
435 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
437 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
438 call mpi_type_commit(filetype, ierr)
441 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native',
mpi_info_int, ierr)
443 call mpi_file_read(ifile, z_cb_glb, data_size, mpi_p, status, ierr)
444 call mpi_file_close(ifile, ierr)
446 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
460 deallocate (x_cb_glb, y_cb_glb, z_cb_glb)
479 integer,
intent(in) :: t_step
480 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: m_mok, n_mok, p_mok
481 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: wp_mok, mok, str_mok, nvars_mok
483 integer :: ifile, ierr, data_size
484 integer,
dimension(MPI_STATUS_SIZE) :: status
485 integer(KIND=MPI_OFFSET_KIND) :: disp, var_mok
486 character(LEN=path_len + 2*name_len) :: file_loc
487 logical :: file_exist
488 character(len=10) :: t_step_string
494 write (file_loc,
'(I0,A1,I7.7,A)') t_step,
'_',
proc_rank,
'.dat'
495 file_loc = trim(
case_dir)//
'/restart_data/lustre_'//trim(t_step_string)//trim(
mpiiofs)//trim(file_loc)
496 inquire (file=trim(file_loc), exist=file_exist)
499 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
514 data_size = (
m + 3)*(
n + 3)*(
p + 3)
517 data_size = (
m + 1)*(
n + 1)*(
p + 1)
521 m_mok = int(
m_glb + 1, mpi_offset_kind)
522 n_mok = int(
n_glb + 1, mpi_offset_kind)
523 p_mok = int(
p_glb + 1, mpi_offset_kind)
524 wp_mok = int(8._wp, mpi_offset_kind)
525 mok = int(1._wp, mpi_offset_kind)
526 str_mok = int(name_len, mpi_offset_kind)
527 nvars_mok = int(
sys_size, mpi_offset_kind)
532 var_mok = int(i, mpi_offset_kind)
533 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
534 mpi_io_p, status, ierr)
538 var_mok = int(i, mpi_offset_kind)
539 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
540 mpi_io_p, status, ierr)
545 call mpi_file_close(ifile, ierr)
555 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
559 write (file_loc,
'(I0,A)') t_step,
'.dat'
560 file_loc = trim(
case_dir)//
'/restart_data'//trim(
mpiiofs)//trim(file_loc)
561 inquire (file=trim(file_loc), exist=file_exist)
564 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly,
mpi_info_int, ifile, ierr)
570 var_mok = int(i, mpi_offset_kind)
573 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1)
575 call mpi_file_set_view(ifile, disp, mpi_p,
mpi_io_data%view(i), &
577 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, &
578 mpi_io_p, status, ierr)
582 call mpi_file_close(ifile, ierr)
586 call s_mpi_abort(
'File '//trim(file_loc)//
' is missing. Exiting.')
631 allocate (
bc_type(1, 1)%sf(0:0, 0:
n, 0:
p))
632 allocate (
bc_type(1, 2)%sf(0:0, 0:
n, 0:
p))
Noncharacteristic and processor boundary condition application for ghost cells and buffer regions.
subroutine, public s_read_parallel_boundary_condition_files(bc_type)
Reads boundary condition type and buffer data from per-rank parallel files using MPI I/O.
subroutine, public s_read_serial_boundary_condition_files(step_dirpath, bc_type)
Reads boundary condition type and buffer data from serial (unformatted) restart files.
subroutine, public s_assign_default_bc_type(bc_type)
Initializes the per-cell boundary condition type arrays with the global default BC values.
Platform-specific file and directory operations: create, delete, inquire, getcwd, and basename.
impure subroutine my_inquire(fileloc, dircheck)
Inquires on the existence of a directory.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the post-process: domain geometry, equation of state, and output database setti...
logical, parameter chemistry
Chemistry modeling.
integer beta_idx
Index of lagrange bubbles beta.
real(wp), dimension(:), allocatable y_cc
integer proc_rank
Rank of the local processor.
type(mpi_io_ib_var), public mpi_io_ib_data
real(wp), dimension(:), allocatable y_cb
character(len=name_len) mpiiofs
integer, dimension(:), allocatable start_idx
Starting cell-center index of local processor in global grid.
integer sys_size
Number of unknowns in the system of equations.
real(wp), dimension(:), allocatable dz
integer buff_size
Number of cells in buffer region. For the variables which feature a buffer region,...
real(wp), dimension(:), allocatable z_cb
integer num_dims
Number of spatial dimensions.
real(wp), dimension(:), allocatable x_cc
real(wp), dimension(:), allocatable x_cb
real(wp), dimension(:), allocatable dy
integer t_step_save
Interval between consecutive time-step directory.
real(wp), dimension(:), allocatable z_cc
character(len=path_len) case_dir
Case folder location.
logical mhd
Magnetohydrodynamics.
logical parallel_io
Format of the data files.
logical down_sample
down sampling of the database file(s)
logical file_per_process
output format
logical elasticity
elasticity modeling, true for hyper or hypo
type(mpi_io_var), public mpi_io_data
real(wp), dimension(:), allocatable dx
Cell-width distributions in the x-, y- and z-coordinate directions.
Utility routines for bubble model setup, coordinate transforms, array sampling, and special functions...
elemental subroutine, public s_int_to_str(i, res)
Converts an integer to its trimmed string representation.
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_barrier
Halts all processes until all have reached barrier.
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, beta)
subroutine s_initialize_mpi_data_ds(q_cons_vf)
MPI gather and scatter operations for distributing post-process grid and flow-variable data.
Derived type annexing an integer scalar field (SF).
Derived type annexing a scalar field (SF).