33 integer,
intent(in) :: t_step
52 character(len=*),
intent(in) :: t_step_dir
53 character(len=1),
intent(in) :: direction
54 real(wp),
dimension(-1:),
intent(out) :: cb_array
55 real(wp),
dimension(0:),
intent(out) :: d_array
56 real(wp),
dimension(0:),
intent(out) :: cc_array
57 integer,
intent(in) :: size_dim
58 character(LEN=len_trim(t_step_dir) + 10) :: file_loc
61 file_loc = trim(t_step_dir) //
'/' // direction //
'_cb.dat'
62 inquire (file=trim(file_loc), exist=file_check)
65 open (1, file=trim(file_loc), form=
'unformatted', status=
'old', action=
'read')
66 read (1) cb_array(-1:size_dim)
69 call s_mpi_abort(
'File ' // direction //
'_cb.dat is missing in ' // trim(t_step_dir) //
'. Exiting.')
72 d_array(0:size_dim) = cb_array(0:size_dim) - cb_array(-1:size_dim - 1)
73 cc_array(0:size_dim) = cb_array(-1:size_dim - 1) + d_array(0:size_dim)/2._wp
81 integer,
intent(out) :: data_size
82 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: m_mok, n_mok, p_mok
83 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: wp_mok, mok, str_mok, nvars_mok
91 data_size = (m + 1)*(n + 1)*(p + 1)
93 m_mok = int(
m_glb + 1, mpi_offset_kind)
94 n_mok = int(
n_glb + 1, mpi_offset_kind)
95 p_mok = int(
p_glb + 1, mpi_offset_kind)
96 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
97 mok = int(1._wp, mpi_offset_kind)
98 str_mok = int(name_len, mpi_offset_kind)
99 nvars_mok = int(sys_size, mpi_offset_kind)
107 character(len=*),
intent(in) :: file_loc_base
108 integer,
intent(in),
optional :: t_step
109 character(LEN=path_len + 2*name_len) :: file_loc
110 logical :: file_exist
111 integer :: ifile, ierr, data_size
112 character(len=10) :: t_step_string
115 integer,
dimension(MPI_STATUS_SIZE) :: status
116 integer(KIND=MPI_OFFSET_KIND) :: disp
117 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok, mok, wp_mok, var_mok
118 integer :: save_index
123 if (parallel_io .and. file_per_process)
then
125 write (file_loc,
'(A,I0,A,i7.7,A)')
'ib_markers_', t_step,
'_',
proc_rank,
'.dat'
126 file_loc = trim(case_dir) //
'/restart_data/lustre_' // trim(t_step_string) //
'/' // trim(file_loc)
127 else if (parallel_io)
then
128 write (file_loc,
'(A)') trim(file_loc_base) //
'ib.dat'
130 write (file_loc,
'(A)') trim(file_loc_base) //
'/ib_data.dat'
132 inquire (file=trim(file_loc), exist=file_exist)
135 if (parallel_io .and. file_per_process)
then
137 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
140 data_size = (m + 1)*(n + 1)*(p + 1)
142 call mpi_file_read(ifile,
mpi_io_ib_data%var%sf, data_size, mpi_integer, status, ierr)
144 call mpi_file_close(ifile, ierr)
146 else if (parallel_io)
then
148 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
151 m_mok = int(
m_glb + 1, mpi_offset_kind)
152 n_mok = int(
n_glb + 1, mpi_offset_kind)
153 p_mok = int(
p_glb + 1, mpi_offset_kind)
154 mok = int(1._wp, mpi_offset_kind)
155 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
156 save_index = t_step/t_step_save
158 data_size = (m + 1)*(n + 1)*(p + 1)
159 var_mok = int(sys_size + 1, mpi_offset_kind)
160 if (t_step == 0)
then
163 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1 + int(save_index, mpi_offset_kind))
166 call mpi_file_set_view(ifile, disp, mpi_integer,
mpi_io_ib_data%view,
'native', mpi_info_int, ierr)
167 call mpi_file_read(ifile,
mpi_io_ib_data%var%sf, data_size, mpi_integer, status, ierr)
169 call mpi_file_close(ifile, ierr)
172 open (2, file=trim(file_loc), form=
'unformatted', action=
'read', status=
'old')
177 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
185 integer,
intent(in) :: local_start_idx, end_x, end_y, end_z
189 allocate (
q_cons_vf(i)%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
190 allocate (
q_prim_vf(i)%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
194 allocate (
ib_markers%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
197 if (chemistry .or. heat_conduction)
then
198 allocate (
q_t_sf%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
208 integer,
intent(in) :: t_step
209 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_dir
210 character(LEN=len_trim(case_dir) + 3*name_len) :: file_loc
211 character(len=int(floor(log10(real(sys_size, wp)))) + 1) :: file_num
213 logical :: file_check
216 write (t_step_dir,
'(A,I0,A,I0)')
'/p_all/p',
proc_rank,
'/', t_step
217 t_step_dir = trim(case_dir) // trim(t_step_dir)
219 file_loc = trim(t_step_dir) //
'/.'
222 if (dir_check .neqv. .true.)
then
223 call s_mpi_abort(
'Time-step folder ' // trim(t_step_dir) //
' is missing. Exiting.')
247 write (file_num,
'(I0)') i
248 file_loc = trim(t_step_dir) //
'/q_cons_vf' // trim(file_num) //
'.dat'
249 inquire (file=trim(file_loc), exist=file_check)
252 open (1, file=trim(file_loc), form=
'unformatted', status=
'old', action=
'read')
256 call s_mpi_abort(
'File q_cons_vf' // trim(file_num) //
'.dat is missing in ' // trim(t_step_dir) //
'. Exiting.')
268 integer,
intent(in) :: t_step
271 real(wp),
allocatable,
dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb
272 integer :: ifile, ierr, data_size, filetype, stride
273 integer,
dimension(MPI_STATUS_SIZE) :: status
274 integer(KIND=MPI_OFFSET_KIND) :: disp
275 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok
276 integer(KIND=MPI_OFFSET_KIND) :: wp_mok, var_mok, str_mok
277 integer(KIND=MPI_OFFSET_KIND) :: nvars_mok
278 integer(KIND=MPI_OFFSET_KIND) :: mok
279 integer(kind=MPI_OFFSET_KIND) :: offset
280 character(LEN=path_len + 2*name_len) :: file_loc
281 logical :: file_exist
282 integer(kind=8) :: file_bytes, bytes_needed
283 character(len=10) :: case_m_str, file_m_str
284 character(len=10) :: t_step_string
287 allocate (x_cb_glb(-1:
m_glb))
288 allocate (y_cb_glb(-1:
n_glb))
289 allocate (z_cb_glb(-1:
p_glb))
291 if (down_sample)
then
297 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'x_cb.dat'
298 inquire (file=trim(file_loc), exist=file_exist, size=file_bytes)
307 bytes_needed = (int(stride, 8)*int(
m_glb + 1, 8) + 1_8)*int(storage_size(0._wp)/8, 8)
308 if (file_bytes < bytes_needed .or. (.not. down_sample .and. file_bytes /= bytes_needed))
then
310 call s_int_to_str(int(file_bytes/int(storage_size(0._wp)/8, 8)) - 2, file_m_str)
311 call s_mpi_abort(
'Restart grid mismatch: this case has m = ' // trim(case_m_str) //
' but ' // trim(file_loc) &
312 & //
' was written with m = ' // trim(file_m_str) &
313 & //
'. Post-processing must use the same grid as the run that wrote the ' &
314 & //
'restart files, or it reads past the end of every file and writes NaN.')
316 data_size =
m_glb + 2
317 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
320 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
321 call mpi_type_commit(filetype, ierr)
324 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
326 call mpi_file_read(ifile, x_cb_glb, data_size, mpi_p, status, ierr)
327 call mpi_file_close(ifile, ierr)
329 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
337 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'y_cb.dat'
338 inquire (file=trim(file_loc), exist=file_exist)
341 data_size =
n_glb + 2
342 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
345 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
346 call mpi_type_commit(filetype, ierr)
349 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
351 call mpi_file_read(ifile, y_cb_glb, data_size, mpi_p, status, ierr)
352 call mpi_file_close(ifile, ierr)
354 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
361 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'z_cb.dat'
362 inquire (file=trim(file_loc), exist=file_exist)
365 data_size =
p_glb + 2
366 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
369 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
370 call mpi_type_commit(filetype, ierr)
373 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
375 call mpi_file_read(ifile, z_cb_glb, data_size, mpi_p, status, ierr)
376 call mpi_file_close(ifile, ierr)
378 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
388 deallocate (x_cb_glb, y_cb_glb, z_cb_glb)
403 integer,
intent(in) :: t_step
404 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: m_mok, n_mok, p_mok
405 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: wp_mok, mok, str_mok, nvars_mok
406 integer :: ifile, ierr, data_size
407 integer,
dimension(MPI_STATUS_SIZE) :: status
408 integer(KIND=MPI_OFFSET_KIND) :: disp, var_mok
409 character(LEN=path_len + 2*name_len) :: file_loc
410 logical :: file_exist
411 character(len=10) :: t_step_string
414 if (file_per_process)
then
416 write (file_loc,
'(I0,A1,I7.7,A)') t_step,
'_',
proc_rank,
'.dat'
417 file_loc = trim(case_dir) //
'/restart_data/lustre_' // trim(t_step_string) // trim(mpiiofs) // trim(file_loc)
418 inquire (file=trim(file_loc), exist=file_exist)
421 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
424 if (down_sample)
then
434 if (down_sample)
then
435 data_size = (m + 3)*(n + 3)*(p + 3)
437 data_size = (m + 1)*(n + 1)*(p + 1)
440 m_mok = int(
m_glb + 1, mpi_offset_kind)
441 n_mok = int(
n_glb + 1, mpi_offset_kind)
442 p_mok = int(
p_glb + 1, mpi_offset_kind)
443 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
444 mok = int(1._wp, mpi_offset_kind)
445 str_mok = int(name_len, mpi_offset_kind)
446 nvars_mok = int(sys_size, mpi_offset_kind)
448 if (bubbles_euler .or. hypoelasticity .or. mhd)
then
450 var_mok = int(i, mpi_offset_kind)
451 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
455 var_mok = int(i, mpi_offset_kind)
456 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
461 call mpi_file_close(ifile, ierr)
463 if (down_sample)
then
471 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
474 write (file_loc,
'(I0,A)') t_step,
'.dat'
475 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) // trim(file_loc)
476 inquire (file=trim(file_loc), exist=file_exist)
479 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
485 var_mok = int(i, mpi_offset_kind)
487 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1)
489 call mpi_file_set_view(ifile, disp, mpi_p,
mpi_io_data%view(i),
'native', mpi_info_int, ierr)
490 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
494 call mpi_file_close(ifile, ierr)
498 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
517 if (down_sample)
then
519 allocate (
q_cons_temp(i)%sf(-1:m + 1,-1:n + 1,-1:p + 1))
529 allocate (
bc_type(1:num_dims,1:2))
531 allocate (
bc_type(1, 1)%sf(0:0,0:n,0:p))
532 allocate (
bc_type(1, 2)%sf(0:0,0:n,0:p))
542 if (parallel_io .neqv. .true.)
then
558 if (down_sample)
then
571 if (chemistry .or. heat_conduction)
then
Noncharacteristic and processor boundary condition application for ghost cells and buffer regions.
Boundary condition restart I/O, capillary/IGR buffer population, and grid-variable buffers.
subroutine s_assign_default_bc_type(bc_type)
Initialize the per-cell boundary condition type arrays with the global default BC values.
subroutine s_read_serial_boundary_condition_files(step_dirpath, bc_type)
Read boundary condition type and buffer data from serial (unformatted) restart files.
subroutine s_read_parallel_boundary_condition_files(bc_type)
Read boundary condition type and buffer data from per-rank parallel files using MPI I/O.
Platform-specific file and directory operations: create, delete, inquire, getcwd, and basename.
impure subroutine my_inquire(fileloc, dircheck)
Inquire on the existence of a directory or file.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the post-process: domain geometry, equation of state, and output database setti...
type(int_bounds_info) offset_y
type(int_bounds_info) bc_z
real(wp), dimension(:), allocatable y_cc
integer proc_rank
Rank of the local processor.
type(mpi_io_ib_var), public mpi_io_ib_data
real(wp), dimension(:), allocatable y_cb
real(wp), dimension(:), allocatable dz
integer buff_size
Number of ghost cells for boundary condition storage.
real(wp), dimension(:), allocatable z_cb
type(int_bounds_info) bc_y
real(wp), dimension(:), allocatable x_cc
type(int_bounds_info) bc_x
real(wp), dimension(:), allocatable x_cb
real(wp), dimension(:), allocatable dy
type(int_bounds_info) offset_x
real(wp), dimension(:), allocatable z_cc
type(int_bounds_info) offset_z
type(mpi_io_var), public mpi_io_data
real(wp), dimension(:), allocatable dx
Cell-width distributions in the x-, y- and z-coordinate directions.
Utility routines for bubble model setup, coordinate transforms, array sampling, and special functions...
elemental subroutine, public s_int_to_str(i, res)
Convert an integer to its trimmed string representation.
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_barrier
Halts all processes until all have reached barrier.
impure subroutine s_check_mpi_file_open(ierr, file_loc)
Abort naming the file if MPI_FILE_OPEN failed. File errors default to MPI_ERRORS_RETURN,...
subroutine s_apply_grid_from_global_dim(x_cb_glb, m_dim_glb, m_dim, sidx, bc_beg, bc_end, cb_lo, cb_hi, cw_lo, cw_hi, x_cb_loc, x_cc_loc, dx_loc)
Populate the local cell-boundary, cell-center, and cell-width arrays in one direction directly from t...
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, ib_mpi_data, beta, qbmm_pb, qbmm_mv)
Set up MPI I/O data views and variable pointers for parallel file output.
subroutine s_initialize_mpi_data_ds(m_ds, n_ds, p_ds, q_cons_vf)
Set up MPI I/O data views for downsampled (coarsened) parallel file output.
MPI gather and scatter operations for distributing post-process grid and flow-variable data.
Derived type annexing an integer scalar field (SF).
Derived type annexing a scalar field (SF).