33 integer,
intent(in) :: t_step
52 character(len=*),
intent(in) :: t_step_dir
53 character(len=1),
intent(in) :: direction
54 real(wp),
dimension(-1:),
intent(out) :: cb_array
55 real(wp),
dimension(0:),
intent(out) :: d_array
56 real(wp),
dimension(0:),
intent(out) :: cc_array
57 integer,
intent(in) :: size_dim
58 character(LEN=len_trim(t_step_dir) + 10) :: file_loc
61 file_loc = trim(t_step_dir) //
'/' // direction //
'_cb.dat'
62 inquire (file=trim(file_loc), exist=file_check)
65 open (1, file=trim(file_loc), form=
'unformatted', status=
'old', action=
'read')
66 read (1) cb_array(-1:size_dim)
69 call s_mpi_abort(
'File ' // direction //
'_cb.dat is missing in ' // trim(t_step_dir) //
'. Exiting.')
72 d_array(0:size_dim) = cb_array(0:size_dim) - cb_array(-1:size_dim - 1)
73 cc_array(0:size_dim) = cb_array(-1:size_dim - 1) + d_array(0:size_dim)/2._wp
81 integer,
intent(out) :: data_size
82 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: m_mok, n_mok, p_mok
83 integer(KIND=MPI_OFFSET_KIND),
intent(out) :: wp_mok, mok, str_mok, nvars_mok
91 data_size = (m + 1)*(n + 1)*(p + 1)
93 m_mok = int(
m_glb + 1, mpi_offset_kind)
94 n_mok = int(
n_glb + 1, mpi_offset_kind)
95 p_mok = int(
p_glb + 1, mpi_offset_kind)
96 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
97 mok = int(1._wp, mpi_offset_kind)
98 str_mok = int(name_len, mpi_offset_kind)
99 nvars_mok = int(sys_size, mpi_offset_kind)
107 character(len=*),
intent(in) :: file_loc_base
108 integer,
intent(in),
optional :: t_step
109 character(LEN=path_len + 2*name_len) :: file_loc
110 logical :: file_exist
111 integer :: ifile, ierr, data_size
112 character(len=10) :: t_step_string
115 integer,
dimension(MPI_STATUS_SIZE) :: status
116 integer(KIND=MPI_OFFSET_KIND) :: disp
117 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok, mok, wp_mok, var_mok
118 integer :: save_index
123 if (parallel_io .and. file_per_process)
then
125 write (file_loc,
'(A,I0,A,i7.7,A)')
'ib_markers_', t_step,
'_',
proc_rank,
'.dat'
126 file_loc = trim(case_dir) //
'/restart_data/lustre_' // trim(t_step_string) //
'/' // trim(file_loc)
127 else if (parallel_io)
then
128 write (file_loc,
'(A)') trim(file_loc_base) //
'ib.dat'
130 write (file_loc,
'(A)') trim(file_loc_base) //
'/ib_data.dat'
132 inquire (file=trim(file_loc), exist=file_exist)
135 if (parallel_io .and. file_per_process)
then
137 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
139 data_size = (m + 1)*(n + 1)*(p + 1)
141 call mpi_file_read(ifile,
mpi_io_ib_data%var%sf, data_size, mpi_integer, status, ierr)
143 call mpi_file_close(ifile, ierr)
145 else if (parallel_io)
then
147 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
149 m_mok = int(
m_glb + 1, mpi_offset_kind)
150 n_mok = int(
n_glb + 1, mpi_offset_kind)
151 p_mok = int(
p_glb + 1, mpi_offset_kind)
152 mok = int(1._wp, mpi_offset_kind)
153 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
154 save_index = t_step/t_step_save
156 data_size = (m + 1)*(n + 1)*(p + 1)
157 var_mok = int(sys_size + 1, mpi_offset_kind)
158 if (t_step == 0)
then
161 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1 + int(save_index, mpi_offset_kind))
164 call mpi_file_set_view(ifile, disp, mpi_integer,
mpi_io_ib_data%view,
'native', mpi_info_int, ierr)
165 call mpi_file_read(ifile,
mpi_io_ib_data%var%sf, data_size, mpi_integer, status, ierr)
167 call mpi_file_close(ifile, ierr)
170 open (2, file=trim(file_loc), form=
'unformatted', action=
'read', status=
'old')
175 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
183 integer,
intent(in) :: local_start_idx, end_x, end_y, end_z
187 allocate (
q_cons_vf(i)%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
188 allocate (
q_prim_vf(i)%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
192 allocate (
ib_markers%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
195 if (chemistry .or. heat_conduction)
then
196 allocate (
q_t_sf%sf(local_start_idx:end_x,local_start_idx:end_y,local_start_idx:end_z))
206 integer,
intent(in) :: t_step
207 character(LEN=len_trim(case_dir) + 2*name_len) :: t_step_dir
208 character(LEN=len_trim(case_dir) + 3*name_len) :: file_loc
209 character(len=int(floor(log10(real(sys_size, wp)))) + 1) :: file_num
211 logical :: file_check
214 write (t_step_dir,
'(A,I0,A,I0)')
'/p_all/p',
proc_rank,
'/', t_step
215 t_step_dir = trim(case_dir) // trim(t_step_dir)
217 file_loc = trim(t_step_dir) //
'/.'
220 if (dir_check .neqv. .true.)
then
221 call s_mpi_abort(
'Time-step folder ' // trim(t_step_dir) //
' is missing. Exiting.')
245 write (file_num,
'(I0)') i
246 file_loc = trim(t_step_dir) //
'/q_cons_vf' // trim(file_num) //
'.dat'
247 inquire (file=trim(file_loc), exist=file_check)
250 open (1, file=trim(file_loc), form=
'unformatted', status=
'old', action=
'read')
254 call s_mpi_abort(
'File q_cons_vf' // trim(file_num) //
'.dat is missing in ' // trim(t_step_dir) //
'. Exiting.')
266 integer,
intent(in) :: t_step
269 real(wp),
allocatable,
dimension(:) :: x_cb_glb, y_cb_glb, z_cb_glb
270 integer :: ifile, ierr, data_size, filetype, stride
271 integer,
dimension(MPI_STATUS_SIZE) :: status
272 integer(KIND=MPI_OFFSET_KIND) :: disp
273 integer(KIND=MPI_OFFSET_KIND) :: m_mok, n_mok, p_mok
274 integer(KIND=MPI_OFFSET_KIND) :: wp_mok, var_mok, str_mok
275 integer(KIND=MPI_OFFSET_KIND) :: nvars_mok
276 integer(KIND=MPI_OFFSET_KIND) :: mok
277 integer(kind=MPI_OFFSET_KIND) :: offset
278 character(LEN=path_len + 2*name_len) :: file_loc
279 logical :: file_exist
280 integer(kind=8) :: file_bytes, bytes_needed
281 character(len=10) :: case_m_str, file_m_str
282 character(len=10) :: t_step_string
285 allocate (x_cb_glb(-1:
m_glb))
286 allocate (y_cb_glb(-1:
n_glb))
287 allocate (z_cb_glb(-1:
p_glb))
289 if (down_sample)
then
295 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'x_cb.dat'
296 inquire (file=trim(file_loc), exist=file_exist, size=file_bytes)
305 bytes_needed = (int(stride, 8)*int(
m_glb + 1, 8) + 1_8)*int(storage_size(0._wp)/8, 8)
306 if (file_bytes < bytes_needed .or. (.not. down_sample .and. file_bytes /= bytes_needed))
then
308 call s_int_to_str(int(file_bytes/int(storage_size(0._wp)/8, 8)) - 2, file_m_str)
309 call s_mpi_abort(
'Restart grid mismatch: this case has m = ' // trim(case_m_str) //
' but ' // trim(file_loc) &
310 & //
' was written with m = ' // trim(file_m_str) &
311 & //
'. Post-processing must use the same grid as the run that wrote the ' &
312 & //
'restart files, or it reads past the end of every file and writes NaN.')
314 data_size =
m_glb + 2
315 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
317 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
318 call mpi_type_commit(filetype, ierr)
321 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
323 call mpi_file_read(ifile, x_cb_glb, data_size, mpi_p, status, ierr)
324 call mpi_file_close(ifile, ierr)
326 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
334 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'y_cb.dat'
335 inquire (file=trim(file_loc), exist=file_exist)
338 data_size =
n_glb + 2
339 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
341 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
342 call mpi_type_commit(filetype, ierr)
345 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
347 call mpi_file_read(ifile, y_cb_glb, data_size, mpi_p, status, ierr)
348 call mpi_file_close(ifile, ierr)
350 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
357 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) //
'z_cb.dat'
358 inquire (file=trim(file_loc), exist=file_exist)
361 data_size =
p_glb + 2
362 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
364 call mpi_type_vector(data_size, 1, stride, mpi_p, filetype, ierr)
365 call mpi_type_commit(filetype, ierr)
368 call mpi_file_set_view(ifile, offset, mpi_p, filetype,
'native', mpi_info_int, ierr)
370 call mpi_file_read(ifile, z_cb_glb, data_size, mpi_p, status, ierr)
371 call mpi_file_close(ifile, ierr)
373 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
383 deallocate (x_cb_glb, y_cb_glb, z_cb_glb)
398 integer,
intent(in) :: t_step
399 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: m_mok, n_mok, p_mok
400 integer(KIND=MPI_OFFSET_KIND),
intent(inout) :: wp_mok, mok, str_mok, nvars_mok
401 integer :: ifile, ierr, data_size
402 integer,
dimension(MPI_STATUS_SIZE) :: status
403 integer(KIND=MPI_OFFSET_KIND) :: disp, var_mok
404 character(LEN=path_len + 2*name_len) :: file_loc
405 logical :: file_exist
406 character(len=10) :: t_step_string
409 if (file_per_process)
then
411 write (file_loc,
'(I0,A1,I7.7,A)') t_step,
'_',
proc_rank,
'.dat'
412 file_loc = trim(case_dir) //
'/restart_data/lustre_' // trim(t_step_string) // trim(mpiiofs) // trim(file_loc)
413 inquire (file=trim(file_loc), exist=file_exist)
416 call mpi_file_open(mpi_comm_self, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
418 if (down_sample)
then
428 if (down_sample)
then
429 data_size = (m + 3)*(n + 3)*(p + 3)
431 data_size = (m + 1)*(n + 1)*(p + 1)
434 m_mok = int(
m_glb + 1, mpi_offset_kind)
435 n_mok = int(
n_glb + 1, mpi_offset_kind)
436 p_mok = int(
p_glb + 1, mpi_offset_kind)
437 wp_mok = int(storage_size(0._stp)/8, mpi_offset_kind)
438 mok = int(1._wp, mpi_offset_kind)
439 str_mok = int(name_len, mpi_offset_kind)
440 nvars_mok = int(sys_size, mpi_offset_kind)
442 if (bubbles_euler .or. hypoelasticity .or. mhd)
then
444 var_mok = int(i, mpi_offset_kind)
445 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
449 var_mok = int(i, mpi_offset_kind)
450 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
455 call mpi_file_close(ifile, ierr)
457 if (down_sample)
then
465 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
468 write (file_loc,
'(I0,A)') t_step,
'.dat'
469 file_loc = trim(case_dir) //
'/restart_data' // trim(mpiiofs) // trim(file_loc)
470 inquire (file=trim(file_loc), exist=file_exist)
473 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
478 var_mok = int(i, mpi_offset_kind)
480 disp = m_mok*max(mok, n_mok)*max(mok, p_mok)*wp_mok*(var_mok - 1)
482 call mpi_file_set_view(ifile, disp, mpi_p,
mpi_io_data%view(i),
'native', mpi_info_int, ierr)
483 call mpi_file_read_all(ifile,
mpi_io_data%var(i)%sf, data_size*mpi_io_type, mpi_io_p, status, ierr)
487 call mpi_file_close(ifile, ierr)
491 call s_mpi_abort(
'File ' // trim(file_loc) //
' is missing. Exiting.')
510 if (down_sample)
then
512 allocate (
q_cons_temp(i)%sf(-1:m + 1,-1:n + 1,-1:p + 1))
522 allocate (
bc_type(1:num_dims,1:2))
524 allocate (
bc_type(1, 1)%sf(0:0,0:n,0:p))
525 allocate (
bc_type(1, 2)%sf(0:0,0:n,0:p))
535 if (parallel_io .neqv. .true.)
then
551 if (down_sample)
then
564 if (chemistry .or. heat_conduction)
then
Noncharacteristic and processor boundary condition application for ghost cells and buffer regions.
Boundary condition restart I/O, capillary/IGR buffer population, and grid-variable buffers.
subroutine s_assign_default_bc_type(bc_type)
Initialize the per-cell boundary condition type arrays with the global default BC values.
subroutine s_read_serial_boundary_condition_files(step_dirpath, bc_type)
Read boundary condition type and buffer data from serial (unformatted) restart files.
subroutine s_read_parallel_boundary_condition_files(bc_type)
Read boundary condition type and buffer data from per-rank parallel files using MPI I/O.
Platform-specific file and directory operations: create, delete, inquire, getcwd, and basename.
impure subroutine my_inquire(fileloc, dircheck)
Inquire on the existence of a directory or file.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the post-process: domain geometry, equation of state, and output database setti...
type(int_bounds_info) offset_y
type(int_bounds_info) bc_z
real(wp), dimension(:), allocatable y_cc
integer proc_rank
Rank of the local processor.
type(mpi_io_ib_var), public mpi_io_ib_data
real(wp), dimension(:), allocatable y_cb
real(wp), dimension(:), allocatable dz
integer buff_size
Number of ghost cells for boundary condition storage.
real(wp), dimension(:), allocatable z_cb
type(int_bounds_info) bc_y
real(wp), dimension(:), allocatable x_cc
type(int_bounds_info) bc_x
real(wp), dimension(:), allocatable x_cb
real(wp), dimension(:), allocatable dy
type(int_bounds_info) offset_x
real(wp), dimension(:), allocatable z_cc
type(int_bounds_info) offset_z
type(mpi_io_var), public mpi_io_data
real(wp), dimension(:), allocatable dx
Cell-width distributions in the x-, y- and z-coordinate directions.
Utility routines for bubble model setup, coordinate transforms, array sampling, and special functions...
elemental subroutine, public s_int_to_str(i, res)
Convert an integer to its trimmed string representation.
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_barrier
Halts all processes until all have reached barrier.
subroutine s_apply_grid_from_global_dim(x_cb_glb, m_dim_glb, m_dim, sidx, bc_beg, bc_end, cb_lo, cb_hi, cw_lo, cw_hi, x_cb_loc, x_cc_loc, dx_loc)
Populate the local cell-boundary, cell-center, and cell-width arrays in one direction directly from t...
impure subroutine s_initialize_mpi_data(q_cons_vf, ib_markers, ib_mpi_data, beta, qbmm_pb, qbmm_mv)
Set up MPI I/O data views and variable pointers for parallel file output.
subroutine s_initialize_mpi_data_ds(m_ds, n_ds, p_ds, q_cons_vf)
Set up MPI I/O data views for downsampled (coarsened) parallel file output.
MPI gather and scatter operations for distributing post-process grid and flow-variable data.
Derived type annexing an integer scalar field (SF).
Derived type annexing a scalar field (SF).