MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_boundary_io.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
2!>
3!! @file
4!! @brief Contains module m_boundary_io
5
6!> @brief Boundary condition restart I/O, capillary/IGR buffer population, and grid-variable buffers
7# 1 "/home/runner/work/MFC/MFC/src/common/include/case.fpp" 1
8! This file exists so that Fypp can be run without generating case.fpp files for
9! each target. This is useful when generating documentation, for example. This
10! should also let MFC be built with CMake directly, without invoking mfc.sh.
11
12! For pre-process.
13# 8 "/home/runner/work/MFC/MFC/src/common/include/case.fpp"
14
15! For moving immersed boundaries in simulation
16# 12 "/home/runner/work/MFC/MFC/src/common/include/case.fpp"
17# 7 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp" 2
18# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
19# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
20# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
21# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
22# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
23# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
24# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
25# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
26
27# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
28# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
29# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
30
31# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
32
33# 46 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
34
35# 58 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
36
37# 68 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
38
39# 98 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
40
41# 110 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
42
43# 120 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
44
45# 145 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
46! New line at end of file is required for FYPP
47# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
48# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
49# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
50# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
51# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
55
56# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
57# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
59
60# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
61
62# 46 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
63
64# 58 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
65
66# 68 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
67
68# 98 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
69
70# 110 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 120 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 145 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75! New line at end of file is required for FYPP
76# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
77
78# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
79# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
80# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
81# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
82# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
83
84# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
85
86# 43 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
87
88# 48 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
89
90# 53 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91
92# 58 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93
94# 63 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 68 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 76 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 81 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 86 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 91 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 96 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 151 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 192 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 206 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 242 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 244 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129# 255 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
130
131# 284 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
132
133# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
134
135# 304 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
136
137# 313 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
138
139# 330 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
140
141# 340 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 347 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 359 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 365 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 371 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 377 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154! New line at end of file is required for FYPP
155# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
156# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
157# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
158# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
159# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
160# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
161# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
162# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
163
164# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
165# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
166# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
167
168# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
169
170# 46 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171
172# 58 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173
174# 68 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175
176# 98 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177
178# 110 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179
180# 120 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
181
182# 145 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
183! New line at end of file is required for FYPP
184# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
185
186# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
187
188# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
189
190# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
191
192# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
193
194# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
195
196# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
197
198# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
199
200# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
201
202# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
203
204# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
205
206# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
207
208# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
209
210# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
211
212# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
213
214# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
215
216# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
217
218# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
219
220# 160 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
221
222# 175 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
223
224# 193 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
225
226# 215 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
227
228# 244 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
229
230# 259 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
231
232# 269 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
233
234# 278 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
235
236# 294 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
237
238# 304 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
239
240# 311 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
241! New line at end of file is required for FYPP
242# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
243
244! GPU parallel region (scalar reductions, maxval/minval)
245# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
246
247! GPU parallel loop over threads (most common GPU macro)
248# 43 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
249
250! Required closing for GPU_PARALLEL_LOOP
251# 55 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
252
253! Mark routine for device compilation
254# 112 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
255
256! Declare device-resident data
257# 130 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
258
259! Inner loop within a GPU parallel region
260# 145 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
261
262! Scoped GPU data region
263# 164 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
264
265! Host code with device pointers (for MPI with GPU buffers)
266# 193 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
267
268! Allocate device memory (unscoped)
269# 207 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
270
271! Free device memory
272# 219 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
273
274! Atomic operation on device
275# 231 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
276
277! End atomic capture block
278# 242 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
279
280! Copy data between host and device
281# 254 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
282
283! Synchronization barrier
284# 266 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
285
286! Import GPU library module (openacc or omp_lib)
287# 275 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
288
289! Emit code only for AMD compiler
290# 282 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
291
292! Emit code for non-Cray compilers
293# 289 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
294
295! Emit code only for Cray compiler
296# 296 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
297
298! Emit code for non-NVIDIA compilers
299# 303 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
300
301# 305 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
302# 306 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
303! New line at end of file is required for FYPP
304# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
305
306# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
307
308! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
309! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
310! example see misc/nvidia_uvm/bind.sh.
311# 57 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
312
313! Allocate and create GPU device memory
314# 77 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
315
316! Free GPU device memory and deallocate
317# 85 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
318
319! Cray-specific GPU pointer setup for vector fields
320# 109 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
321
322! Cray-specific GPU pointer setup for scalar fields
323# 125 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
324
325! Cray-specific GPU pointer setup for acoustic source spatials
326# 150 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
327
328# 156 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
329
330# 163 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
331! New line at end of file is required for FYPP
332# 8 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp" 2
333
334module m_boundary_io
335
336 use m_derived_types
337 use m_global_parameters
338 use m_mpi_proxy
339 use m_constants
340 use m_delay_file_access
341 use m_compile_specific
342 use m_boundary_primitives
343
344 implicit none
345
346#ifdef MFC_MPI
347 integer, dimension(1:3,1:2) :: mpi_bc_type_type
348 integer, dimension(1:3,1:2) :: mpi_bc_buffer_type
349#endif
350
351contains
352
353 !> Create MPI derived datatypes for boundary condition type arrays and buffer arrays used in parallel I/O.
354 impure subroutine s_create_mpi_types(bc_type)
355
356 type(integer_field), dimension(1:num_dims,1:2), intent(in) :: bc_type
357
358#ifdef MFC_MPI
359 integer :: dir, loc
360 integer, dimension(3) :: sf_start_idx, sf_extents_loc
361 integer :: ierr
362
363 do dir = 1, num_dims
364 do loc = 1, 2
365 sf_start_idx = (/0, 0, 0/)
366 sf_extents_loc = shape(bc_type(dir, loc)%sf)
367
368 call mpi_type_create_subarray(num_dims, sf_extents_loc, sf_extents_loc, sf_start_idx, mpi_order_fortran, &
369 & mpi_integer, mpi_bc_type_type(dir, loc), ierr)
370 call mpi_type_commit(mpi_bc_type_type(dir, loc), ierr)
371 end do
372 end do
373
374 do dir = 1, num_dims
375 do loc = 1, 2
376 sf_start_idx = (/0, 0, 0/)
377 sf_extents_loc = shape(bc_buffers(dir, loc)%sf)
378
379 call mpi_type_create_subarray(num_dims, sf_extents_loc*mpi_io_type, sf_extents_loc*mpi_io_type, sf_start_idx, &
380 & mpi_order_fortran, mpi_io_p, mpi_bc_buffer_type(dir, loc), ierr)
381 call mpi_type_commit(mpi_bc_buffer_type(dir, loc), ierr)
382 end do
383 end do
384#endif
385
386 end subroutine s_create_mpi_types
387
388 !> Write boundary condition type and buffer data to serial (unformatted) restart files.
389 subroutine s_write_serial_boundary_condition_files(q_prim_vf, bc_type, step_dirpath, old_grid_in, q_T_sf)
390
391 type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
392 type(integer_field), dimension(1:num_dims,1:2), intent(in) :: bc_type
393 logical, intent(in) :: old_grid_in
394 character(LEN=*), intent(in) :: step_dirpath
395 integer :: dir, loc
396 character(len=path_len) :: file_path
397 character(len=10) :: status
398 type(scalar_field), optional, intent(in) :: q_T_sf
399
400 if (old_grid_in) then
401 status = 'old'
402 else
403 status = 'new'
404 end if
405
406 call s_pack_boundary_condition_buffers(q_prim_vf, q_t_sf)
407
408 file_path = trim(step_dirpath) // '/bc_type.dat'
409 open (1, file=trim(file_path), form='unformatted', status=status)
410 do dir = 1, num_dims
411 do loc = 1, 2
412 write (1) bc_type(dir, loc)%sf
413 end do
414 end do
415 close (1)
416
417 file_path = trim(step_dirpath) // '/bc_buffers.dat'
418 open (1, file=trim(file_path), form='unformatted', status=status)
419 do dir = 1, num_dims
420 do loc = 1, 2
421 write (1) bc_buffers(dir, loc)%sf
422 end do
423 end do
424 close (1)
425
426 end subroutine s_write_serial_boundary_condition_files
427
428 !> Write boundary condition type and buffer data to per-rank parallel files using MPI I/O.
429 subroutine s_write_parallel_boundary_condition_files(q_prim_vf, bc_type, q_T_sf)
430
431 type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
432 type(integer_field), dimension(1:num_dims,1:2), intent(in) :: bc_type
433 integer :: dir, loc
434 character(len=path_len) :: file_loc, file_path
435 type(scalar_field), intent(in), optional :: q_T_sf
436
437#ifdef MFC_MPI
438 integer :: ierr
439 integer :: file_id
440 character(len=7) :: proc_rank_str
441 logical :: dir_check
442 integer :: nelements
443
444 call s_pack_boundary_condition_buffers(q_prim_vf, q_t_sf)
445
446 file_loc = trim(case_dir) // '/restart_data/boundary_conditions'
447 if (proc_rank == 0) then
448 call my_inquire(file_loc, dir_check)
449 if (dir_check .neqv. .true.) then
450 call s_create_directory(trim(file_loc))
451 end if
452 end if
453
454 call s_create_mpi_types(bc_type)
455
456 call s_mpi_barrier()
457
458 call delayfileaccess(proc_rank)
459
460 write (proc_rank_str, '(I7.7)') proc_rank
461 file_path = trim(file_loc) // '/bc_' // trim(proc_rank_str) // '.dat'
462 call mpi_file_open(mpi_comm_self, trim(file_path), mpi_mode_create + mpi_mode_wronly, mpi_info_null, file_id, ierr)
463
464 ! Write bc_types
465 do dir = 1, num_dims
466 do loc = 1, 2
467#ifdef MFC_MIXED_PRECISION
468 nelements = sizeof(bc_type(dir, loc)%sf)
469 call mpi_file_write_all(file_id, bc_type(dir, loc)%sf, nelements, mpi_byte, mpi_status_ignore, ierr)
470#else
471 nelements = sizeof(bc_type(dir, loc)%sf)/4
472 call mpi_file_write_all(file_id, bc_type(dir, loc)%sf, nelements, mpi_integer, mpi_status_ignore, ierr)
473#endif
474 end do
475 end do
476
477 ! Write bc_buffers
478 do dir = 1, num_dims
479 do loc = 1, 2
480 nelements = sizeof(bc_buffers(dir, loc)%sf)*mpi_io_type/stp
481 call mpi_file_write_all(file_id, bc_buffers(dir, loc)%sf, nelements, mpi_io_p, mpi_status_ignore, ierr)
482 end do
483 end do
484
485 call mpi_file_close(file_id, ierr)
486#endif
487
488 end subroutine s_write_parallel_boundary_condition_files
489
490 !> Read boundary condition type and buffer data from serial (unformatted) restart files.
491 subroutine s_read_serial_boundary_condition_files(step_dirpath, bc_type)
492
493 character(LEN=*), intent(in) :: step_dirpath
494 type(integer_field), dimension(1:num_dims,1:2), intent(inout) :: bc_type
495 integer :: dir, loc
496 logical :: file_exist
497 character(len=path_len) :: file_path
498
499 ! Read bc_types
500
501 file_path = trim(step_dirpath) // '/bc_type.dat'
502 inquire (file=trim(file_path), exist=file_exist)
503 if (.not. file_exist) then
504 call s_mpi_abort(trim(file_path) // ' is missing. Exiting.')
505 end if
506
507 open (1, file=trim(file_path), form='unformatted', status='unknown')
508 do dir = 1, num_dims
509 do loc = 1, 2
510 read (1) bc_type(dir, loc)%sf
511
512# 186 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
513#if defined(MFC_OpenACC)
514# 186 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
515!$acc update device(bc_type(dir, loc)%sf)
516# 186 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
517#elif defined(MFC_OpenMP)
518# 186 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
519!$omp target update to(bc_type(dir, loc)%sf)
520# 186 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
521#endif
522 end do
523 end do
524 close (1)
525
526 ! Read bc_buffers
527 file_path = trim(step_dirpath) // '/bc_buffers.dat'
528 inquire (file=trim(file_path), exist=file_exist)
529 if (.not. file_exist) then
530 call s_mpi_abort(trim(file_path) // ' is missing. Exiting.')
531 end if
532
533 open (1, file=trim(file_path), form='unformatted', status='unknown')
534 do dir = 1, num_dims
535 do loc = 1, 2
536 read (1) bc_buffers(dir, loc)%sf
537
538# 202 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
539#if defined(MFC_OpenACC)
540# 202 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
541!$acc update device(bc_buffers(dir, loc)%sf)
542# 202 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
543#elif defined(MFC_OpenMP)
544# 202 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
545!$omp target update to(bc_buffers(dir, loc)%sf)
546# 202 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
547#endif
548 end do
549 end do
550 close (1)
551
552 end subroutine s_read_serial_boundary_condition_files
553
554 !> Read boundary condition type and buffer data from per-rank parallel files using MPI I/O.
555 subroutine s_read_parallel_boundary_condition_files(bc_type)
556
557 type(integer_field), dimension(1:num_dims,1:2), intent(inout) :: bc_type
558 integer :: dir, loc
559 character(len=path_len) :: file_loc, file_path
560
561#ifdef MFC_MPI
562 integer :: ierr
563 integer :: file_id
564 character(len=7) :: proc_rank_str
565 logical :: dir_check
566 integer :: nelements
567
568 file_loc = trim(case_dir) // '/restart_data/boundary_conditions'
569
570 if (proc_rank == 0) then
571 call my_inquire(file_loc, dir_check)
572 if (dir_check .neqv. .true.) then
573 call s_mpi_abort(trim(file_loc) // ' is missing. Exiting.')
574 end if
575 end if
576
577 call s_create_mpi_types(bc_type)
578
579 call s_mpi_barrier()
580
581 call delayfileaccess(proc_rank)
582
583 write (proc_rank_str, '(I7.7)') proc_rank
584 file_path = trim(file_loc) // '/bc_' // trim(proc_rank_str) // '.dat'
585 call mpi_file_open(mpi_comm_self, trim(file_path), mpi_mode_rdonly, mpi_info_null, file_id, ierr)
586
587 ! Read bc_types
588 do dir = 1, num_dims
589 do loc = 1, 2
590#ifdef MFC_MIXED_PRECISION
591 nelements = sizeof(bc_type(dir, loc)%sf)
592 call mpi_file_read_all(file_id, bc_type(dir, loc)%sf, nelements, mpi_byte, mpi_status_ignore, ierr)
593#else
594 nelements = sizeof(bc_type(dir, loc)%sf)/4
595 call mpi_file_read_all(file_id, bc_type(dir, loc)%sf, nelements, mpi_integer, mpi_status_ignore, ierr)
596#endif
597
598# 252 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
599#if defined(MFC_OpenACC)
600# 252 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
601!$acc update device(bc_type(dir, loc)%sf)
602# 252 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
603#elif defined(MFC_OpenMP)
604# 252 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
605!$omp target update to(bc_type(dir, loc)%sf)
606# 252 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
607#endif
608 end do
609 end do
610
611 ! Read bc_buffers
612 do dir = 1, num_dims
613 do loc = 1, 2
614 nelements = sizeof(bc_buffers(dir, loc)%sf)*mpi_io_type/stp
615 call mpi_file_read_all(file_id, bc_buffers(dir, loc)%sf, nelements, mpi_io_p, mpi_status_ignore, ierr)
616
617# 261 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
618#if defined(MFC_OpenACC)
619# 261 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
620!$acc update device(bc_buffers(dir, loc)%sf)
621# 261 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
622#elif defined(MFC_OpenMP)
623# 261 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
624!$omp target update to(bc_buffers(dir, loc)%sf)
625# 261 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
626#endif
627 end do
628 end do
629
630 call mpi_file_close(file_id, ierr)
631#endif
632
633 end subroutine s_read_parallel_boundary_condition_files
634
635 !> Pack primitive variable boundary slices into bc_buffers arrays for serialization.
636 subroutine s_pack_boundary_condition_buffers(q_prim_vf, q_T_sf)
637
638 type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
639 integer :: i, j, k
640 type(scalar_field), intent(in), optional :: q_T_sf
641
642 do k = 0, p
643 do j = 0, n
644 do i = 1, sys_size
645 bc_buffers(1, 1)%sf(i, j, k) = q_prim_vf(i)%sf(0, j, k)
646 bc_buffers(1, 2)%sf(i, j, k) = q_prim_vf(i)%sf(m, j, k)
647 end do
648 if (chemistry .and. present(q_t_sf)) then
649 bc_buffers(1, 1)%sf(sys_size + 1, j, k) = q_t_sf%sf(0, j, k)
650 bc_buffers(1, 2)%sf(sys_size + 1, j, k) = q_t_sf%sf(m, j, k)
651 end if
652 end do
653 end do
654
655# 291 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
656 if (n > 0) then
657 do k = 0, p
658 do j = 1, sys_size
659 do i = 0, m
660 bc_buffers(2, 1)%sf(i, j, k) = q_prim_vf(j)%sf(i, 0, k)
661 bc_buffers(2, 2)%sf(i, j, k) = q_prim_vf(j)%sf(i, n, k)
662 end do
663 end do
664 if (chemistry .and. present(q_t_sf)) then
665 do i = 0, m
666 bc_buffers(2, 1)%sf(i, sys_size + 1, k) = q_t_sf%sf(i, 0, k)
667 bc_buffers(2, 2)%sf(i, sys_size + 1, k) = q_t_sf%sf(i, n, k)
668 end do
669 end if
670 end do
671
672# 308 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
673 if (p > 0) then
674 do k = 1, sys_size
675 do j = 0, n
676 do i = 0, m
677 bc_buffers(3, 1)%sf(i, j, k) = q_prim_vf(k)%sf(i, j, 0)
678 bc_buffers(3, 2)%sf(i, j, k) = q_prim_vf(k)%sf(i, j, p)
679 end do
680 end do
681 end do
682 if (chemistry .and. present(q_t_sf)) then
683 do j = 0, n
684 do i = 0, m
685 bc_buffers(3, 1)%sf(i, j, sys_size + 1) = q_t_sf%sf(i, j, 0)
686 bc_buffers(3, 2)%sf(i, j, sys_size + 1) = q_t_sf%sf(i, j, p)
687 end do
688 end do
689 end if
690 end if
691# 327 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
692 end if
693# 329 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
694
695 end subroutine s_pack_boundary_condition_buffers
696
697 !> Initialize the per-cell boundary condition type arrays with the global default BC values.
698 subroutine s_assign_default_bc_type(bc_type)
699
700 type(integer_field), dimension(1:num_dims,1:2), intent(in) :: bc_type
701
702 bc_type(1, 1)%sf(:,:,:) = int(min(bc_x%beg, 0), kind=1)
703 bc_type(1, 2)%sf(:,:,:) = int(min(bc_x%end, 0), kind=1)
704
705# 339 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
706#if defined(MFC_OpenACC)
707# 339 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
708!$acc update device(bc_type(1, 1)%sf, bc_type(1, 2)%sf)
709# 339 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
710#elif defined(MFC_OpenMP)
711# 339 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
712!$omp target update to(bc_type(1, 1)%sf, bc_type(1, 2)%sf)
713# 339 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
714#endif
715
716# 342 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
717 if (n > 0) then
718 bc_type(2, 1)%sf(:,:,:) = int(min(bc_y%beg, 0), kind=1)
719 bc_type(2, 2)%sf(:,:,:) = int(min(bc_y%end, 0), kind=1)
720
721# 345 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
722#if defined(MFC_OpenACC)
723# 345 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
724!$acc update device(bc_type(2, 1)%sf, bc_type(2, 2)%sf)
725# 345 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
726#elif defined(MFC_OpenMP)
727# 345 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
728!$omp target update to(bc_type(2, 1)%sf, bc_type(2, 2)%sf)
729# 345 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
730#endif
731# 347 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
732 if (p > 0) then
733 bc_type(3, 1)%sf(:,:,:) = int(min(bc_z%beg, 0), kind=1)
734 bc_type(3, 2)%sf(:,:,:) = int(min(bc_z%end, 0), kind=1)
735
736# 350 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
737#if defined(MFC_OpenACC)
738# 350 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
739!$acc update device(bc_type(3, 1)%sf, bc_type(3, 2)%sf)
740# 350 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
741#elif defined(MFC_OpenMP)
742# 350 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
743!$omp target update to(bc_type(3, 1)%sf, bc_type(3, 2)%sf)
744# 350 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
745#endif
746 end if
747# 353 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
748 end if
749# 355 "/home/runner/work/MFC/MFC/src/common/m_boundary_io.fpp"
750
751 end subroutine s_assign_default_bc_type
752
753end module m_boundary_io
m_boundary_io
Boundary condition restart I/O, capillary/IGR buffer population, and grid-variable buffers.
Definition m_boundary_io.fpp.f90:334
m_boundary_io::mpi_bc_type_type
integer, dimension(1:3, 1:2) mpi_bc_type_type
Definition m_boundary_io.fpp.f90:347
m_boundary_io::s_write_parallel_boundary_condition_files
subroutine s_write_parallel_boundary_condition_files(q_prim_vf, bc_type, q_t_sf)
Write boundary condition type and buffer data to per-rank parallel files using MPI I/O.
Definition m_boundary_io.fpp.f90:430
m_boundary_io::mpi_bc_buffer_type
integer, dimension(1:3, 1:2) mpi_bc_buffer_type
Definition m_boundary_io.fpp.f90:348
m_boundary_io::s_assign_default_bc_type
subroutine s_assign_default_bc_type(bc_type)
Initialize the per-cell boundary condition type arrays with the global default BC values.
Definition m_boundary_io.fpp.f90:699
m_boundary_io::s_read_serial_boundary_condition_files
subroutine s_read_serial_boundary_condition_files(step_dirpath, bc_type)
Read boundary condition type and buffer data from serial (unformatted) restart files.
Definition m_boundary_io.fpp.f90:492
m_boundary_io::s_create_mpi_types
impure subroutine s_create_mpi_types(bc_type)
Create MPI derived datatypes for boundary condition type arrays and buffer arrays used in parallel I/...
Definition m_boundary_io.fpp.f90:355
m_boundary_io::s_pack_boundary_condition_buffers
subroutine s_pack_boundary_condition_buffers(q_prim_vf, q_t_sf)
Pack primitive variable boundary slices into bc_buffers arrays for serialization.
Definition m_boundary_io.fpp.f90:637
m_boundary_io::s_read_parallel_boundary_condition_files
subroutine s_read_parallel_boundary_condition_files(bc_type)
Read boundary condition type and buffer data from per-rank parallel files using MPI I/O.
Definition m_boundary_io.fpp.f90:556
m_boundary_io::s_write_serial_boundary_condition_files
subroutine s_write_serial_boundary_condition_files(q_prim_vf, bc_type, step_dirpath, old_grid_in, q_t_sf)
Write boundary condition type and buffer data to serial (unformatted) restart files.
Definition m_boundary_io.fpp.f90:390
m_boundary_primitives
Per-cell noncharacteristic boundary condition primitives applied in the ghost cells.
Definition m_boundary_primitives.fpp.f90:334
m_boundary_primitives::bc_buffers
type(scalar_field), dimension(:,:), allocatable bc_buffers
Definition m_boundary_primitives.fpp.f90:343
m_compile_specific
Platform-specific file and directory operations: create, delete, inquire, getcwd, and basename.
Definition m_compile_specific.f90:6
m_compile_specific::my_inquire
impure subroutine my_inquire(fileloc, dircheck)
Inquire on the existence of a directory or file.
Definition m_compile_specific.f90:56
m_compile_specific::s_create_directory
impure subroutine s_create_directory(dir_name)
Create a directory and all its parents if it does not exist.
Definition m_compile_specific.f90:17
m_constants
Compile-time constant parameters: default values, tolerances, and physical constants.
Definition m_constants.fpp.f90:7
m_delay_file_access
Rank-staggered file access delays to prevent I/O contention on parallel file systems.
Definition m_delay_file_access.f90:6
m_delay_file_access::delayfileaccess
impure subroutine, public delayfileaccess(processrank)
Introduce a rank-dependent busy-wait delay to stagger parallel file access and reduce I/O contention.
Definition m_delay_file_access.f90:22
m_derived_types
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Definition m_derived_types.fpp.f90:323
m_global_parameters
Defines global parameters for the computational domain, simulation algorithm, and initial conditions.
Definition m_global_parameters.fpp.f90:19
m_global_parameters::bc_z
type(int_bounds_info) bc_z
Boundary conditions in the x-, y- and z-coordinate directions.
Definition m_global_parameters.fpp.f90:66
m_global_parameters::proc_rank
integer proc_rank
Rank of the local processor Number of cells in the x-, y- and z-coordinate directions.
Definition m_global_parameters.fpp.f90:39
m_global_parameters::bc_y
type(int_bounds_info) bc_y
Definition m_global_parameters.fpp.f90:66
m_global_parameters::bc_x
type(int_bounds_info) bc_x
Definition m_global_parameters.fpp.f90:66
m_mpi_proxy
Broadcasts user inputs and decomposes the domain across MPI ranks for pre-processing.
Definition m_mpi_proxy.fpp.f90:7
m_derived_types::integer_field
Derived type annexing an integer scalar field (SF).
Definition m_derived_types.fpp.f90:347
m_derived_types::scalar_field
Derived type annexing a scalar field (SF).
Definition m_derived_types.fpp.f90:337