MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_amr_restart.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
2!>
3!!@file
4!!@brief Contains module m_amr_restart
5
6# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
7# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
8# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
9# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
10# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
11# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14
15# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
16# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
17# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18
19# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
20# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
21! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
22! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
23# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
24# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
25# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
26
27# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
28# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
29
30# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
31
32# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
33
34# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
35
36# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
37
38# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
39
40# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
41
42# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
43
44# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
45! New line at end of file is required for FYPP
46# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
47# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
48# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
49# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
50# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
51# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54
55# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
56# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
57# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58
59# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
60# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
61! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
62! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
63# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
64# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
65# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
66
67# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
68# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
69
70# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75
76# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
77
78# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
79
80# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
81
82# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
83
84# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
85! New line at end of file is required for FYPP
86# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
87
88# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
89# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
90# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93
94# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129
130# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
131
132# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
133
134# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
135
136# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
137
138# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
139# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
140
141# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154
155# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
156
157# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
158
159# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
160
161# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
162
163# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
164! New line at end of file is required for FYPP
165# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
166# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
167# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
168# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
169# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
170# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173
174# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
176# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177
178# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
180! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
181! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
182# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
183# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
184# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
185
186# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
187# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
188
189# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
190
191# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
192
193# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
194
195# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
196
197# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
198
199# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
200
201# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
202
203# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
204! New line at end of file is required for FYPP
205# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
206
207# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
208
209# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
210
211# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
212
213# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
214
215# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
216
217# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
218
219# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
220
221# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
222
223# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
224
225# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
226
227# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
228
229# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
230
231# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
232
233# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
234
235# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
236
237# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
238
239# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
240
241# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
242
243# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
244
245# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
246
247# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
248
249# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
250
251# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
252
253# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
254
255# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
256
257# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
258
259# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
260
261# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
262! New line at end of file is required for FYPP
263# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
264
265! GPU parallel region (scalar reductions, maxval/minval)
266# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
267
268! GPU parallel loop over threads (most common GPU macro)
269# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
270
271! Required closing for GPU_PARALLEL_LOOP
272# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
273
274! Mark routine for device compilation
275# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
276
277! Declare device-resident data
278# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
279
280! Inner loop within a GPU parallel region
281# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
282
283! Scoped GPU data region
284# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
285
286! Host code with device pointers (for MPI with GPU buffers)
287# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
288
289! Allocate device memory (unscoped)
290# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
291
292! Free device memory
293# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
294
295! Atomic operation on device
296# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
297
298! End atomic capture block
299# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
300
301! Copy data between host and device
302# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
303
304! Synchronization barrier
305# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
306
307! Import GPU library module (openacc or omp_lib)
308# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
309
310! Emit code only for AMD compiler
311# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
312
313! Emit code for non-Cray compilers
314# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
315
316! Emit code only for Cray compiler
317# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
318
319! Cray compiler directive, emitted from Fypp so the formatter cannot turn it into a comment
320# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
321
322! Emit code for non-NVIDIA compilers
323# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
324
325# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
326# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
327! New line at end of file is required for FYPP
328# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
329
330# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
331
332! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
333! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
334! example see misc/nvidia_uvm/bind.sh.
335# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
336
337! Allocate and create GPU device memory
338# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
339
340! Free GPU device memory and deallocate
341# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
342
343! Cray-specific GPU pointer setup for vector fields
344# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
345
346! Cray-specific GPU pointer setup for scalar fields
347# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
348
349! Cray-specific GPU pointer teardown for scalar fields - exact inverse of ACC_SETUP_SFs. Removes the
350! present-table entries ACC_SETUP_SFs added (the scalar_field descriptor and its %sf copyin). REQUIRED when
351! an ACC_SETUP'd field is freed MID-RUN (e.g. AMR slot free on regrid/restart): Cray 'exit data delete'
352! decrements the reference counter, so the lone @:DEALLOCATE(arg%sf) only undoes the @:ALLOCATE create ref
353! and leaves the descriptor + the ACC_SETUP %sf copyin dangling. Call BEFORE @:DEALLOCATE(arg%sf).
354# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
355
356! Cray-specific GPU pointer setup for acoustic source spatials
357# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
358
359! Whether stress index idx is a shear component. Spelled out because NVHPC 26.9 at -O2 vectorizes any(idx == shear_indices)
360! inside a loop to always true, which doubled the normal-stress elastic energy.
361# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
362
363# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
364
365# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
366! New line at end of file is required for FYPP
367# 6 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp" 2
368
369!> @brief AMR fine-level restart I/O: writes/reads the fine-level restart file alongside the level-0 restart (serial per-rank
370!! unformatted files, or one shared MPI-IO file under parallel_io).
372
373#ifdef MFC_MPI
374 use mpi !< mpi-io for the parallel_io amr restart file
375#endif
376
377 use m_derived_types ! scalar_field
380 use m_mpi_proxy, only: s_mpi_abort
384 use m_amr_state, only: max_f
385 use m_amr, only: s_amr_reduce_xchg_flag, amr_slots, amr_cons_st, amr_loc_of, amr_seam_pairs_dirty, amr_mesh_epoch, &
386 & s_amr_alloc_slot, s_amr_reconcile_slots, s_amr_assign_block_owners, s_set_amr_fine_geometry
389
390 implicit none
391
392 private
394
395contains
396
397 !> Serial: one unformatted file per rank in the level-0 step directory just created by the serial write.
398 impure subroutine s_amr_restart_write_serial(t_step)
399
400 integer, intent(in) :: t_step
401 character(LEN=path_len + 3*name_len) :: file_loc
402 integer :: i, k, wext(3)
403
404 file_loc = f_amr_restart_path(t_step)
405 open (2, file=trim(file_loc), form='unformatted', status='new')
406 write (2) num_procs, amr_num_blocks, sys_size, amr_ref_ratio, amr_max_level, storage_size(0._stp)/8
407 do k = 1, amr_num_blocks
408 ! per-block header: region box, level (the reader needs it to rebuild multi-level geometry), extents; -1 extents
409 ! mark a slot this rank does not own (the reader's data-presence flag)
410 wext = -1
411 if (amr_owns_all(k)) wext = [amr_slots(k)%m, amr_slots(k)%n, amr_slots(k)%p]
412 write (2) amr_region_lo_all(:,k), amr_region_hi_all(:,k), amr_block_level(k), wext
413 if (amr_owns_all(k)) then
414 do i = 1, sys_size
415 write (2) amr_cons_st(0:amr_slots(k)%m,0:amr_slots(k)%n,0:amr_slots(k)%p,i, amr_loc_of(k))
416 end do
417 end if
418 end do
419 close (2)
420
421 end subroutine s_amr_restart_write_serial
422
423#ifdef MFC_MPI
424 !> Parallel: one shared MPI-IO file.
425 impure subroutine s_amr_restart_write_parallel(t_step)
426
427 integer, intent(in) :: t_step
428 character(LEN=path_len + 3*name_len) :: file_loc
429 integer :: i, k
430 integer :: ifile, ierr, cnt, idx, fi, fj, fk, bhdr(amr_restart_blk_hdr_ints), ibytes, sbytes
431 integer, allocatable :: myown_all(:)
432 integer, dimension(MPI_STATUS_SIZE) :: status
433 integer(kind=MPI_OFFSET_KIND) :: my_off, disp0, ddisp
434 integer(kind=MPI_OFFSET_KIND), allocatable :: my_cnt_vec(:), my_off_vec(:), tot_cnt_vec(:)
435 logical :: file_exist
436 real(stp), allocatable :: buf(:)
437
438 ibytes = storage_size(0)/8; sbytes = storage_size(0._stp)/8
439 file_loc = f_amr_restart_path(t_step)
440 inquire (file=trim(file_loc), exist=file_exist)
441 if (file_exist .and. proc_rank == 0) then
442 call mpi_file_delete(file_loc, mpi_info_int, ierr)
443 end if
444 call mpi_file_open(mpi_comm_world, file_loc, ior(mpi_mode_wronly, mpi_mode_create), mpi_info_int, ifile, ierr)
445 ! MPI-IO errors are silent unless checked
446 if (ierr /= mpi_success) call s_mpi_abort('amr restart write: MPI_FILE_OPEN failed for ' // trim(file_loc))
447 if (proc_rank == 0) call mpi_file_write_at(ifile, int(0, mpi_offset_kind), [-num_procs, amr_num_blocks, sys_size, &
448 & amr_ref_ratio, amr_max_level, sbytes], amr_restart_ghdr_ints, mpi_integer, status, ierr)
449 disp0 = int(amr_restart_ghdr_ints*ibytes, mpi_offset_kind) ! running byte offset past the global header
450 ! one EXSCAN/ALLREDUCE over all blocks, not per block
451 allocate (my_cnt_vec(amr_num_blocks), my_off_vec(amr_num_blocks), tot_cnt_vec(amr_num_blocks))
452 allocate (myown_all(amr_restart_blk_own_ints*amr_num_blocks))
453 do k = 1, amr_num_blocks
454 cnt = sys_size*(amr_slots(k)%m + 1)*(amr_slots(k)%n + 1)*(amr_slots(k)%p + 1)
455 if (.not. amr_owns_all(k)) cnt = 0
456 my_cnt_vec(k) = int(cnt, mpi_offset_kind)
458 if (amr_owns_all(k)) myown_all(amr_restart_blk_own_ints*(k - 1) + 1:amr_restart_blk_own_ints*k) = [proc_rank + 1, &
459 & amr_slots(k)%m, amr_slots(k)%n, amr_slots(k)%p]
460 end do
461 my_off_vec = int(0, mpi_offset_kind)
462 call mpi_exscan(my_cnt_vec, my_off_vec, amr_num_blocks, mpi_offset, mpi_sum, mpi_comm_world, ierr)
463 if (proc_rank == 0) my_off_vec = int(0, mpi_offset_kind)
464 call mpi_allreduce(my_cnt_vec, tot_cnt_vec, amr_num_blocks, mpi_offset, mpi_sum, mpi_comm_world, ierr)
465 ! Per-block (owner + 1, m, n, p): every rank contributes zeros but the owner, so MAX recovers the record. The reader
466 ! checks it against its own decomposition, catching an ownership pattern that would misalign the concatenated data.
467 call mpi_allreduce(mpi_in_place, myown_all, amr_restart_blk_own_ints*amr_num_blocks, mpi_integer, mpi_max, &
468 & mpi_comm_world, ierr)
469 do k = 1, amr_num_blocks
470 cnt = int(my_cnt_vec(k), kind(cnt))
471 my_off = my_off_vec(k)
472 if (proc_rank == 0) then
473 ! block header (layout single-sourced in m_constants): region box (6) + level
474 bhdr(1:3) = amr_region_lo_all(:,k); bhdr(4:6) = amr_region_hi_all(:,k)
476 call mpi_file_write_at(ifile, disp0, bhdr, amr_restart_blk_hdr_ints, mpi_integer, status, ierr)
477 end if
478 if (proc_rank == 0) then
479 call mpi_file_write_at(ifile, disp0 + int(amr_restart_blk_hdr_ints*ibytes, mpi_offset_kind), &
480 & myown_all(amr_restart_blk_own_ints*(k - 1) + 1), amr_restart_blk_own_ints, mpi_integer, &
481 & status, ierr)
482 end if
483 ddisp = disp0 + int((amr_restart_blk_hdr_ints + amr_restart_blk_own_ints)*ibytes, mpi_offset_kind)
484 allocate (buf(max(cnt, 1)))
485 idx = 0
486 ! non-owners have cnt == 0 and an unallocated q_cons, so skip packing; the collective write below still runs
487 if (cnt > 0) then
488 do i = 1, sys_size
489 do fk = 0, amr_slots(k)%p
490 do fj = 0, amr_slots(k)%n
491 do fi = 0, amr_slots(k)%m
492 idx = idx + 1
493 buf(idx) = amr_cons_st(fi, fj, fk, i, amr_loc_of(k))
494 end do
495 end do
496 end do
497 end do
498 end if
499 call mpi_file_write_at_all(ifile, ddisp + my_off*int(sbytes, mpi_offset_kind), buf, cnt*mpi_io_type, mpi_io_p, &
500 & status, ierr)
501 if (ierr /= mpi_success) &
502 & call s_mpi_abort('amr restart write: data write failed (disk full/quota?); the file is unusable')
503 deallocate (buf)
504 disp0 = ddisp + tot_cnt_vec(k)*int(sbytes, mpi_offset_kind)
505 end do
506 deallocate (my_cnt_vec, my_off_vec, tot_cnt_vec, myown_all)
507 ! close is where buffered MPI-IO data flushes: a failure here truncates the file
508 call mpi_file_close(ifile, ierr)
509 if (ierr /= mpi_success) call s_mpi_abort('amr restart write: MPI_FILE_CLOSE failed; the file may be truncated')
510
511 end subroutine s_amr_restart_write_parallel
512#endif
513
514 !> Write the fine-level restart file for save step t_step alongside the level-0 restart: the writing rank count, the active
515 !! block count, and per block its box + level and each rank's fine conservative state. Serial: one unformatted file per rank in
516 !! its level-0 step directory. Parallel: one shared MPI-IO file (header [-np, nboxes, sys_size, ratio, max level, word bytes],
517 !! then per block a box+level header, a 4-int ownership record [owner+1, m, n, p], then the ranks' blocks concatenated in rank
518 !! order). At the same rank count the ownership record must match on read; a different rank count repartitions.
519 impure subroutine s_write_amr_restart(t_step)
520
521 integer, intent(in) :: t_step
522 integer :: k
523
524 if (.not. amr) return
525 ! host consumer: fine state is device-current during stepping
526 do k = 1, amr_num_blocks
527 if (amr_owns_all(k)) then
528
529# 166 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
530#if defined(MFC_OpenACC)
531# 166 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
532!$acc update host(amr_cons_st(:, :, :, :, amr_loc_of(k)))
533# 166 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
534#elif defined(MFC_OpenMP)
535# 166 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
536!$omp target update from(amr_cons_st(:, :, :, :, amr_loc_of(k)))
537# 166 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
538#endif
539# 167 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
540 end if
541 end do
542
543 call s_amr_write_level_sums(t_step)
544 if (.not. parallel_io) then
545 call s_amr_restart_write_serial(t_step)
546 else
547#ifdef MFC_MPI
549#endif
550 end if
551
552 end subroutine s_write_amr_restart
553
554 !> Write D/amr_levels.<t_step>.dat: per refined level, its fine cell count and the mean of |q| over its cells for each equation.
555 !! Puts fine-level data under the golden comparison; the level-0 output only sees it restricted.
556 impure subroutine s_amr_write_level_sums(t_step)
557
558 integer, intent(in) :: t_step
559 real(wp) :: sums(sys_size + 1, amr_max_level)
560 character(LEN=path_len + 2*name_len) :: file_loc
561 integer :: i, k, l
562
563 sums = 0._wp
564 do k = 1, amr_num_blocks
565 if (.not. amr_owns_all(k)) cycle
567 sums(1, l) = sums(1, l) + real((amr_slots(k)%m + 1)*(amr_slots(k)%n + 1)*(amr_slots(k)%p + 1), wp)
568 do i = 1, sys_size
569 sums(i + 1, l) = sums(i + 1, l) + sum(abs(real(amr_cons_st(0:amr_slots(k)%m,0:amr_slots(k)%n,0:amr_slots(k)%p,i, &
570 & amr_loc_of(k)), wp)))
571 end do
572 end do
573 call s_mpi_allreduce_vectors_sum(sums, sums, sys_size + 1, amr_max_level)
574 ! means, not sums: per-cell differences within a golden tolerance would add up past it in a sum
575 do l = 1, amr_max_level
576 sums(2:,l) = sums(2:,l)/max(sums(1, l), 1._wp)
577 end do
578 call s_create_d_dir()
579 if (proc_rank /= 0) return
580
581 write (file_loc, '(A,I6.6,A)') trim(case_dir) // '/D/amr_levels.', t_step, '.dat'
582 open (2, file=trim(file_loc), form='formatted', status='replace')
583 do l = 1, amr_max_level
584 write (2, '(I0,*(1X,ES24.16E3))') l, sums(:,l)
585 end do
586 close (2)
587
588 end subroutine s_amr_write_level_sums
589
590 !> Restore the fine level from the AMR restart file at t_step_start (n_start under cfl_dt), if one exists: rebuild each saved
591 !! block via s_set_amr_fine_geometry and read the fine state (exact stp round-trip). parallel_io repartitions across rank
592 !! counts; serial needs the writing rank count. restored = false on a fresh start (the caller prolongs from coarse); a missing
593 !! file aborts. Collective.
594 impure subroutine s_read_amr_restart(restored)
595
596 logical, intent(out) :: restored
597 character(LEN=path_len + 3*name_len) :: file_loc
598 logical :: file_exist
599 integer :: ts, have_loc, have_glb
600
601 restored = .false.
602 if (.not. amr) return
603 if (cfl_dt) then
604 ts = n_start
605 else
606 ts = t_step_start
607 end if
608 if (ts == 0) return ! fresh start: the fine level is prolonged from the pre_process ICs
609
610 file_loc = f_amr_restart_path(ts)
611 inquire (file=trim(file_loc), exist=file_exist)
612 have_loc = merge(1, 0, file_exist)
613 call s_mpi_allreduce_integer_min(have_loc, have_glb)
614 if (have_glb == 0) call s_mpi_abort('amr restart: no AMR fine-level restart file ' // trim(file_loc) &
615 & // ' (missing on at ' // 'least one rank); the fine level cannot be restored')
616
617 if (parallel_io) then
618 call s_amr_restart_read_parallel(file_loc)
619 else
620 call s_amr_restart_read_serial(file_loc)
621 end if
622
623 call s_amr_select_slot(1)
624 ! the per-level fill waves iterate 2..amr_num_levels, which would skip every level>=2 fill until the first regrid
626 amr_seam_pairs_dirty = .true. ! restored a new block set: the cached seam-pair list must be rebuilt
628 call s_amr_check_seam_topology() ! abort on seam topologies no halo reconciles (e.g. restart mode-switch)
629 restored = .true.
630
631 end subroutine s_read_amr_restart
632
633 !> Install the restored block set: owners from the regions (or the recorded table), this run's owned slots (freeing stale init
634 !! slots), then each block's geometry.
635 impure subroutine s_amr_restart_install_blocks(own)
636
637 integer, intent(in), optional :: own(:) !< recorded owner per block, when it is valid for this run
638 integer :: k
639
640 if (present(own)) then
642 else
644 end if
645 call s_amr_reconcile_slots()
646 do k = 1, amr_num_blocks
647 amr_cur = k
648 call s_set_amr_fine_geometry(amr_region_lo_all(:,k), amr_region_hi_all(:,k))
649 end do
650 call s_amr_reduce_xchg_flag()
651
652 end subroutine s_amr_restart_install_blocks
653
654 !> Serial (per-rank files) read: every block's region, level and, where this rank owned it at write (rm >= 0), its fine state.
655 !! The presence flag is verified against this run's ownership once the owners are rebuilt.
656 impure subroutine s_amr_restart_read_serial(file_loc)
657
658 character(LEN=*), intent(in) :: file_loc
659 integer :: i, k, ghdr(amr_restart_ghdr_ints), reg(6), lvl, rm, rn, rp, nblk
660 logical, allocatable :: had_data(:)
661
662 open (2, file=trim(file_loc), form='unformatted', action='read', status='old')
663 read (2) ghdr
664 call s_amr_restart_check_header(ghdr, sys_size, 'amr restart', nblk, amr_ref_ratio, amr_max_level)
666 amr_num_blocks = nblk
667 allocate (had_data(amr_num_blocks))
668 do k = 1, amr_num_blocks
669 read (2) reg, lvl, rm, rn, rp
670 call s_amr_restart_check_record(k, reg, lvl)
671 had_data(k) = rm >= 0
672 if (.not. had_data(k)) cycle
673 ! owner extents are region-derived per level; a disagreeing stored extent is corrupt
674 if (rm /= (amr_ref_ratio**lvl)*(reg(4) - reg(1) + 1) - 1 .or. rn /= merge((amr_ref_ratio**lvl)*(reg(5) - reg(2) + 1) &
675 & - 1, 0, n_glb > 0) .or. rp /= merge((amr_ref_ratio**lvl)*(reg(6) - reg(3) + 1) - 1, 0, p_glb > 0)) then
676 call s_mpi_abort('amr restart: block fine extents disagree with the region (corrupt file)')
677 end if
678 call s_amr_alloc_slot(k)
679 ! zero the whole host column first: the read fills only the interior but the push covers the padded column
680 amr_cons_st(:,:,:,:,amr_loc_of(k)) = 0._stp
681 do i = 1, sys_size
682 read (2) amr_cons_st(0:rm,0:rn,0:rp,i, amr_loc_of(k))
683 end do
684 ! Push this block before the next s_amr_alloc_slot: a store growth pulls device->host first and would overwrite this
685 ! block with the stale device copy. The store is device-authoritative at every alloc point.
686
687# 313 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
688#if defined(MFC_OpenACC)
689# 313 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
690!$acc update device(amr_cons_st(:, :, :, :, amr_loc_of(k)))
691# 313 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
692#elif defined(MFC_OpenMP)
693# 313 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
694!$omp target update to(amr_cons_st(:, :, :, :, amr_loc_of(k)))
695# 313 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
696#endif
697# 314 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
698 end do
699 close (2)
701 do k = 1, amr_num_blocks
702 if (had_data(k) .neqv. amr_owns_all(k)) then
703 call s_mpi_abort('amr restart decomposition mismatch: the file''s block ownership differs from this' &
704 & // ' run''s (identical decomposition and rank count required)')
705 end if
706 end do
707 deallocate (had_data)
708
709 end subroutine s_amr_restart_read_serial
710
711 !> Parallel (MPI-IO) read. The writer's rank count sets only the file layout: each block's data is one contiguous chunk, so any
712 !! rank count can read it; owners are re-assigned and each new owner reads its blocks. At the writer's rank count the file's
713 !! ownership record must reproduce this run's decomposition.
714 impure subroutine s_amr_restart_read_parallel(file_loc)
715
716 character(LEN=*), intent(in) :: file_loc
717
718#ifdef MFC_MPI
719 integer :: i, k, ghdr(amr_restart_ghdr_ints), nblk, ifile, ierr, cnt, idx, fi, fj, fk
720 integer :: mown(amr_restart_blk_own_ints)
721 type(t_amr_restart_catalog) :: cat
722 integer, dimension(MPI_STATUS_SIZE) :: status
723 integer(kind=MPI_OFFSET_KIND) :: fsz
724 real(stp), allocatable :: buf(:)
725
726 call mpi_file_open(mpi_comm_world, file_loc, mpi_mode_rdonly, mpi_info_int, ifile, ierr)
727 if (ierr /= mpi_success) call s_mpi_abort('amr restart read: MPI_FILE_OPEN failed for ' // trim(file_loc))
728 ! a read past EOF returns short with an uninitialized tail, so compare the file size to the catalog's layout below
729 call mpi_file_get_size(ifile, fsz, ierr)
730 call mpi_file_read_at_all(ifile, int(0, mpi_offset_kind), ghdr, amr_restart_ghdr_ints, mpi_integer, status, ierr)
731 call s_amr_restart_check_header(ghdr, sys_size, 'amr restart', nblk, amr_ref_ratio, amr_max_level)
733 amr_num_blocks = nblk
734 call s_amr_restart_read_catalog(ifile, amr_num_blocks, sys_size, cat)
735 if (cat%end_disp /= fsz) call s_mpi_abort('amr restart read: file size does not match the expected layout ' &
736 & // '(truncated or corrupt amr restart file)')
737 do k = 1, amr_num_blocks
738 call s_amr_restart_check_record(k, cat%reg(:,k), cat%lvl(k))
739 end do
740 ! at the writer's rank count, keep the recorded owners (every fine block must carry one)
741 if (abs(ghdr(1)) == num_procs .and. all(cat%own(1,1:amr_num_blocks) > 0)) then
742 call s_amr_restart_install_blocks(cat%own(1,1:amr_num_blocks) - 1)
743 else
745 end if
746 do k = 1, amr_num_blocks
747 cnt = 0
748 if (amr_owns_all(k)) then
749 cnt = sys_size*(amr_slots(k)%m + 1)*(amr_slots(k)%n + 1)*(amr_slots(k)%p + 1)
750 mown = [proc_rank + 1, amr_slots(k)%m, amr_slots(k)%n, amr_slots(k)%p]
751 if (abs(ghdr(1)) == num_procs .and. any(cat%own(:,k) /= mown)) then
752 call s_mpi_abort('amr restart: the per-block owner/extent record in the file does not match ' &
753 & // 'this run''s decomposition; with the same rank count the ownership must match ' &
754 & // 'the run that wrote the restart')
755 end if
756 end if
757 allocate (buf(max(cnt, 1)))
758 ! collective: every rank calls it, non-owners with count 0
759 call mpi_file_read_at_all(ifile, cat%data_disp(k), buf, cnt*mpi_io_type, mpi_io_p, status, ierr)
760 ! owner only: zero the whole host column (the unpack fills only the interior, the push covers the padded column)
761 if (cnt > 0) amr_cons_st(:,:,:,:,amr_loc_of(k)) = 0._stp
762 idx = 0
763 do i = 1, sys_size
764 do fk = 0, amr_slots(k)%p
765 do fj = 0, amr_slots(k)%n
766 do fi = 0, amr_slots(k)%m
767 idx = idx + 1
768 amr_cons_st(fi, fj, fk, i, amr_loc_of(k)) = buf(idx)
769 end do
770 end do
771 end do
772 end do
773 deallocate (buf)
774 end do
775 call mpi_file_close(ifile, ierr)
776 ! push the restored state to the device here, not in s_read_amr_restart: the serial reader pushes per block (a store growth
777 ! while allocating the next slot would otherwise lose the host data), and a late whole-set push would copy back columns that
778 ! growth and compaction left undefined
779 do k = 1, amr_num_blocks
780 if (amr_owns_all(k)) then
781
782# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
783#if defined(MFC_OpenACC)
784# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
785!$acc update device(amr_cons_st(:, :, :, :, amr_loc_of(k)))
786# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
787#elif defined(MFC_OpenMP)
788# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
789!$omp target update to(amr_cons_st(:, :, :, :, amr_loc_of(k)))
790# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
791#endif
792# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_restart.fpp"
793 end if
794 end do
795#endif
796
797 end subroutine s_amr_restart_read_parallel
798
799 !> The file's block count must fit this run's metadata.
800 impure subroutine s_amr_restart_check_blocks(nblk)
801
802 integer, intent(in) :: nblk
803
804 if (nblk < 1 .or. nblk > amr_max_blocks) then
805 call s_mpi_abort('amr restart: the file holds more fine blocks than amr_max_blocks ' &
806 & // 'in this run; restart with amr_max_blocks at least the written block count')
807 end if
808
809 end subroutine s_amr_restart_check_blocks
810
811 !> Corrupt/foreign-file guard for block k's record (box outside the domain, level outside 1..amr_max_level, extent over the
812 !! block cap). Stores the accepted region and level, which the owner assignment and geometry build key off.
813 impure subroutine s_amr_restart_check_record(k, reg, lvl)
814
815 integer, intent(in) :: k, reg(6), lvl
816
817 if (reg(1) < 0 .or. reg(4) > m_glb .or. reg(1) > reg(4) .or. (n_glb > 0 .and. (reg(2) < 0 .or. reg(5) > n_glb .or. reg(2) &
818 & > reg(5))) .or. (p_glb > 0 .and. (reg(3) < 0 .or. reg(6) > p_glb .or. reg(3) > reg(6)))) then
819 call s_mpi_abort('amr restart: corrupt block record (box outside the global domain)')
820 end if
821 if (lvl < 1 .or. lvl > amr_max_level) then
822 call s_mpi_abort('amr restart: corrupt block record (block level outside 1..amr_max_level)')
823 end if
824 ! slot coordinate arrays are sized to max_f, which derives from the rank count unless amr_max_grid_size is pinned
825 if (any(merge((amr_ref_ratio**lvl)*(reg(4:6) - reg(1:3) + 1) - 1 > max_f, .false., amr_dim))) then
826 call s_mpi_abort('amr restart: a block exceeds this rank count''s block cap; restart on no more ranks ' &
827 & // 'than wrote the checkpoint, or set amr_max_grid_size in both runs')
828 end if
829 amr_region_lo_all(:,k) = reg(1:3); amr_region_hi_all(:,k) = reg(4:6)
830 amr_block_level(k) = lvl
831
832 end subroutine s_amr_restart_check_record
833
834end module m_amr_restart
real(wp), intent(inout) rm
integer, intent(in) k
integer, intent(in) l
Rank decomposition, box arithmetic and block ownership (SFC cut, owner lookups, owned-block lists).
impure subroutine, public s_amr_adopt_block_owners(own)
Install a recorded owner table instead of recomputing it (restart at the writer's rank count): the co...
impure subroutine, public s_amr_assign_block_owners()
Dynamic regrid for the AMR level set: density-gradient tagging, clustering (m_amr_cluster),...
impure subroutine, public s_amr_check_seam_topology()
Abort on same-level seam topologies no halo reconciles: partial-transverse face adjacency or box inte...
The AMR fine-level restart file layout shared by m_amr_restart (writer/reader) and post_process's ove...
character(len=path_len+3 *name_len) function, public f_amr_restart_path(t_step)
The fine-level restart file for time step t_step: one file per rank without parallel_io,...
integer, parameter, public amr_restart_ghdr_ints
global header length (ints)
impure subroutine, public s_amr_restart_check_header(ghdr, nvar, who, nblk, ref_ratio, max_level)
Validate the global header: the rank count (serial files need a match), nvar, the stored word size an...
impure subroutine, public s_amr_restart_read_catalog(ifile, nblk, nvar, cat)
Read every block's header and ownership record from an open parallel_io file (rank 0 reads,...
AMR fine-level restart I/O: writes/reads the fine-level restart file alongside the level-0 restart (s...
impure subroutine s_amr_restart_check_record(k, reg, lvl)
Corrupt/foreign-file guard for block k's record (box outside the domain, level outside 1....
impure subroutine s_amr_write_level_sums(t_step)
Write D/amr_levels.<t_step>.dat: per refined level, its fine cell count and the mean of |q| over its ...
impure subroutine, public s_read_amr_restart(restored)
Restore the fine level from the AMR restart file at t_step_start (n_start under cfl_dt),...
impure subroutine s_amr_restart_write_parallel(t_step)
Parallel: one shared MPI-IO file.
impure subroutine, public s_write_amr_restart(t_step)
Write the fine-level restart file for save step t_step alongside the level-0 restart: the writing ran...
impure subroutine s_amr_restart_read_parallel(file_loc)
Parallel (MPI-IO) read. The writer's rank count sets only the file layout: each block's data is one c...
impure subroutine s_amr_restart_read_serial(file_loc)
Serial (per-rank files) read: every block's region, level and, where this rank owned it at write (rm ...
impure subroutine s_amr_restart_check_blocks(nblk)
The file's block count must fit this run's metadata.
impure subroutine s_amr_restart_write_serial(t_step)
Serial: one unformatted file per rank in the level-0 step directory just created by the serial write.
impure subroutine s_amr_restart_install_blocks(own)
Install the restored block set: owners from the regions (or the recorded table), this run's owned slo...
Module-scope AMR state (block pool, flat field store, wave buffers) and the leaf helpers every AMR mo...
type(t_level), dimension(:), allocatable amr_slots
Fixed pool of refined-block slots; the working slot amr_cur selects which one per-block routines oper...
integer, dimension(:), allocatable amr_loc_of
Dense local index for live slots: amr_slots is sparse over 1:amr_max_blocks, the field store is dense...
integer, dimension(3) max_f
Per-slot sizing: max fine cells per dim and the buffered array bounds. amr_slot_live(k): slot k's fie...
logical amr_seam_pairs_dirty
real(stp), dimension(:,:,:,:,:), allocatable amr_cons_st
Flat per-block field store (x, y, z, var, local slot) so one kernel can run over every live block; al...
Block-structured AMR: up to amr_max_blocks refined blocks (2:1 or 4:1 per amr_ref_ratio),...
Compile-time constant parameters: default values, tolerances, and physical constants.
integer, parameter amr_restart_blk_own_ints
AMR restart per-block ownership record [owner + 1, m, n, p]; owner + 1 keeps rank 0 distinct from an ...
integer, parameter amr_restart_blk_hdr_ints
AMR fine-level restart per-block header: region box (6) + amr_block_level (1); the writer and both re...
Writes solution data, run-time stability diagnostics (ICFL, VCFL, CCFL, Rc), and probe/center-of-mass...
impure subroutine, public s_create_d_dir()
Create case_dir/D on rank 0 if it does not exist.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the computational domain, fluid properties, and simulation algorithm configurat...
integer, dimension(:,:), allocatable amr_region_lo_all
Per-slot mirrors (allocated 1:amr_max_blocks by m_amr): region box, rank intersection and ownership f...
integer proc_rank
Rank of the local processor.
integer(8) amr_mesh_epoch
Monotone mesh epoch, incremented wherever amr_seam_pairs_dirty is set and at the end of each slot rec...
integer, dimension(:,:), allocatable amr_region_hi_all
logical, dimension(:), allocatable amr_owns_all
integer, dimension(:), allocatable amr_block_level
Refinement level of each active block (1..amr_max_level) and the deepest level populated....
logical, dimension(3) amr_dim
The active dimensions, and this rank's coarse subdomain origin and extent padded to 3-vectors (0 in c...
integer num_procs
Number of processors.
integer amr_num_blocks
Number of active AMR fine-block slots (>= 1) and the working slot index selecting which one the per-b...
subroutine s_amr_select_slot(islot)
Make block slot islot the working slot: set amr_cur and copy its stored mirrors (region,...
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_allreduce_vectors_sum(var_loc, var_glb, num_vectors, vector_length)
Reduce an array of vectors to their global sums across all MPI ranks.
impure subroutine s_mpi_allreduce_integer_min(var_loc, var_glb)
Reduce a local integer value to its global minimum across all MPI ranks.
MPI halo exchange, domain decomposition, and buffer packing/unpacking for the simulation solver.
Every block's header, ownership record and data-chunk byte offset, read collectively by all ranks.