MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_amr_restart_io.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/common/m_amr_restart_io.fpp"
2!>
3!!@file
4!!@brief Contains module m_amr_restart_io
5
6# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
7# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
8# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
9# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
10# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
11# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14
15# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
16# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
17# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18
19# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
20# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
21! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
22! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
23# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
24# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
25# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
26
27# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
28# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
29
30# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
31
32# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
33
34# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
35
36# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
37
38# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
39
40# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
41
42# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
43
44# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
45! New line at end of file is required for FYPP
46# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
47# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
48# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
49# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
50# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
51# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54
55# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
56# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
57# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58
59# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
60# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
61! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
62! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
63# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
64# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
65# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
66
67# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
68# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
69
70# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75
76# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
77
78# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
79
80# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
81
82# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
83
84# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
85! New line at end of file is required for FYPP
86# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
87
88# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
89# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
90# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93
94# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129
130# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
131
132# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
133
134# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
135
136# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
137
138# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
139# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
140
141# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154
155# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
156
157# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
158
159# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
160
161# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
162
163# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
164! New line at end of file is required for FYPP
165# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
166# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
167# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
168# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
169# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
170# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173
174# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
176# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177
178# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
180! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
181! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
182# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
183# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
184# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
185
186# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
187# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
188
189# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
190
191# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
192
193# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
194
195# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
196
197# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
198
199# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
200
201# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
202
203# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
204! New line at end of file is required for FYPP
205# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
206
207# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
208
209# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
210
211# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
212
213# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
214
215# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
216
217# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
218
219# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
220
221# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
222
223# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
224
225# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
226
227# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
228
229# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
230
231# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
232
233# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
234
235# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
236
237# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
238
239# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
240
241# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
242
243# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
244
245# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
246
247# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
248
249# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
250
251# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
252
253# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
254
255# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
256
257# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
258
259# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
260
261# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
262! New line at end of file is required for FYPP
263# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
264
265! GPU parallel region (scalar reductions, maxval/minval)
266# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
267
268! GPU parallel loop over threads (most common GPU macro)
269# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
270
271! Required closing for GPU_PARALLEL_LOOP
272# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
273
274! Mark routine for device compilation
275# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
276
277! Declare device-resident data
278# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
279
280! Inner loop within a GPU parallel region
281# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
282
283! Scoped GPU data region
284# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
285
286! Host code with device pointers (for MPI with GPU buffers)
287# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
288
289! Allocate device memory (unscoped)
290# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
291
292! Free device memory
293# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
294
295! Atomic operation on device
296# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
297
298! End atomic capture block
299# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
300
301! Copy data between host and device
302# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
303
304! Synchronization barrier
305# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
306
307! Import GPU library module (openacc or omp_lib)
308# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
309
310! Emit code only for AMD compiler
311# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
312
313! Emit code for non-Cray compilers
314# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
315
316! Emit code only for Cray compiler
317# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
318
319! Cray compiler directive, emitted from Fypp so the formatter cannot turn it into a comment
320# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
321
322! Emit code for non-NVIDIA compilers
323# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
324
325# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
326# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
327! New line at end of file is required for FYPP
328# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
329
330# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
331
332! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
333! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
334! example see misc/nvidia_uvm/bind.sh.
335# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
336
337! Allocate and create GPU device memory
338# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
339
340! Free GPU device memory and deallocate
341# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
342
343! Cray-specific GPU pointer setup for vector fields
344# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
345
346! Cray-specific GPU pointer setup for scalar fields
347# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
348
349! Cray-specific GPU pointer teardown for scalar fields - exact inverse of ACC_SETUP_SFs. Removes the
350! present-table entries ACC_SETUP_SFs added (the scalar_field descriptor and its %sf copyin). REQUIRED when
351! an ACC_SETUP'd field is freed MID-RUN (e.g. AMR slot free on regrid/restart): Cray 'exit data delete'
352! decrements the reference counter, so the lone @:DEALLOCATE(arg%sf) only undoes the @:ALLOCATE create ref
353! and leaves the descriptor + the ACC_SETUP %sf copyin dangling. Call BEFORE @:DEALLOCATE(arg%sf).
354# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
355
356! Cray-specific GPU pointer setup for acoustic source spatials
357# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
358
359! Whether stress index idx is a shear component. Spelled out because NVHPC 26.9 at -O2 vectorizes any(idx == shear_indices)
360! inside a loop to always true, which doubled the normal-stress elastic energy.
361# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
362
363# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
364
365# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
366! New line at end of file is required for FYPP
367# 6 "/home/runner/work/MFC/MFC/src/common/m_amr_restart_io.fpp" 2
368
369!> @brief The AMR fine-level restart file layout shared by m_amr_restart (writer/reader) and post_process's overlay reader
370!! (m_data_input): path, fine cell-boundary subdivision, global-header check and the parallel_io per-block catalog. Callers read the
371!! data themselves.
372!!
373!! Layout (parallel_io): amr_restart_ghdr_ints-int header [-num_procs, nblk, nvar, amr_ref_ratio, amr_max_level, bytes per stored
374!! word] (the rank count is positive in the serial files), then per block amr_restart_blk_hdr_ints ints [region lo(3), hi(3), level], amr_restart_blk_own_ints ints [owner + 1, m, n, p] and the owner's
375!! data chunk nvar*(m+1)*(n+1)*(p+1) stp words (absent without an owner). The serial per-rank file
376!! (p_all/p<rank>/<step>/amr_fine.dat) holds the same records as unformatted sequential records, data only for owned blocks.
378
379#ifdef MFC_MPI
380 use mpi
381#endif
384 use m_mpi_common, only: proc_rank, num_procs, s_mpi_abort
386
387 implicit none
388
389 private
391
392 integer, parameter :: amr_restart_ghdr_ints = 6 !< global header length (ints)
393
394#ifdef MFC_MPI
396
397 !> Every block's header, ownership record and data-chunk byte offset, read collectively by all ranks.
399 integer :: nblk = 0
400 integer, allocatable :: reg(:,:) !< (6, nblk): region lo(3), hi(3) in global coarse cells
401 integer, allocatable :: lvl(:) !< (nblk): refinement level
402 integer, allocatable :: own(:,:) !< (amr_restart_blk_own_ints, nblk): owner + 1, m, n, p
403 integer(kind=MPI_OFFSET_KIND), allocatable :: data_disp(:) !< (nblk): byte offset of the block's data chunk
404 integer(kind=MPI_OFFSET_KIND) :: end_disp = 0 !< byte offset past the last chunk = the file's expected size
405 end type t_amr_restart_catalog
406#endif
407
408contains
409
410 !> The fine-level restart file for time step t_step: one file per rank without parallel_io, one shared file with it.
411 function f_amr_restart_path(t_step) result(file_loc)
412
413 integer, intent(in) :: t_step
414 character(LEN=path_len + 3*name_len) :: file_loc
415
416 if (.not. parallel_io) then
417 write (file_loc, '(A,I0,A,I0,A)') trim(case_dir) // '/p_all/p', proc_rank, '/', t_step, '/amr_fine.dat'
418 else
419 write (file_loc, '(A,I0,A)') 'amr_', t_step, '.dat'
420 file_loc = trim(case_dir) // '/restart_data' // trim(mpiiofs) // trim(file_loc)
421 end if
422
423 end function f_amr_restart_path
424
425 !> Fine cell boundaries fcb(-1:nfine) by rr-way subdivision of the coarse cells of pcb starting at coarse index lo; pcb_lb is
426 !! lbound(pcb) in the caller (an assumed-shape dummy resets it to 1). Coarse right edges are copied exactly, so a subdivision of
427 !! a subdivision reproduces the deeper level bit-for-bit.
428 pure subroutine s_amr_subdivide_cb(pcb, pcb_lb, lo, nfine, rr, fcb)
429
430 real(wp), intent(in) :: pcb(:)
431 integer, intent(in) :: pcb_lb, lo, nfine, rr
432 real(wp), intent(inout) :: fcb(-1:)
433 integer :: fi, c, off, k
434 real(wp) :: xl, xr
435
436 off = 1 - pcb_lb ! pcb(j) = coarse_cb(j + pcb_lb - 1); coarse_cb(c) = pcb(c + off)
437 fcb(-1) = pcb(lo - 1 + off)
438 do fi = 0, nfine
439 c = lo + fi/rr
440 xl = pcb(c - 1 + off); xr = pcb(c + off)
441 k = mod(fi, rr)
442 if (k == rr - 1) then
443 fcb(fi) = xr
444 else
445 fcb(fi) = (real(rr - 1 - k, wp)*xl + real(k + 1, wp)*xr)/real(rr, wp)
446 end if
447 end do
448
449 end subroutine s_amr_subdivide_cb
450
451 !> Validate the global header: the rank count (serial files need a match), nvar, the stored word size and, when given, the run's
452 !! amr_ref_ratio (equal) and amr_max_level (at least the file's). Returns the block count.
453 impure subroutine s_amr_restart_check_header(ghdr, nvar, who, nblk, ref_ratio, max_level)
454
455 integer, intent(in) :: ghdr(amr_restart_ghdr_ints), nvar
456 character(len=*), intent(in) :: who !< message prefix ('amr restart' / 'amr post')
457 integer, intent(out) :: nblk
458 integer, intent(in), optional :: ref_ratio, max_level
459 character(LEN=300) :: msg
460
461 if (.not. parallel_io .and. ghdr(1) /= num_procs) then
462 write (msg, '(A,I0,A,I0,A)') trim(who) // ': the AMR restart file was written with ', ghdr(1), &
463 & ' ranks but this run has ', num_procs, '; restart with the same rank count, or use parallel_io'
464 call s_mpi_abort(trim(msg))
465 end if
466 if (ghdr(3) /= nvar) then
467 write (msg, '(A,I0,A,I0,A)') trim(who) // ': the AMR restart file has ', ghdr(3), &
468 & ' conserved variables but this run has ', nvar, &
469 & '; the physics configuration ' // '(num_fluids/model_eqns/bubbles/chemistry) must match the run that wrote it'
470 call s_mpi_abort(trim(msg))
471 end if
472 if (ghdr(6) /= storage_size(0._stp)/8) call s_mpi_abort(trim(who) // ': the AMR restart file was written at a different ' &
473 & // 'storage precision (mixed_precision) than this run')
474 if (present(ref_ratio)) then
475 if (ghdr(4) /= ref_ratio) call s_mpi_abort(trim(who) // ': amr_ref_ratio differs from the run that wrote the file')
476 end if
477 if (present(max_level)) then
478 if (ghdr(5) > max_level) call s_mpi_abort(trim(who) // ': the file holds blocks deeper than this run''s amr_max_level')
479 end if
480 nblk = ghdr(2)
481
482 end subroutine s_amr_restart_check_header
483
484#ifdef MFC_MPI
485 !> Read every block's header and ownership record from an open parallel_io file (rank 0 reads, all ranks receive) and lay out
486 !! the data offsets.
487 impure subroutine s_amr_restart_read_catalog(ifile, nblk, nvar, cat)
488
489 integer, intent(in) :: ifile, nblk, nvar
490 type(t_amr_restart_catalog), intent(out) :: cat
491 integer, parameter :: rec = amr_restart_blk_hdr_ints + amr_restart_blk_own_ints
492 integer :: k, ierr, ibytes, sbytes, brec(rec)
493 integer, dimension(MPI_STATUS_SIZE) :: status
494 integer(kind=MPI_OFFSET_KIND) :: disp, chunk
495
496 ibytes = storage_size(0)/8; sbytes = storage_size(0._stp)/8
497 cat%nblk = nblk
498 allocate (cat%reg(6, nblk), cat%lvl(nblk), cat%own(amr_restart_blk_own_ints, nblk), cat%data_disp(nblk))
499 if (proc_rank == 0) then
500 disp = int(amr_restart_ghdr_ints*ibytes, mpi_offset_kind)
501 do k = 1, nblk
502 call mpi_file_read_at(ifile, disp, brec, rec, mpi_integer, status, ierr)
503 cat%reg(:,k) = brec(1:6); cat%lvl(k) = brec(amr_restart_blk_hdr_ints)
504 cat%own(:,k) = brec(amr_restart_blk_hdr_ints + 1:rec)
505 cat%data_disp(k) = disp + int(rec*ibytes, mpi_offset_kind)
506 ! a block with no owner contributed no data at all
507 chunk = int(0, mpi_offset_kind)
508 if (cat%own(1, k) > 0) chunk = int(nvar, mpi_offset_kind)*int(cat%own(2, k) + 1, mpi_offset_kind)*int(cat%own(3, &
509 & k) + 1, mpi_offset_kind)*int(cat%own(4, k) + 1, mpi_offset_kind)
510 disp = cat%data_disp(k) + chunk*int(sbytes, mpi_offset_kind)
511 end do
512 cat%end_disp = disp
513 end if
514 call mpi_bcast(cat%reg, 6*nblk, mpi_integer, 0, mpi_comm_world, ierr)
515 call mpi_bcast(cat%lvl, nblk, mpi_integer, 0, mpi_comm_world, ierr)
516 call mpi_bcast(cat%own, amr_restart_blk_own_ints*nblk, mpi_integer, 0, mpi_comm_world, ierr)
517 call mpi_bcast(cat%data_disp, nblk, mpi_offset, 0, mpi_comm_world, ierr)
518 call mpi_bcast(cat%end_disp, 1, mpi_offset, 0, mpi_comm_world, ierr)
519
520 end subroutine s_amr_restart_read_catalog
521#endif
522end module m_amr_restart_io
integer, intent(in) k
The AMR fine-level restart file layout shared by m_amr_restart (writer/reader) and post_process's ove...
pure subroutine, public s_amr_subdivide_cb(pcb, pcb_lb, lo, nfine, rr, fcb)
Fine cell boundaries fcb(-1:nfine) by rr-way subdivision of the coarse cells of pcb starting at coars...
character(len=path_len+3 *name_len) function, public f_amr_restart_path(t_step)
The fine-level restart file for time step t_step: one file per rank without parallel_io,...
integer, parameter, public amr_restart_ghdr_ints
global header length (ints)
impure subroutine, public s_amr_restart_check_header(ghdr, nvar, who, nblk, ref_ratio, max_level)
Validate the global header: the rank count (serial files need a match), nvar, the stored word size an...
impure subroutine, public s_amr_restart_read_catalog(ifile, nblk, nvar, cat)
Read every block's header and ownership record from an open parallel_io file (rank 0 reads,...
Compile-time constant parameters: default values, tolerances, and physical constants.
integer, parameter amr_restart_blk_own_ints
AMR restart per-block ownership record [owner + 1, m, n, p]; owner + 1 keeps rank 0 distinct from an ...
integer, parameter name_len
Maximum name length.
integer, parameter amr_restart_blk_hdr_ints
AMR fine-level restart per-block header: region box (6) + amr_block_level (1); the writer and both re...
integer, parameter path_len
Maximum path length.
Shared global parameters and equation-index setup for all three executables. Each per-target m_global...
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
Working-precision kind selection (half/single/double) and corresponding MPI datatype parameters.
integer, parameter wp
Change to single_precision if needed.
Every block's header, ownership record and data-chunk byte offset, read collectively by all ranks.