MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_amr_state.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
2!>
3!!@file
4!!@brief Contains module m_amr_state
5
6# 7 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
7# 8 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
8# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
9# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
10# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
11# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
15# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
16
17# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
19# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
20
21# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
22# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
23! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
24! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
25# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
26# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
27# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
28
29# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
30# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
31
32# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
33
34# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
35
36# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
37
38# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
39
40# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
41
42# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
43
44# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
45
46# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
47! New line at end of file is required for FYPP
48# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
49# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
50# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
51# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
55# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
56
57# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
59# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
60
61# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
62# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
63! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
64! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
65# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
66# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
67# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
68
69# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
70# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75
76# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
77
78# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
79
80# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
81
82# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
83
84# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
85
86# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
87! New line at end of file is required for FYPP
88# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
89
90# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
94# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129
130# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
131
132# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
133
134# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
135
136# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
137
138# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
139
140# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
141# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154
155# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
156
157# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
158
159# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
160
161# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
162
163# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
164
165# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
166! New line at end of file is required for FYPP
167# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
168# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
169# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
170# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
174# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175
176# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
178# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179
180# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
181# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
182! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
183! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
184# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
185# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
186# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
187
188# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
189# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
190
191# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
192
193# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
194
195# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
196
197# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
198
199# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
200
201# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
202
203# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
204
205# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
206! New line at end of file is required for FYPP
207# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
208
209# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
210
211# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
212
213# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
214
215# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
216
217# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
218
219# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
220
221# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
222
223# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
224
225# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
226
227# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
228
229# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
230
231# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
232
233# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
234
235# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
236
237# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
238
239# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
240
241# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
242
243# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
244
245# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
246
247# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
248
249# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
250
251# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
252
253# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
254
255# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
256
257# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
258
259# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
260
261# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
262
263# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
264! New line at end of file is required for FYPP
265# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
266
267! GPU parallel region (scalar reductions, maxval/minval)
268# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
269
270! GPU parallel loop over threads (most common GPU macro)
271# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
272
273! Required closing for GPU_PARALLEL_LOOP
274# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
275
276! Mark routine for device compilation
277# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
278
279! Declare device-resident data
280# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
281
282! Inner loop within a GPU parallel region
283# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
284
285! Scoped GPU data region
286# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
287
288! Host code with device pointers (for MPI with GPU buffers)
289# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
290
291! Allocate device memory (unscoped)
292# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
293
294! Free device memory
295# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
296
297! Atomic operation on device
298# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
299
300! End atomic capture block
301# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
302
303! Copy data between host and device
304# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
305
306! Synchronization barrier
307# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
308
309! Import GPU library module (openacc or omp_lib)
310# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
311
312! Emit code only for AMD compiler
313# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
314
315! Emit code for non-Cray compilers
316# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
317
318! Emit code only for Cray compiler
319# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
320
321! Cray compiler directive, emitted from Fypp so the formatter cannot turn it into a comment
322# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
323
324! Emit code for non-NVIDIA compilers
325# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
326
327# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
328# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
329! New line at end of file is required for FYPP
330# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
331
332# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
333
334! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
335! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
336! example see misc/nvidia_uvm/bind.sh.
337# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
338
339! Allocate and create GPU device memory
340# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
341
342! Free GPU device memory and deallocate
343# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
344
345! Cray-specific GPU pointer setup for vector fields
346# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
347
348! Cray-specific GPU pointer setup for scalar fields
349# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
350
351! Cray-specific GPU pointer teardown for scalar fields - exact inverse of ACC_SETUP_SFs. Removes the
352! present-table entries ACC_SETUP_SFs added (the scalar_field descriptor and its %sf copyin). REQUIRED when
353! an ACC_SETUP'd field is freed MID-RUN (e.g. AMR slot free on regrid/restart): Cray 'exit data delete'
354! decrements the reference counter, so the lone @:DEALLOCATE(arg%sf) only undoes the @:ALLOCATE create ref
355! and leaves the descriptor + the ACC_SETUP %sf copyin dangling. Call BEFORE @:DEALLOCATE(arg%sf).
356# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
357
358! Cray-specific GPU pointer setup for acoustic source spatials
359# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
360
361! Whether stress index idx is a shear component. Spelled out because NVHPC 26.9 at -O2 vectorizes any(idx == shear_indices)
362! inside a loop to always true, which doubled the normal-stress elastic energy.
363# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
364
365# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
366
367# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
368! New line at end of file is required for FYPP
369# 9 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp" 2
370
371!> @brief Module-scope AMR state (block pool, flat field store, wave buffers) and the leaf helpers every AMR module needs.
373
374 use m_derived_types ! scalar_field, t_box, int_bounds_info
377 use m_constants, only: mapcells
378 use m_mpi_proxy, only: s_mpi_abort
382
383 implicit none
384
385 !> One refined level: its own grid and metadata (host-only). Fields live in the flat stores amr_cons_st/amr_stor_st, indexed by
386 !! amr_loc_of.
388 integer :: amr_ref_ratio
389 type(t_box) :: region !< block extent in parent (level-0) cell indices
390 integer :: m, n, p !< this level's interior extents
391 integer :: buff_size
393 real(wp), allocatable :: x_cb(:), x_cc(:), dx(:)
394 real(wp), allocatable :: y_cb(:), y_cc(:), dy(:)
395 real(wp), allocatable :: z_cb(:), z_cc(:), dz(:)
396 end type t_level
397
398 !> Fixed pool of refined-block slots; the working slot amr_cur selects which one per-block routines operate on.
399 type(t_level), allocatable :: amr_slots(:)
400
401 !> Dense local index for live slots: amr_slots is sparse over 1:amr_max_blocks, the field store is dense. Freed indices are
402 !! recycled through amr_loc_free.
403 integer, allocatable :: amr_loc_of(:) !< global slot -> dense local index, 0 if not live
404 integer, allocatable :: amr_loc_free(:) !< stack of recycled local indices
405 integer :: amr_loc_n = 0 !< high-water mark of local indices handed out
406 integer :: amr_loc_nfree = 0 !< depth of the recycle stack
407
408 !> Flat per-block field store (x, y, z, var, local slot) so one kernel can run over every live block; all slots share the mbuf
409 !! extents. Sized by s_amr_st_reserve.
410 real(stp), allocatable, dimension(:,:,:,:,:) :: amr_cons_st, amr_stor_st
411
412# 50 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
413#if defined(MFC_OpenACC)
414# 50 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
415!$acc declare create(amr_cons_st, amr_stor_st)
416# 50 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
417#elif defined(MFC_OpenMP)
418# 50 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
419!$omp declare target (amr_cons_st, amr_stor_st)
420# 50 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
421#endif
422# 51 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
423 !> Moving bodies: the primitive state of the RHS pass of each block whose IB correction waits for the bodies to move
424 !! (s_amr_fine_stage_ib), one column per such block, in the store's (x, y, z, var) layout.
425 real(stp), allocatable, dimension(:,:,:,:,:) :: amr_ibp
426
427# 54 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
428#if defined(MFC_OpenACC)
429# 54 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
430!$acc declare create(amr_ibp)
431# 54 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
432#elif defined(MFC_OpenMP)
433# 54 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
434!$omp declare target (amr_ibp)
435# 54 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
436#endif
437# 55 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
438 integer, allocatable :: amr_ibp_blk(:) !< the block of each amr_ibp column
439 integer :: amr_ibp_n = 0 !< columns in use this stage
440 !> local slots the store is sized for; grows only (s_amr_compact_store keeps the index space dense)
441 integer :: amr_st_cap = 0
442
443 !> Copy bridge to the shared solver, whose routines take type(scalar_field) and serve the monolithic path too (a pointer view
444 !! into the flat store is not attachable on OpenMP offload): load from the store, call, store back, in both directions since the
445 !! dummies are intent(inout). Not GPU_DECLARE'd: a declare binds the present-table entry to the descriptor that move_alloc then
446 !! swaps out (CCE find_in_present_table failed); allocate via move_alloc + GPU_ENTER_DATA instead (CCE lib-4425 otherwise).
447 type(scalar_field), allocatable :: amr_cons_br(:)
448 !> Batched advance: the bridge spans amr_br_batch blocks along the last active dimension, member i at offset (i-1)*amr_bat_w
449 !! with its own ghost shell, so no block's stencil reaches another's interior. amr_bat_loc = members' flat-store columns
450 !! (device-resident).
451 integer :: amr_bat_loc(amr_bat_max) = 0
452
453# 69 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
454#if defined(MFC_OpenACC)
455# 69 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
456!$acc declare create(amr_bat_loc)
457# 69 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
458#elif defined(MFC_OpenMP)
459# 69 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
460!$omp declare target (amr_bat_loc)
461# 69 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
462#endif
463# 70 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
464 !> Pooled advance scratch shared by every fine block (the fused advance leaves no cross-block lifetime); same pattern as amr_cg.
465 type(scalar_field), allocatable :: amr_scr_prim(:), amr_scr_rhs(:)
466 !> block-frame prim scratch for the batched per-member IB correction (ib only); the slab prim stacks members along amr_bat_sd
467 type(scalar_field), allocatable :: amr_scr_prim_blk(:)
468 !> Blocks per batched s_compute_rhs call (amr_bat_max once initialized); bounded to avoid exhausting device memory.
469 integer :: amr_br_batch = 1
470 integer :: amr_maxc(3) !< max coarse block cells per dim: (m_glb+1)/2 etc.; 1 for collapsed dims
471
472 !> Per-slot sizing: max fine cells per dim and the buffered array bounds. amr_slot_live(k): slot k's field arrays are allocated
473 !! (lazy, owned-only).
474 integer :: max_f(3), mbuf_lo(3), mbuf_hi(3)
475 logical, allocatable :: amr_slot_live(:)
476 !> cached same-level seam list (xb, yb, seam-dim) so s_amr_fine_fine_post iterates O(#seams); rebuilt when amr_seam_pairs_dirty
477 !! is set or the block count changes
478 integer, allocatable :: amr_seam_pairs(:,:)
481 ! amr_mesh_epoch lives in m_global_parameters (m_amr_registers keys on it and cannot use m_amr)
482 !> Tag base of the clusterer's narrow-node reductions, above the per-box tag space [1..amr_max_blocks]; checked against
483 !! MPI_TAG_UB at init.
484 integer :: amr_tag_narrow = 0
485 !> Keyed wave tags: amr_tag_base + band*65536 + gen*4096 + seq, checked against MPI_TAG_UB. gen (mod 16) bumps per wave entry on
486 !! every rank; seq is the message's index in the pair's canonically ordered transfer list (block id, dim, lo before hi), so
487 !! matching is independent of posting order.
488 integer, parameter :: amr_band_reflux = 0, amr_band_freg = 1, amr_band_parent_fill = 2, amr_band_stage_fill = 3, &
490 integer, parameter :: amr_n_bands = 8
491 integer :: amr_tag_base = 0
492 integer :: amr_tag_gen(0:amr_n_bands - 1) = 0
493 integer, allocatable :: amr_tsq(:,:) !< (0:np-1, dir) in-wave per-peer seq counters; touched-reset
494 integer, allocatable :: amr_tsq_tch(:,:) !< touched peers per dir, so the reset is O(active peers), not O(P)
495 integer :: amr_n_tsq(2) = 0
496 !> cached per-block P2P overlap-rank lists, rebuilt with the seam list: amr_ovl_gather(:,k) = ranks whose owned coarse range
497 !! meets block k's padded patch box; amr_ovl_scatter(:,k) = ranks whose coarse interior meets k's region box. Rank-ascending and
498 !! not owner-excluded, so iterating gives the same MPI order as a 0..num_procs-1 scan.
499 !> (max-overlap, amr_max_blocks); sized in s_amr_build_seam_pairs
500 integer, allocatable :: amr_ovl_gather(:,:), amr_ovl_scatter(:,:)
501 !> Block indices this rank owns, ascending (the paired MPI_SENDRECVs depend on the order); built in s_amr_assign_block_owners.
502 integer, allocatable :: amr_my_blk(:)
503 integer :: amr_n_my = 0
504 !> Level-1 blocks this rank receives restriction for (not its own, overlapping its interior), rebuilt once per mesh epoch. Keyed
505 !! on amr_mesh_epoch, not amr_myblk_dirty (owner writes only). Ownership here is amr_block_owner(k) == proc_rank, not the
506 !! multi-owner amr_rank_owns_block.
507 integer, allocatable :: amr_l1r_blk(:)
508 integer, allocatable :: amr_l1p_blk(:) !< padded variant (region +/- amr_cpat_mar vs my coarse range): stage-fill gather
509 integer :: amr_n_l1p = 0
510 integer :: amr_n_l1r = 0
511 integer, allocatable :: amr_fch_blk(:) !< foreign children of my owned parents (level >= 2, powner == me, cowner /= me)
512 integer :: amr_n_fch = 0
513 integer, allocatable :: amr_own_blk(:) !< blocks this rank intersects (amr_owns_all, the multi-owner notion, all levels)
514 integer :: amr_n_own = 0
515 !> cached f_amr_parent_block (0 for level <= 1), avoiding an O(global blocks) scan per block per stage
516 integer, allocatable :: amr_parent_blk(:)
517 integer, allocatable :: amr_child_ptr(:) !< children of p = amr_child_idx(amr_child_ptr(p-1)+1 : amr_child_ptr(p)), ascending
518 integer, allocatable :: amr_child_idx(:) !< (amr_child_ptr(0) = 0)
519 integer(8) :: amr_l1r_epoch = -1_8
520 integer :: amr_l1r_nblk = -1 !< second key half, matching s_amr_reg_prepare's (epoch, num_blocks) pair
521 !> Set wherever amr_block_owner is written (assigner, inheritance, restart/migration), so lists built from it never go stale.
522 logical :: amr_myblk_dirty = .true.
523 integer, allocatable :: amr_ovl_gather_n(:), amr_ovl_scatter_n(:) !< per-block list lengths
524 !> Wire pools of the pooled waves (m_amr_wave lays them out; contents never survive a wave). The fill and restrict waves share
525 !! amr_fw_sq/rq; the seam wave is posted before the fills and drained after, so it has its own pair. The zero-copy reflux waves
526 !! list their blocks in amr_fw_rblk.
527 real(wp), allocatable :: amr_fw_sq(:), amr_fw_rq(:), amr_sw_sq(:), amr_sw_rq(:)
528 real(wp), allocatable :: amr_rf_sq(:), amr_rf_rq(:) !< device-resident pools of the packed reflux-faces wave
529 integer, allocatable :: amr_fw_rblk(:)
530 !> Device-resident wire pools: with rdma_mpi MPI sends/receives them by device address, and the step-path kernels find them
531 !! present, so slices do not cross PCIe per wave.
532 logical :: amr_fw_dev = .false.
533 !> seam pairs both of whose blocks this rank owns: exchanged by one batched kernel at drain, no wire
534 integer :: amr_sw_nsame = 0
535 integer, allocatable :: amr_sw_plx(:), amr_sw_ply(:), amr_sw_pd(:), amr_sw_pxhi(:), amr_sw_pfm(:,:)
536 !> Fused exchange packs (amr_device_pack): one row per wave transfer (slab corner (1:3), extents (4:6), payload offset in the
537 !! wire pool (7); the child pack also has source store slot (8) and the child's patch frame (9:11)) plus the exclusive element
538 !! prefix, so one kernel walks a family's transfers. Rebuilt per wave.
539 integer, allocatable :: amr_fx_pl(:,:), amr_fx_pre(:)
540 !> (0:num_procs-1, 1:amr_max_level) SFC Morton-key upper bound per rank, one independent cost-weighted cut per level (owner =
541 !! cut search, f_amr_owner); O(P) storage. Per level because same-level boxes have distinct Morton keys, which the binary search
542 !! needs.
543 integer(kind=8), allocatable :: amr_fine_cut(:,:)
544
545 !> Regrid box size cap per dim (fixed for the run, identical on all ranks; 1 in collapsed dims): a box of at most min-over-ranks
546 !! of (local extent + 1)/2 cells intersects every rank in at most (its extent + 1)/2 cells, so the per-rank scratch constraint
547 !! 2*(isect cells) - 1 <= local extent holds by construction. Equals amr_maxc at np=1.
548 integer :: amr_maxc_fit(3) = 1
549
550 !> Swap contract (s_amr_swap_to_fine / s_amr_restore_coarse): the fine advance swaps these coarse globals to the block's values
551 !! and restores after: m/n/p, idwint/idwbuff, the nine coordinate arrays (sw_x_cb..sw_dz), acoustic_source; WENO/hypoelastic/IGR
552 !! spacing coefficients are recomputed instead. Any new module-level state derived from m/n/p/idwint/idwbuff/coords that a
553 !! kernel reads on the fine grid must be swapped here or refreshed per fine call, including its device copy (a stale device copy
554 !! reads out of range under CCE OpenACC; cv_int is refreshed by a per-call GPU_UPDATE in s_compute_rhs). amr_swap_depth makes
555 !! the swap re-entrant and guards an unpaired restore.
556 !> Saved coarse-level global state for swap/restore
557 integer :: sw_m, sw_n, sw_p
560
561 !> IGR sigma-state bounce: the fine solve reuses jac/jac_old at fine indices, so the coarse contents (jac_old is the Jacobi warm
562 !! start) are saved here across the fine advance.
563 real(wp), allocatable :: sw_jac(:,:,:), sw_jac_old(:,:,:)
564
565# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
566#if defined(MFC_OpenACC)
567# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
568!$acc declare create(sw_jac, sw_jac_old)
569# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
570#elif defined(MFC_OpenMP)
571# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
572!$omp declare target (sw_jac, sw_jac_old)
573# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
574#endif
575# 171 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
576
577 !> Swap nesting depth: only the outermost swap saves the coarse state and only its restore puts it back; inner swaps re-install,
578 !! inner restores are no-ops.
579 integer :: amr_swap_depth = 0
580 !> Global coarse cell-boundary arrays (-1:X_glb), assembled once at init; the fine-distribution owner rebuilds whole-block fine
581 !! coordinates from them.
582 real(wp), allocatable :: amr_gxcb(:), amr_gycb(:), amr_gzcb(:)
583 real(wp), allocatable :: sw_x_cb(:), sw_x_cc(:), sw_dx(:)
584 real(wp), allocatable :: sw_y_cb(:), sw_y_cc(:), sw_dy(:)
585 real(wp), allocatable :: sw_z_cb(:), sw_z_cc(:), sw_dz(:)
586
587 !> True (identically on all ranks) iff some rank's fine ghost fill reads its coarse ghost cells: the solver populates only prim
588 !! ghosts, so the cons ghosts must be halo-exchanged first. Never true at np=1.
589 logical :: amr_xchg_coarse_ghosts = .false.
590 !> local accumulator behind amr_xchg_coarse_ghosts; s_amr_reduce_xchg_flag does one allreduce per scan
591 integer :: amr_xchg_bad = 0
592
593 !> Per-block gathered coarse patch: the owner may not hold the coarse cells its block refines, so before each prolongation/ghost
594 !! fill the patch region_lo-amr_cpat_mar : region_hi+amr_cpat_mar is gathered point-to-point into amr_cg (stp scalar_fields,
595 !! block-local frame: cell 0 = global coarse cell amr_cpat_off(d)). Messages carry wp cast to stp, so at np=1 the patch equals
596 !! the local coarse read bitwise. Sized to the largest block. amr_slab_tab: device-resident slab table for the shell/ghost
597 !! kernels, rows lo(1:3),hi(1:3),soff,scnt over <= 6 slabs, refreshed by one GPU_UPDATE per launch.
598 integer, allocatable :: amr_slab_tab(:,:)
599 integer :: amr_slab_n = 0 !< rows queued in amr_slab_tab (s_amr_slab_add)
600 integer :: amr_cg_np = 1 !< patches amr_cg holds, stacked along dim 3 (s_amr_cg_reserve)
601 integer :: amr_cg_nuse = 0 !< patches the level-1 pool gather filled (s_amr_l1_pool_gather)
602 integer, allocatable :: amr_cg_pz(:) !< dim-3 offset of each owned level-1 box's patch in the pool
603 type(scalar_field), allocatable :: amr_cg(:)
604 integer :: amr_cpat_mar = 0 !< coarse-cell stencil reach (f_amr_cpat_mar)
605 integer :: amr_cpat_hi(3) = 0 !< amr_cg upper local bounds per dim (0 in collapsed dims)
606 integer :: amr_cpat_off(3) = 0 !< global coarse index of amr_cg local cell 0 (region_lo - amr_cpat_mar)
607 integer :: amr_wcur = 1 !< consume cursor into the open fill wave's receive transfers (box-major)
608
609 !> Subcycling patch cache: the pooled level-1 patches (amr_cg, amr_cg_nuse patches) at the step start (time level 0) then the
610 !! step end (1), each flat with g1 fastest, then g2, g3, variable.
611 real(stp), allocatable :: amr_pc(:)
612
613contains
614
615 !> Fill fcb/fcc/fdx by subdividing parent cells (s_amr_subdivide_cb); pcb_lb is lbound(pcb, 1). Only 0..nfine is filled.
616 subroutine s_build_level_coords(pcb, pcb_lb, lo, nfine, fcb, fcc, fdx)
617
618 real(wp), intent(in) :: pcb(:)
619 integer, intent(in) :: pcb_lb, lo, nfine
620 real(wp), allocatable, intent(inout) :: fcb(:), fcc(:), fdx(:)
621 integer :: fi
622
623 call s_amr_subdivide_cb(pcb, pcb_lb, lo, nfine, amr_slots(amr_cur)%amr_ref_ratio, fcb)
624 do fi = 0, nfine
625 fdx(fi) = fcb(fi) - fcb(fi - 1)
626 fcc(fi) = 0.5_wp*(fcb(fi - 1) + fcb(fi))
627 end do
628
629 end subroutine s_build_level_coords
630
631 !> Block k's region box in global coarse cells; collapsed dimensions read 0:0.
632 pure subroutine s_amr_region_box(k, rlo, rhi)
633
634 integer, intent(in) :: k
635 integer, intent(out) :: rlo(3), rhi(3)
636
637 rlo = merge(amr_region_lo_all(:,k), 0, amr_dim); rhi = merge(amr_region_hi_all(:,k), 0, amr_dim)
638
639 end subroutine s_amr_region_box
640
641 !> Block k's padded coarse patch box [plo, phi] (region +/- amr_cpat_mar; collapsed dims 0:0).
642 pure subroutine s_amr_patch_box(k, plo, phi)
643
644 integer, intent(in) :: k
645 integer, intent(out) :: plo(3), phi(3)
646
647 plo = merge(amr_region_lo_all(:,k) - amr_cpat_mar, 0, amr_dim); phi = merge(amr_region_hi_all(:,k) + amr_cpat_mar, 0, &
648 & amr_dim)
649
650 end subroutine s_amr_patch_box
651
652 !> How many parent cells inside a child's footprint the ghost prolongation reads: 1 for limited linear, 2 for the quartic.
653 pure integer function f_amr_prolong_reach()
654
655 f_amr_prolong_reach = merge(2, 1, amr_prolong == 2)
656
657 end function f_amr_prolong_reach
658
659 !> Coarse-patch margin amr_cpat_mar: the coarse cells a buff_size fine ghost layer spans plus the prolongation reach.
660 pure integer function f_amr_cpat_mar()
661
662 f_amr_cpat_mar = (buff_size + amr_ref_ratio - 1)/amr_ref_ratio + f_amr_prolong_reach()
663
664 end function f_amr_cpat_mar
665
666 !> Core [clo, chi] of the patch box that the ghost fill never reads, so the fill waves ship only the shell around it.
667 pure subroutine s_amr_patch_core(k, clo, chi)
668
669 integer, intent(in) :: k
670 integer, intent(out) :: clo(3), chi(3)
671
672 clo = merge(amr_region_lo_all(:,k) + f_amr_prolong_reach(), 0, amr_dim)
673 chi = merge(amr_region_hi_all(:,k) - f_amr_prolong_reach(), 0, amr_dim)
674
675 end subroutine s_amr_patch_core
676
677 !> Fine cell coordinates of block k in dimension d, rebuilt from the global coarse boundaries gcb by replaying k's ancestor
678 !! chain from replicated metadata, so any rank can reproduce them. Do not bisect amr_slots(parent)%x_cb instead: the parent's
679 !! slot exists only on its owner, which a level>=2 block's owner need not be.
680 impure subroutine s_amr_build_block_coords(k, gcb, glb, fcb, fcc, fdx, d)
681
682 integer, intent(in) :: k, d
683 real(wp), intent(in) :: gcb(:) !< global L0 cell boundaries
684 integer, intent(in) :: glb !< index of gcb's first element (-1, or -1 - buff_size for a padded build)
685 real(wp), allocatable, intent(inout) :: fcb(:), fcc(:), fdx(:)
686 integer :: chain(0:amr_max_level), lev, j, a, lo, nf, span, rr, plo(3), phi(3), clb
687 real(wp), allocatable :: cur(:), scb(:), scc(:), sdx(:)
688
689 rr = amr_ref_ratio
690 lev = amr_block_level(k)
691 a = k
692 do j = lev, 1, -1 ! chain(j) = k's ancestor at level j; chain(lev) = k
693 chain(j) = a
694 if (j > 1) a = f_amr_parent_block(a)
695 end do
696
697 ! level 0: the allocatable assignment takes the dummy's lbound (1), so cur's true first index travels in clb; levels below
698 ! are built into (-1:nf) arrays
699 cur = gcb; clb = glb
700 do j = 1, lev
701 a = chain(j)
702 span = amr_region_hi_all(d, a) - amr_region_lo_all(d, a) + 1 ! L0 cells the box covers
703 nf = rr**j*span - 1 ! its fine extent at level j
704 if (j == 1) then
705 lo = amr_region_lo_all(d, a) ! global L0 index of the box's low corner
706 else
707 call s_amr_parent_foot(a, chain(j - 1), plo, phi) ! low corner in the parent's fine frame
708 lo = plo(d)
709 end if
710 if (j == lev) then
711 call s_build_level_coords(cur, clb, lo, nf, fcb, fcc, fdx)
712 else
713 if (allocated(scb)) deallocate (scb, scc, sdx)
714 allocate (scb(-1:nf), scc(0:nf), sdx(0:nf))
715 call s_build_level_coords(cur, clb, lo, nf, scb, scc, sdx)
716 cur = scb; clb = -1
717 end if
718 end do
719 if (allocated(scb)) deallocate (scb, scc, sdx)
720
721 end subroutine s_amr_build_block_coords
722
723 !> Assemble the global coarse cell-boundary arrays over (-1-pad : G+pad): each rank writes its own cells' boundaries into a
724 !! sentinel-filled array and an elementwise MAX allreduce recovers the global one. A larger-pad rebuild gives a consistent
725 !! superset.
726 impure subroutine s_amr_build_global_cb(pad)
727
728 integer, intent(in) :: pad
729 integer :: j
730 real(wp), parameter :: sentinel = -huge(1._wp)
731
732# 328 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
733 if (amr_dim(1)) then
734# 329 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
735 if (allocated(amr_gxcb)) deallocate (amr_gxcb)
736# 330 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
737 allocate (amr_gxcb(-1 - pad:m_glb + pad)); amr_gxcb = sentinel
738# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
739 do j = -1 - pad, m + pad
740# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
741 amr_gxcb(start_idx(1) + j) = x_cb(j)
742# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
743 end do
744 call s_mpi_allreduce_array_max(amr_gxcb, m_glb + 2 + 2*pad)
745# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
746 end if
747# 328 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
748 if (amr_dim(2)) then
749# 329 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
750 if (allocated(amr_gycb)) deallocate (amr_gycb)
751# 330 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
752 allocate (amr_gycb(-1 - pad:n_glb + pad)); amr_gycb = sentinel
753# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
754 do j = -1 - pad, n + pad
755# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
756 amr_gycb(start_idx(2) + j) = y_cb(j)
757# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
758 end do
759 call s_mpi_allreduce_array_max(amr_gycb, n_glb + 2 + 2*pad)
760# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
761 end if
762# 328 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
763 if (amr_dim(3)) then
764# 329 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
765 if (allocated(amr_gzcb)) deallocate (amr_gzcb)
766# 330 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
767 allocate (amr_gzcb(-1 - pad:p_glb + pad)); amr_gzcb = sentinel
768# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
769 do j = -1 - pad, p + pad
770# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
771 amr_gzcb(start_idx(3) + j) = z_cb(j)
772# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
773 end do
774 call s_mpi_allreduce_array_max(amr_gzcb, p_glb + 2 + 2*pad)
775# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
776 end if
777# 337 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
778
779 end subroutine s_amr_build_global_cb
780
781 !> Do two coarse-index boxes [alo:ahi] and [blo:bhi] overlap? Collapsed dims never disqualify.
782 pure logical function f_amr_boxes_overlap(alo, ahi, blo, bhi) result(ov)
783
784 integer, intent(in) :: alo(3), ahi(3), blo(3), bhi(3)
785
786 ov = all((alo <= bhi .and. ahi >= blo) .or. .not. amr_dim)
787
788 end function f_amr_boxes_overlap
789
790 !> Index of the level-(level(k)-1) block that block k refines, or 0 for level 1 (parent is the L0 grid). Regions are in L0 cell
791 !! indices at every level; proper nesting guarantees one overlapping parent.
792 pure integer function f_amr_parent_block(k) result(p)
793
794 integer, intent(in) :: k
795 integer :: j
796
797 p = 0
798 if (amr_block_level(k) <= 1) return
799 do j = 1, amr_num_blocks
802 p = j
803 return
804 end if
805 end do
806
807 end function f_amr_parent_block
808
809 !> Pack (dir=+1) / unpack (dir=-1) the fine cells of slot's q_cons over [dlo:dhi] in dim d (interior transverse, all sys_size)
810 !! in a fixed order, so a packer and unpacker with matching extents align cell-for-cell. On GPU only this near-seam slab moves
811 !! device<->host.
812 impure subroutine s_amr_fine_slice(slot, d, dlo, dhi, buf, dir)
813
814 integer, intent(in) :: slot, d, dlo, dhi, dir
815 real(wp), intent(inout), contiguous :: buf(:)
816 integer :: i, a, b, c, fm(3), na, nb, nc, loc
817
818 fm(1) = amr_slots(slot)%m; fm(2) = amr_slots(slot)%n; fm(3) = amr_slots(slot)%p
819 loc = amr_loc_of(slot)
820 nc = dhi - dlo + 1
821 ! Pack on the device into the contiguous buf and move only buf: flang miscomputes a strided section of a block's field in a
822 ! target-update map clause (corrupts 2D+ np>1 seam ghosts). buf runs a fastest, then b, c, i; na/nb = transverse sizes.
823# 383 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
824 if (d == 1) then
825# 384 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
826 na = fm(2) + 1; nb = fm(3) + 1 ! scalars: kernel bounds use na-1/nb-1, not host fm(..) (nvfortran/Cray)
827# 385 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
828 if (dir == 1) then ! host <- device: pack on the device, copyout moves the contiguous buffer to host
829
830# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
831
832# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
833#if defined(MFC_OpenACC)
834# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
835!$acc parallel loop collapse(4) gang vector default(present) copyout(buf)
836# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
837#elif defined(MFC_OpenMP)
838# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
839
840# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
841
842# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
843
844# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
845!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(from:buf)
846# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
847#endif
848# 387 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
849 do i = 1, sys_size
850 do c = dlo, dhi
851 do b = 0, nb - 1
852 do a = 0, na - 1
853 buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))) = real(amr_cons_st(c, a, b, i, loc), wp)
854# 392 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
855 end do
856 end do
857 end do
858 end do
859
860# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
861#if defined(MFC_OpenACC)
862# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
863!$acc end parallel loop
864# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
865#elif defined(MFC_OpenMP)
866# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
867
868# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
869!$omp end target teams loop
870# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
871#endif
872# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
873 else ! device <- host: copyin moves the contiguous buffer to device, then unpack on the device
874
875# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
876
877# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
878#if defined(MFC_OpenACC)
879# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
880!$acc parallel loop collapse(4) gang vector default(present) copyin(buf)
881# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
882#elif defined(MFC_OpenMP)
883# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
884
885# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
886
887# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
888
889# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
890!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(to:buf)
891# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
892#endif
893# 399 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
894 do i = 1, sys_size
895 do c = dlo, dhi
896 do b = 0, nb - 1
897 do a = 0, na - 1
898 amr_cons_st(c, a, b, i, loc) = real(buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))), stp)
899# 404 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
900 end do
901 end do
902 end do
903 end do
904
905# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
906#if defined(MFC_OpenACC)
907# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
908!$acc end parallel loop
909# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
910#elif defined(MFC_OpenMP)
911# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
912
913# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
914!$omp end target teams loop
915# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
916#endif
917# 409 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
918 end if
919 end if
920# 383 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
921 if (d == 2) then
922# 384 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
923 na = fm(1) + 1; nb = fm(3) + 1 ! scalars: kernel bounds use na-1/nb-1, not host fm(..) (nvfortran/Cray)
924# 385 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
925 if (dir == 1) then ! host <- device: pack on the device, copyout moves the contiguous buffer to host
926
927# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
928
929# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
930#if defined(MFC_OpenACC)
931# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
932!$acc parallel loop collapse(4) gang vector default(present) copyout(buf)
933# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
934#elif defined(MFC_OpenMP)
935# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
936
937# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
938
939# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
940
941# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
942!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(from:buf)
943# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
944#endif
945# 387 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
946 do i = 1, sys_size
947 do c = dlo, dhi
948 do b = 0, nb - 1
949 do a = 0, na - 1
950 buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))) = real(amr_cons_st(a, c, b, i, loc), wp)
951# 392 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
952 end do
953 end do
954 end do
955 end do
956
957# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
958#if defined(MFC_OpenACC)
959# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
960!$acc end parallel loop
961# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
962#elif defined(MFC_OpenMP)
963# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
964
965# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
966!$omp end target teams loop
967# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
968#endif
969# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
970 else ! device <- host: copyin moves the contiguous buffer to device, then unpack on the device
971
972# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
973
974# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
975#if defined(MFC_OpenACC)
976# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
977!$acc parallel loop collapse(4) gang vector default(present) copyin(buf)
978# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
979#elif defined(MFC_OpenMP)
980# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
981
982# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
983
984# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
985
986# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
987!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(to:buf)
988# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
989#endif
990# 399 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
991 do i = 1, sys_size
992 do c = dlo, dhi
993 do b = 0, nb - 1
994 do a = 0, na - 1
995 amr_cons_st(a, c, b, i, loc) = real(buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))), stp)
996# 404 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
997 end do
998 end do
999 end do
1000 end do
1001
1002# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1003#if defined(MFC_OpenACC)
1004# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1005!$acc end parallel loop
1006# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1007#elif defined(MFC_OpenMP)
1008# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1009
1010# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1011!$omp end target teams loop
1012# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1013#endif
1014# 409 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1015 end if
1016 end if
1017# 383 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1018 if (d == 3) then
1019# 384 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1020 na = fm(1) + 1; nb = fm(2) + 1 ! scalars: kernel bounds use na-1/nb-1, not host fm(..) (nvfortran/Cray)
1021# 385 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1022 if (dir == 1) then ! host <- device: pack on the device, copyout moves the contiguous buffer to host
1023
1024# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1025
1026# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1027#if defined(MFC_OpenACC)
1028# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1029!$acc parallel loop collapse(4) gang vector default(present) copyout(buf)
1030# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1031#elif defined(MFC_OpenMP)
1032# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1033
1034# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1035
1036# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1037
1038# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1039!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(from:buf)
1040# 386 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1041#endif
1042# 387 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1043 do i = 1, sys_size
1044 do c = dlo, dhi
1045 do b = 0, nb - 1
1046 do a = 0, na - 1
1047 buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))) = real(amr_cons_st(a, b, c, i, loc), wp)
1048# 392 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1049 end do
1050 end do
1051 end do
1052 end do
1053
1054# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1055#if defined(MFC_OpenACC)
1056# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1057!$acc end parallel loop
1058# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1059#elif defined(MFC_OpenMP)
1060# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1061
1062# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1063!$omp end target teams loop
1064# 396 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1065#endif
1066# 397 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1067 else ! device <- host: copyin moves the contiguous buffer to device, then unpack on the device
1068
1069# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1070
1071# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1072#if defined(MFC_OpenACC)
1073# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1074!$acc parallel loop collapse(4) gang vector default(present) copyin(buf)
1075# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1076#elif defined(MFC_OpenMP)
1077# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1078
1079# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1080
1081# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1082
1083# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1084!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(4) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) map(to:buf)
1085# 398 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1086#endif
1087# 399 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1088 do i = 1, sys_size
1089 do c = dlo, dhi
1090 do b = 0, nb - 1
1091 do a = 0, na - 1
1092 amr_cons_st(a, b, c, i, loc) = real(buf(1 + a + na*(b + nb*(c - dlo + nc*(i - 1)))), stp)
1093# 404 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1094 end do
1095 end do
1096 end do
1097 end do
1098
1099# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1100#if defined(MFC_OpenACC)
1101# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1102!$acc end parallel loop
1103# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1104#elif defined(MFC_OpenMP)
1105# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1106
1107# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1108!$omp end target teams loop
1109# 408 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1110#endif
1111# 409 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1112 end if
1113 end if
1114# 412 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1115
1116 end subroutine s_amr_fine_slice
1117
1118 !> Convert a physical-space bbox to a global coarse-index bbox padded by pad_cells (uniform spacing; the axisymmetric half axis
1119 !! cell only shrinks dy(0), so the floor stays conservative).
1120 pure subroutine s_amr_phys_to_cells(pmin, pmax, pad_cells, blo, bhi)
1121
1122 real(wp), dimension(3), intent(in) :: pmin, pmax
1123 integer, intent(in) :: pad_cells
1124 integer, intent(out) :: blo(3), bhi(3)
1125
1126 blo = 0; bhi = 0
1127# 425 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1128 if (amr_dim(1)) then
1129# 426 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1130 blo(1) = int((pmin(1) - glb_bounds(1)%beg)/dx(0)) - pad_cells
1131# 427 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1132 bhi(1) = int((pmax(1) - glb_bounds(1)%beg)/dx(0)) + pad_cells
1133# 428 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1134 end if
1135# 425 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1136 if (amr_dim(2)) then
1137# 426 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1138 blo(2) = int((pmin(2) - glb_bounds(2)%beg)/dy(min(1, n))) - pad_cells
1139# 427 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1140 bhi(2) = int((pmax(2) - glb_bounds(2)%beg)/dy(min(1, n))) + pad_cells
1141# 428 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1142 end if
1143# 425 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1144 if (amr_dim(3)) then
1145# 426 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1146 blo(3) = int((pmin(3) - glb_bounds(3)%beg)/dz(0)) - pad_cells
1147# 427 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1148 bhi(3) = int((pmax(3) - glb_bounds(3)%beg)/dz(0)) + pad_cells
1149# 428 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1150 end if
1151# 430 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1152
1153 end subroutine s_amr_phys_to_cells
1154
1155 !> Rank-local per-stage guard: the local bubbles' padded bbox must stay clear of the current block (initial overlap, or drift
1156 !! beyond the regrid margin).
1157 impure subroutine s_amr_check_lag_clear()
1158
1159 real(wp), dimension(3) :: pmin_loc, pmax_loc
1160 integer :: blo(3), bhi(3)
1161
1162 if (.not. bubbles_lagrange) return
1163 call s_lag_cloud_bbox_local(pmin_loc, pmax_loc)
1164 if (pmin_loc(1) > pmax_loc(1)) return ! no bubbles on this rank
1165 call s_amr_phys_to_cells(pmin_loc, pmax_loc, mapcells + 2, blo, bhi)
1166 if (f_amr_boxes_overlap(blo, bhi, amr_slots(amr_cur)%region%lo, amr_slots(amr_cur)%region%hi)) then
1167 call s_mpi_abort('amr with Lagrangian bubbles: the bubble cloud (positions + smearing support) ' &
1168 & // 'overlaps an active fine block, where two-way coupling would be lost. Keep the initial ' &
1169 & // 'block clear of the cloud; under dynamic regrid, reduce amr_regrid_int or increase ' &
1170 & // 'amr_buf so the exclusion margin covers the cloud drift between regrids')
1171 end if
1172
1173 end subroutine s_amr_check_lag_clear
1174
1175 !> minmod slope limiter: 0 if a,b differ in sign, else the smaller-magnitude argument.
1176 pure elemental function minmod(a, b) result(m)
1177
1178
1179# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1180#if MFC_OpenACC
1181# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1182!$acc routine seq
1183# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1184#elif MFC_OpenMP
1185# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1186
1187# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1188
1189# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1190!$omp declare target device_type(any)
1191# 456 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1192#endif
1193# 457 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1194 real(wp), intent(in) :: a, b
1195 real(wp) :: m
1196
1197 if (a*b <= 0._wp) then
1198 m = 0._wp
1199 else if (abs(a) < abs(b)) then
1200 m = a
1201 else
1202 m = b
1203 end if
1204
1205 end function minmod
1206
1207 !> Minmod-limited linear value at fine offset (xix, xiy, xiz) in coarse cell u0, from its face neighbours (m, p) per direction;
1208 !! pass u0 for both neighbours of a direction without a slope.
1209 pure real(wp) function f_amr_lin(u0, xm, xp, ym, yp, zm, zp, xix, xiy, xiz) result(v)
1210
1211# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1212#if MFC_OpenACC
1213# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1214!$acc routine seq
1215# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1216#elif MFC_OpenMP
1217# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1218
1219# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1220
1221# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1222!$omp declare target device_type(any)
1223# 473 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1224#endif
1225# 474 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1226 real(wp), intent(in) :: u0, xm, xp, ym, yp, zm, zp, xix, xiy, xiz
1227
1228 v = u0 + minmod(xp - u0, u0 - xm)*xix + minmod(yp - u0, u0 - ym)*xiy + minmod(zp - u0, u0 - zm)*xiz
1229
1230 end function f_amr_lin
1231
1232 !> Admissible fine cell: positive density and (except model 4) positive internal energy; a NaN state is inadmissible.
1233 pure logical function f_amr_cons_ok(fi, fj, fk, loc) result(ok)
1234
1235
1236# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1237#if MFC_OpenACC
1238# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1239!$acc routine seq
1240# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1241#elif MFC_OpenMP
1242# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1243
1244# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1245
1246# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1247!$omp declare target device_type(any)
1248# 483 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1249#endif
1250# 484 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1251 integer, intent(in) :: fi, fj, fk, loc
1252 real(wp) :: rho, ke
1253 integer :: v
1254
1255 rho = 0._wp; ke = 0._wp
1256 do v = eqn_idx%cont%beg, eqn_idx%cont%end
1257 rho = rho + real(amr_cons_st(fi, fj, fk, v, loc), wp)
1258 end do
1259 ok = rho > 0._wp
1260 if (ok .and. model_eqns /= 4) then
1261 do v = eqn_idx%mom%beg, eqn_idx%mom%end
1262 ke = ke + real(amr_cons_st(fi, fj, fk, v, loc), wp)**2
1263 end do
1264 ok = real(amr_cons_st(fi, fj, fk, eqn_idx%E, loc), wp) - 0.5_wp*ke/rho > 0._wp
1265 end if
1266
1267 end function f_amr_cons_ok
1268
1269 !> Average over one half (xi < 0 left, > 0 right; |xi| = 1/4) of the middle of five cell averages q: the conservative quartic,
1270 !! or MC-limited linear where either half leaves the range of the three nearest averages. That range is widened only where the
1271 !! three second differences share a sign (smooth data): by the smallest of them, c, at a local extremum of the averages, and
1272 !! where they are monotone by c/4 times the ratio of the smallest to the largest, never across zero. A smooth extremum in the
1273 !! next cell needs c/8 (without it the fill is second order there) and has a ratio near one; at the foot of a steep front c is
1274 !! as large as q(0) and the ratio is small; at an under- or overshoot beside a front the differences change sign and the
1275 !! quartic's slope is the front's, not the cell's. Both halves are tested so the two children share one reconstruction (their
1276 !! mean stays q(0)).
1277 pure real(wp) function f_amr_quartic_half(q, xi) result(v)
1278
1279# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1280#if MFC_OpenACC
1281# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1282!$acc routine seq
1283# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1284#elif MFC_OpenMP
1285# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1286
1287# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1288
1289# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1290!$omp declare target device_type(any)
1291# 511 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1292#endif
1293# 512 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1294 real(wp), intent(in) :: q(-2:2), xi
1295 real(wp) :: c, a, b, d, dl, dc, dr, lo, hi
1296
1297 d = (3._wp*(q(-2) - q(2)) - 22._wp*(q(-1) - q(1)))/128._wp
1298 a = q(1) - q(0); b = q(0) - q(-1)
1299 dl = q(-2) - 2._wp*q(-1) + q(0); dc = a - b; dr = q(0) - 2._wp*q(1) + q(2)
1300 lo = min(q(-1), q(0), q(1)); hi = max(q(-1), q(0), q(1))
1301 c = 0._wp
1302 if (dl*dc > 0._wp .and. dc*dr > 0._wp) then
1303 c = min(abs(dl), abs(dc), abs(dr))
1304 if (a*b > 0._wp) c = min(0.25_wp*c*c/max(abs(dl), abs(dc), abs(dr)), max(lo, 0._wp, -hi))
1305 end if
1306 lo = lo - c; hi = hi + c
1307 if (min(q(0) - abs(d), q(0) + abs(d)) >= lo .and. q(0) + abs(d) <= hi) then
1308 v = q(0) + sign(1._wp, xi)*d
1309 else
1310 v = q(0)
1311 if (a*b > 0._wp) v = q(0) + sign(min(0.5_wp*abs(a + b), 2._wp*min(abs(a), abs(b))), a)*xi
1312 end if
1313
1314 end function f_amr_quartic_half
1315
1316 !> Allocate the pooled q_prim/rhs advance scratch (idempotent). The lockstep driver associates it for every block, so it must
1317 !! exist on every rank, including one that never allocates a slot. rhs gets the igr widening.
1318 impure subroutine s_amr_scr_init()
1319
1320 integer :: i, slo(3), shi(3)
1321 type(scalar_field), allocatable :: tmp_p(:), tmp_r(:)
1322
1323 if (allocated(amr_scr_prim)) return
1324 ! blocks are stacked along the last active dimension (see amr_cons_br)
1325 slo = [mbuf_lo(1), mbuf_lo(2), mbuf_lo(3)]; shi = [mbuf_hi(1), mbuf_hi(2), mbuf_hi(3)]
1326 shi(num_dims) = slo(num_dims) + amr_br_batch*(shi(num_dims) - slo(num_dims) + 1) - 1
1327 ! CCE OpenMP-offload: a bare module-scope scalar_field array aborts on allocate (lib-4425); allocate a local, move_alloc.
1328 allocate (tmp_p(1:sys_size)); call move_alloc(tmp_p, amr_scr_prim)
1329 allocate (tmp_r(1:sys_size)); call move_alloc(tmp_r, amr_scr_rhs)
1330
1331# 548 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1332#if defined(MFC_OpenACC)
1333# 548 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1334!$acc enter data create(amr_scr_prim, amr_scr_rhs)
1335# 548 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1336#elif defined(MFC_OpenMP)
1337# 548 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1338!$omp target enter data map(always,alloc:amr_scr_prim, amr_scr_rhs)
1339# 548 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1340#endif
1341# 549 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1342 do i = 1, sys_size
1343#ifdef MFC_DEBUG
1344# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1345 block
1346# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1347 use iso_fortran_env, only: output_unit
1348# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1349
1350# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1351 print *, 'm_amr_state.fpp:550: ', '@:ALLOCATE(amr_scr_prim(i)%sf(mbuf_lo(1):shi(1), mbuf_lo(2):shi(2), mbuf_lo(3):shi(3)))'
1352# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1353
1354# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1355 call flush (output_unit)
1356# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1357 end block
1358# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1359#endif
1360# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1361 allocate (amr_scr_prim(i)%sf(mbuf_lo(1):shi(1), mbuf_lo(2):shi(2), mbuf_lo(3):shi(3)))
1362# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1363
1364# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1365
1366# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1367#if defined(MFC_OpenACC)
1368# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1369!$acc enter data create(amr_scr_prim(i)%sf)
1370# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1371#elif defined(MFC_OpenMP)
1372# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1373!$omp target enter data map(always,alloc:amr_scr_prim(i)%sf)
1374# 550 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1375#endif
1376# 551 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1377 if (igr) then
1378#ifdef MFC_DEBUG
1379# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1380 block
1381# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1382 use iso_fortran_env, only: output_unit
1383# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1384
1385# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1386 print *, 'm_amr_state.fpp:552: ', '@:ALLOCATE(amr_scr_rhs(i)%sf(mbuf_lo(1):shi(1), min(mbuf_lo(2), -1):max(shi(2), 1), min(mbuf_lo(3), -1):max(shi(3), 1)))'
1387# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1388
1389# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1390 call flush (output_unit)
1391# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1392 end block
1393# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1394#endif
1395# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1396 allocate (amr_scr_rhs(i)%sf(mbuf_lo(1):shi(1), min(mbuf_lo(2), -1):max(shi(2), 1), min(mbuf_lo(3), -1):max(shi(3), 1)))
1397# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1398
1399# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1400
1401# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1402#if defined(MFC_OpenACC)
1403# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1404!$acc enter data create(amr_scr_rhs(i)%sf)
1405# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1406#elif defined(MFC_OpenMP)
1407# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1408!$omp target enter data map(always,alloc:amr_scr_rhs(i)%sf)
1409# 552 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1410#endif
1411# 554 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1412 else
1413#ifdef MFC_DEBUG
1414# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1415 block
1416# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1417 use iso_fortran_env, only: output_unit
1418# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1419
1420# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1421 print *, 'm_amr_state.fpp:555: ', '@:ALLOCATE(amr_scr_rhs(i)%sf(mbuf_lo(1):shi(1), mbuf_lo(2):shi(2), mbuf_lo(3):shi(3)))'
1422# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1423
1424# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1425 call flush (output_unit)
1426# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1427 end block
1428# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1429#endif
1430# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1431 allocate (amr_scr_rhs(i)%sf(mbuf_lo(1):shi(1), mbuf_lo(2):shi(2), mbuf_lo(3):shi(3)))
1432# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1433
1434# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1435
1436# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1437#if defined(MFC_OpenACC)
1438# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1439!$acc enter data create(amr_scr_rhs(i)%sf)
1440# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1441#elif defined(MFC_OpenMP)
1442# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1443!$omp target enter data map(always,alloc:amr_scr_rhs(i)%sf)
1444# 555 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1445#endif
1446# 556 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1447 end if
1448#ifdef _CRAYFTN
1449# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1450 block
1451# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1452#ifdef MFC_DEBUG
1453# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1454 block
1455# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1456 use iso_fortran_env, only: output_unit
1457# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1458
1459# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1460 print *, 'm_amr_state.fpp:557: ', '@:ACC_SETUP_SFs(amr_scr_prim(i))'
1461# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1462
1463# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1464 call flush (output_unit)
1465# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1466 end block
1467# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1468#endif
1469# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1470
1471# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1472
1473# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1474#if defined(MFC_OpenACC)
1475# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1476!$acc enter data copyin(amr_scr_prim(i))
1477# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1478#elif defined(MFC_OpenMP)
1479# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1480!$omp target enter data map(to:amr_scr_prim(i))
1481# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1482#endif
1483# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1484 if (associated(amr_scr_prim(i)%sf)) then
1485# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1486
1487# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1488#if defined(MFC_OpenACC)
1489# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1490!$acc enter data copyin(amr_scr_prim(i)%sf)
1491# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1492#elif defined(MFC_OpenMP)
1493# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1494!$omp target enter data map(to:amr_scr_prim(i)%sf)
1495# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1496#endif
1497# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1498 end if
1499# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1500 end block
1501# 557 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1502#endif
1503# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1504#ifdef _CRAYFTN
1505# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1506 block
1507# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1508#ifdef MFC_DEBUG
1509# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1510 block
1511# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1512 use iso_fortran_env, only: output_unit
1513# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1514
1515# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1516 print *, 'm_amr_state.fpp:558: ', '@:ACC_SETUP_SFs(amr_scr_rhs(i))'
1517# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1518
1519# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1520 call flush (output_unit)
1521# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1522 end block
1523# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1524#endif
1525# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1526
1527# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1528
1529# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1530#if defined(MFC_OpenACC)
1531# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1532!$acc enter data copyin(amr_scr_rhs(i))
1533# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1534#elif defined(MFC_OpenMP)
1535# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1536!$omp target enter data map(to:amr_scr_rhs(i))
1537# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1538#endif
1539# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1540 if (associated(amr_scr_rhs(i)%sf)) then
1541# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1542
1543# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1544#if defined(MFC_OpenACC)
1545# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1546!$acc enter data copyin(amr_scr_rhs(i)%sf)
1547# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1548#elif defined(MFC_OpenMP)
1549# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1550!$omp target enter data map(to:amr_scr_rhs(i)%sf)
1551# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1552#endif
1553# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1554 end if
1555# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1556 end block
1557# 558 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1558#endif
1559# 559 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1560 end do
1561 if (ib .or. synthetic_turbulence) then
1562 allocate (tmp_p(1:sys_size)); call move_alloc(tmp_p, amr_scr_prim_blk)
1563
1564# 562 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1565#if defined(MFC_OpenACC)
1566# 562 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1567!$acc enter data create(amr_scr_prim_blk)
1568# 562 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1569#elif defined(MFC_OpenMP)
1570# 562 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1571!$omp target enter data map(always,alloc:amr_scr_prim_blk)
1572# 562 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1573#endif
1574# 563 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1575 do i = 1, sys_size
1576#ifdef MFC_DEBUG
1577# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1578 block
1579# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1580 use iso_fortran_env, only: output_unit
1581# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1582
1583# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1584 print *, 'm_amr_state.fpp:564: ', '@:ALLOCATE(amr_scr_prim_blk(i)%sf(mbuf_lo(1):mbuf_hi(1), mbuf_lo(2):mbuf_hi(2), mbuf_lo(3):mbuf_hi(3)))'
1585# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1586
1587# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1588 call flush (output_unit)
1589# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1590 end block
1591# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1592#endif
1593# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1594 allocate (amr_scr_prim_blk(i)%sf(mbuf_lo(1):mbuf_hi(1), mbuf_lo(2):mbuf_hi(2), mbuf_lo(3):mbuf_hi(3)))
1595# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1596
1597# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1598
1599# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1600#if defined(MFC_OpenACC)
1601# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1602!$acc enter data create(amr_scr_prim_blk(i)%sf)
1603# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1604#elif defined(MFC_OpenMP)
1605# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1606!$omp target enter data map(always,alloc:amr_scr_prim_blk(i)%sf)
1607# 564 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1608#endif
1609# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1610#ifdef _CRAYFTN
1611# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1612 block
1613# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1614#ifdef MFC_DEBUG
1615# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1616 block
1617# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1618 use iso_fortran_env, only: output_unit
1619# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1620
1621# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1622 print *, 'm_amr_state.fpp:565: ', '@:ACC_SETUP_SFs(amr_scr_prim_blk(i))'
1623# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1624
1625# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1626 call flush (output_unit)
1627# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1628 end block
1629# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1630#endif
1631# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1632
1633# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1634
1635# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1636#if defined(MFC_OpenACC)
1637# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1638!$acc enter data copyin(amr_scr_prim_blk(i))
1639# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1640#elif defined(MFC_OpenMP)
1641# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1642!$omp target enter data map(to:amr_scr_prim_blk(i))
1643# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1644#endif
1645# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1646 if (associated(amr_scr_prim_blk(i)%sf)) then
1647# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1648
1649# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1650#if defined(MFC_OpenACC)
1651# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1652!$acc enter data copyin(amr_scr_prim_blk(i)%sf)
1653# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1654#elif defined(MFC_OpenMP)
1655# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1656!$omp target enter data map(to:amr_scr_prim_blk(i)%sf)
1657# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1658#endif
1659# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1660 end if
1661# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1662 end block
1663# 565 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1664#endif
1665# 566 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1666 end do
1667 end if
1668
1669 end subroutine s_amr_scr_init
1670
1671 !> Grow-on-demand sizing for append-only tables and pools: a grow preserves the entries already appended.
1672 impure subroutine s_amr_size_int(a, n)
1673
1674 integer, allocatable, intent(inout) :: a(:)
1675 integer, intent(in) :: n
1676 integer, allocatable :: tmp(:)
1677
1678 if (.not. allocated(a)) then
1679 allocate (a(max(n, 64)))
1680 return
1681 end if
1682 if (size(a) >= n) return
1683 call move_alloc(a, tmp)
1684 allocate (a(max(n, 2*size(tmp))))
1685 a(1:size(tmp)) = tmp
1686
1687 end subroutine s_amr_size_int
1688
1689 impure subroutine s_amr_size_int3(a, n)
1690
1691 integer, allocatable, intent(inout) :: a(:,:)
1692 integer, intent(in) :: n
1693
1694 call s_amr_size_rows(a, 3, n)
1695
1696 end subroutine s_amr_size_int3
1697
1698 !> s_amr_size_int for a table of n columns of r ints.
1699 impure subroutine s_amr_size_rows(a, r, n)
1700
1701 integer, allocatable, intent(inout) :: a(:,:)
1702 integer, intent(in) :: r, n
1703 integer, allocatable :: tmp(:,:)
1704
1705 if (.not. allocated(a)) then
1706 allocate (a(r, max(n, 64)))
1707 return
1708 end if
1709 if (size(a, 2) >= n) return
1710 call move_alloc(a, tmp)
1711 allocate (a(r, max(n, 2*size(tmp, 2))))
1712 a(:,1:size(tmp, 2)) = tmp
1713
1714 end subroutine s_amr_size_rows
1715
1716 !> Wire pools, preserving on grow. dev keeps the pool device-resident across (re)allocation: the old image is deleted from the
1717 !! device before it is freed and the new one created after; contents never survive a wave, so nothing is copied.
1718 impure subroutine s_amr_size_real(a, n, dev)
1719
1720 real(wp), allocatable, intent(inout) :: a(:)
1721 integer, intent(in) :: n
1722 logical, intent(in) :: dev
1723 real(wp), allocatable :: tmp(:)
1724
1725 if (.not. allocated(a)) then
1726 allocate (a(max(n, 64)))
1727 if (dev) then
1728
1729# 628 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1730#if defined(MFC_OpenACC)
1731# 628 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1732!$acc enter data create(a)
1733# 628 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1734#elif defined(MFC_OpenMP)
1735# 628 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1736!$omp target enter data map(always,alloc:a)
1737# 628 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1738#endif
1739# 629 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1740 end if
1741 return
1742 end if
1743 if (size(a) >= n) return
1744 if (dev) then
1745
1746# 634 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1747#if defined(MFC_OpenACC)
1748# 634 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1749!$acc exit data delete(a)
1750# 634 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1751#elif defined(MFC_OpenMP)
1752# 634 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1753!$omp target exit data map(release:a)
1754# 634 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1755#endif
1756# 635 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1757 end if
1758 call move_alloc(a, tmp)
1759 allocate (a(max(n, 2*size(tmp))))
1760 a(1:size(tmp)) = tmp
1761 if (dev) then
1762
1763# 640 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1764#if defined(MFC_OpenACC)
1765# 640 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1766!$acc enter data create(a)
1767# 640 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1768#elif defined(MFC_OpenMP)
1769# 640 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1770!$omp target enter data map(always,alloc:a)
1771# 640 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1772#endif
1773# 641 "/home/runner/work/MFC/MFC/src/simulation/m_amr_state.fpp"
1774 end if
1775
1776 end subroutine s_amr_size_real
1777
1778end module m_amr_state
integer, intent(in) k
integer, intent(in) j
AMR flux registers: per-RK-stage refluxing at the coarse/fine block boundary. Depends only on m_deriv...
pure subroutine, public s_amr_parent_foot(k, pblk, plo, phi)
Parent-fine footprint of block k inside its parent pblk from replicated metadata, so every rank gets ...
The AMR fine-level restart file layout shared by m_amr_restart (writer/reader) and post_process's ove...
pure subroutine, public s_amr_subdivide_cb(pcb, pcb_lb, lo, nfine, rr, fcb)
Fine cell boundaries fcb(-1:nfine) by rr-way subdivision of the coarse cells of pcb starting at coars...
Module-scope AMR state (block pool, flat field store, wave buffers) and the leaf helpers every AMR mo...
integer amr_cpat_mar
coarse-cell stencil reach (f_amr_cpat_mar)
integer, dimension(:), allocatable amr_ovl_scatter_n
per-block list lengths
integer, parameter amr_band_freg
integer amr_tag_narrow
Tag base of the clusterer's narrow-node reductions, above the per-box tag space [1....
integer, parameter amr_band_parent_fill
integer, dimension(:,:), allocatable amr_fx_pl
Fused exchange packs (amr_device_pack): one row per wave transfer (slab corner (1:3),...
integer, dimension(:,:), allocatable amr_sw_pfm
logical amr_myblk_dirty
Set wherever amr_block_owner is written (assigner, inheritance, restart/migration),...
real(wp), dimension(:), allocatable sw_y_cc
integer, dimension(:), allocatable amr_fx_pre
integer amr_cg_nuse
patches the level-1 pool gather filled (s_amr_l1_pool_gather)
impure subroutine s_amr_build_global_cb(pad)
Assemble the global coarse cell-boundary arrays over (-1-pad : G+pad): each rank writes its own cells...
real(wp), dimension(:), allocatable sw_z_cb
real(stp), dimension(:,:,:,:,:), allocatable amr_ibp
Moving bodies: the primitive state of the RHS pass of each block whose IB correction waits for the bo...
real(wp), dimension(:), allocatable sw_dx
impure subroutine s_amr_build_block_coords(k, gcb, glb, fcb, fcc, fdx, d)
Fine cell coordinates of block k in dimension d, rebuilt from the global coarse boundaries gcb by rep...
integer amr_swap_depth
Swap nesting depth: only the outermost swap saves the coarse state and only its restore puts it back;...
integer amr_loc_nfree
depth of the recycle stack
integer(8) amr_l1r_epoch
integer amr_wcur
consume cursor into the open fill wave's receive transfers (box-major)
real(wp), dimension(:), allocatable amr_sw_rq
integer amr_num_seam_pairs
integer amr_ibp_n
columns in use this stage
pure subroutine s_amr_phys_to_cells(pmin, pmax, pad_cells, blo, bhi)
Convert a physical-space bbox to a global coarse-index bbox padded by pad_cells (uniform spacing; the...
integer amr_l1r_nblk
second key half, matching s_amr_reg_prepare's (epoch, num_blocks) pair
real(wp), dimension(:), allocatable amr_gycb
integer, parameter amr_band_stage_fill
integer, dimension(:), allocatable amr_child_ptr
children of p = amr_child_idx(amr_child_ptr(p-1)+1 : amr_child_ptr(p)), ascending
integer, dimension(0:amr_n_bands - 1) amr_tag_gen
integer amr_loc_n
high-water mark of local indices handed out
real(wp), dimension(:), allocatable amr_fw_sq
Wire pools of the pooled waves (m_amr_wave lays them out; contents never survive a wave)....
integer, dimension(3) amr_maxc
max coarse block cells per dim: (m_glb+1)/2 etc.; 1 for collapsed dims
integer, parameter amr_band_restrict_l1
integer, parameter amr_band_restrict_parent
pure subroutine s_amr_patch_box(k, plo, phi)
Block k's padded coarse patch box [plo, phi] (region +/- amr_cpat_mar; collapsed dims 0:0).
integer, parameter amr_n_bands
integer, parameter amr_band_reflux
Keyed wave tags: amr_tag_base + band*65536 + gen*4096 + seq, checked against MPI_TAG_UB....
type(scalar_field), dimension(:), allocatable amr_scr_prim_blk
block-frame prim scratch for the batched per-member IB correction (ib only); the slab prim stacks mem...
pure integer function f_amr_cpat_mar()
Coarse-patch margin amr_cpat_mar: the coarse cells a buff_size fine ghost layer spans plus the prolon...
integer, dimension(:), allocatable amr_child_idx
(amr_child_ptr(0) = 0)
type(t_level), dimension(:), allocatable amr_slots
Fixed pool of refined-block slots; the working slot amr_cur selects which one per-block routines oper...
real(wp), dimension(:), allocatable sw_x_cb
impure subroutine s_amr_fine_slice(slot, d, dlo, dhi, buf, dir)
Pack (dir=+1) / unpack (dir=-1) the fine cells of slot's q_cons over [dlo:dhi] in dim d (interior tra...
integer sw_m
Swap contract (s_amr_swap_to_fine / s_amr_restore_coarse): the fine advance swaps these coarse global...
integer(kind=8), dimension(:,:), allocatable amr_fine_cut
(0:num_procs-1, 1:amr_max_level) SFC Morton-key upper bound per rank, one independent cost-weighted c...
real(stp), dimension(:), allocatable amr_pc
Subcycling patch cache: the pooled level-1 patches (amr_cg, amr_cg_nuse patches) at the step start (t...
integer, dimension(:,:), allocatable amr_tsq_tch
touched peers per dir, so the reset is O(active peers), not O(P)
integer, dimension(:), allocatable amr_fch_blk
foreign children of my owned parents (level >= 2, powner == me, cowner /= me)
real(wp), dimension(:), allocatable sw_x_cc
pure elemental real(wp) function minmod(a, b)
minmod slope limiter: 0 if a,b differ in sign, else the smaller-magnitude argument.
pure integer function f_amr_prolong_reach()
How many parent cells inside a child's footprint the ghost prolongation reads: 1 for limited linear,...
real(wp), dimension(:), allocatable amr_rf_rq
device-resident pools of the packed reflux-faces wave
integer, dimension(:,:), allocatable amr_seam_pairs
cached same-level seam list (xb, yb, seam-dim) so s_amr_fine_fine_post iterates O(seams); rebuilt whe...
integer, dimension(:), allocatable amr_parent_blk
cached f_amr_parent_block (0 for level <= 1), avoiding an O(global blocks) scan per block per stage
integer, dimension(:), allocatable amr_ovl_gather_n
real(wp), dimension(:), allocatable sw_y_cb
type(scalar_field), dimension(:), allocatable amr_cons_br
Copy bridge to the shared solver, whose routines take type(scalar_field) and serve the monolithic pat...
pure integer function f_amr_parent_block(k)
Index of the level-(level(k)-1) block that block k refines, or 0 for level 1 (parent is the L0 grid)....
pure real(wp) function f_amr_quartic_half(q, xi)
Average over one half (xi < 0 left, > 0 right; |xi| = 1/4) of the middle of five cell averages q: the...
integer, parameter amr_band_seam
logical sw_acoustic_source
real(wp), dimension(:), allocatable sw_dz
integer amr_seam_pairs_nblk
subroutine s_build_level_coords(pcb, pcb_lb, lo, nfine, fcb, fcc, fdx)
Fill fcb/fcc/fdx by subdividing parent cells (s_amr_subdivide_cb); pcb_lb is lbound(pcb,...
type(scalar_field), dimension(:), allocatable amr_cg
type(scalar_field), dimension(:), allocatable amr_scr_rhs
integer, dimension(:), allocatable amr_loc_of
Dense local index for live slots: amr_slots is sparse over 1:amr_max_blocks, the field store is dense...
real(wp), dimension(:), allocatable amr_sw_sq
impure subroutine s_amr_size_real(a, n, dev)
Wire pools, preserving on grow. dev keeps the pool device-resident across (re)allocation: the old ima...
impure subroutine s_amr_size_rows(a, r, n)
s_amr_size_int for a table of n columns of r ints.
integer amr_br_batch
Blocks per batched s_compute_rhs call (amr_bat_max once initialized); bounded to avoid exhausting dev...
pure subroutine s_amr_region_box(k, rlo, rhi)
Block k's region box in global coarse cells; collapsed dimensions read 0:0.
integer, dimension(3) max_f
Per-slot sizing: max fine cells per dim and the buffered array bounds. amr_slot_live(k): slot k's fie...
real(wp), dimension(:,:,:), allocatable sw_jac_old
integer, dimension(:), allocatable amr_sw_ply
integer, dimension(:), allocatable amr_sw_pd
integer amr_slab_n
rows queued in amr_slab_tab (s_amr_slab_add)
logical, dimension(:), allocatable amr_slot_live
integer, dimension(3) amr_cpat_hi
amr_cg upper local bounds per dim (0 in collapsed dims)
logical amr_fw_dev
Device-resident wire pools: with rdma_mpi MPI sends/receives them by device address,...
pure logical function f_amr_boxes_overlap(alo, ahi, blo, bhi)
Do two coarse-index boxes [alo:ahi] and [blo:bhi] overlap? Collapsed dims never disqualify.
integer, dimension(:), allocatable amr_sw_pxhi
integer amr_sw_nsame
seam pairs both of whose blocks this rank owns: exchanged by one batched kernel at drain,...
impure subroutine s_amr_scr_init()
Allocate the pooled q_prim/rhs advance scratch (idempotent). The lockstep driver associates it for ev...
pure real(wp) function f_amr_lin(u0, xm, xp, ym, yp, zm, zp, xix, xiy, xiz)
Minmod-limited linear value at fine offset (xix, xiy, xiz) in coarse cell u0, from its face neighbour...
integer amr_cg_np
patches amr_cg holds, stacked along dim 3 (s_amr_cg_reserve)
integer amr_st_cap
local slots the store is sized for; grows only (s_amr_compact_store keeps the index space dense)
type(int_bounds_info), dimension(3) sw_idwbuff
real(wp), dimension(:), allocatable sw_dy
real(wp), dimension(:), allocatable amr_gxcb
Global coarse cell-boundary arrays (-1:X_glb), assembled once at init; the fine-distribution owner re...
real(wp), dimension(:), allocatable amr_rf_sq
integer, dimension(:), allocatable amr_fw_rblk
integer, parameter amr_band_migrate
logical amr_seam_pairs_dirty
real(stp), dimension(:,:,:,:,:), allocatable amr_cons_st
Flat per-block field store (x, y, z, var, local slot) so one kernel can run over every live block; al...
integer, dimension(:,:), allocatable amr_ovl_scatter
pure logical function f_amr_cons_ok(fi, fj, fk, loc)
Admissible fine cell: positive density and (except model 4) positive internal energy; a NaN state is ...
integer, dimension(amr_bat_max) amr_bat_loc
Batched advance: the bridge spans amr_br_batch blocks along the last active dimension,...
real(stp), dimension(:,:,:,:,:), allocatable amr_stor_st
integer, dimension(:), allocatable amr_own_blk
blocks this rank intersects (amr_owns_all, the multi-owner notion, all levels)
integer, dimension(:), allocatable amr_ibp_blk
the block of each amr_ibp column
real(wp), dimension(:,:,:), allocatable sw_jac
IGR sigma-state bounce: the fine solve reuses jac/jac_old at fine indices, so the coarse contents (ja...
integer, dimension(3) mbuf_lo
real(wp), dimension(:), allocatable amr_fw_rq
integer, dimension(:), allocatable amr_l1r_blk
Level-1 blocks this rank receives restriction for (not its own, overlapping its interior),...
real(wp), dimension(:), allocatable amr_gzcb
integer, dimension(:), allocatable amr_sw_plx
integer, dimension(:), allocatable amr_cg_pz
dim-3 offset of each owned level-1 box's patch in the pool
integer, dimension(:), allocatable amr_l1p_blk
padded variant (region +/- amr_cpat_mar vs my coarse range): stage-fill gather
integer, dimension(:,:), allocatable amr_slab_tab
Per-block gathered coarse patch: the owner may not hold the coarse cells its block refines,...
impure subroutine s_amr_size_int3(a, n)
integer, dimension(:), allocatable amr_my_blk
Block indices this rank owns, ascending (the paired MPI_SENDRECVs depend on the order); built in s_am...
type(scalar_field), dimension(:), allocatable amr_scr_prim
Pooled advance scratch shared by every fine block (the fused advance leaves no cross-block lifetime);...
integer, dimension(:), allocatable amr_loc_free
stack of recycled local indices
integer, dimension(3) amr_maxc_fit
Regrid box size cap per dim (fixed for the run, identical on all ranks; 1 in collapsed dims): a box o...
integer, dimension(3) mbuf_hi
integer, dimension(:,:), allocatable amr_ovl_gather
cached per-block P2P overlap-rank lists, rebuilt with the seam list: amr_ovl_gather(:,...
integer, dimension(3) amr_cpat_off
global coarse index of amr_cg local cell 0 (region_lo - amr_cpat_mar)
impure subroutine s_amr_check_lag_clear()
Rank-local per-stage guard: the local bubbles' padded bbox must stay clear of the current block (init...
type(int_bounds_info), dimension(3) sw_idwint
impure subroutine s_amr_size_int(a, n)
Grow-on-demand sizing for append-only tables and pools: a grow preserves the entries already appended...
integer, dimension(:,:), allocatable amr_tsq
(0:np-1, dir) in-wave per-peer seq counters; touched-reset
real(wp), dimension(:), allocatable sw_z_cc
pure subroutine s_amr_patch_core(k, clo, chi)
Core [clo, chi] of the patch box that the ghost fill never reads, so the fill waves ship only the she...
logical amr_xchg_coarse_ghosts
True (identically on all ranks) iff some rank's fine ghost fill reads its coarse ghost cells: the sol...
integer, dimension(2) amr_n_tsq
integer amr_xchg_bad
local accumulator behind amr_xchg_coarse_ghosts; s_amr_reduce_xchg_flag does one allreduce per scan
Real fields of a bubble MPI slot, as (names, indices, count) in pack order. Used by both pack and unp...
impure subroutine s_lag_cloud_bbox_local(pmin, pmax)
Rank-local bounding box of Lagrangian bubble positions; pmin > pmax when the rank holds none.
Compile-time constant parameters: default values, tolerances, and physical constants.
integer, parameter mapcells
Number of cells around the bubble where the smoothening function will have effect.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the computational domain, fluid properties, and simulation algorithm configurat...
real(wp), dimension(:), allocatable, target z_cb
integer, dimension(:,:), allocatable amr_region_lo_all
Per-slot mirrors (allocated 1:amr_max_blocks by m_amr): region box, rank intersection and ownership f...
integer, dimension(:,:), allocatable amr_region_hi_all
type(bounds_info), dimension(3) glb_bounds
type(int_bounds_info), dimension(1:3) idwbuff
integer, dimension(:), allocatable amr_block_level
Refinement level of each active block (1..amr_max_level) and the deepest level populated....
integer buff_size
Number of ghost cells for boundary condition storage.
real(wp), dimension(:), allocatable, target y_cc
logical, dimension(3) amr_dim
The active dimensions, and this rank's coarse subdomain origin and extent padded to 3-vectors (0 in c...
integer, parameter amr_bat_max
Batched fine advance: amr_bat_n members (0 outside a batch), their block ids, shared extents,...
real(wp), dimension(:), allocatable, target z_cc
real(wp), dimension(:), allocatable, target x_cc
real(wp), dimension(:), allocatable, target y_cb
integer amr_num_blocks
Number of active AMR fine-block slots (>= 1) and the working slot index selecting which one the per-b...
real(wp), dimension(:), allocatable, target dy
real(wp), dimension(:), allocatable, target dz
real(wp), dimension(:), allocatable, target dx
real(wp), dimension(:), allocatable, target x_cb
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_abort(prnt, code)
The subroutine terminates the MPI execution environment.
impure subroutine s_mpi_allreduce_array_max(arr, arr_len)
In-place elementwise max-reduction of a real array across all ranks: each element is written (identic...
MPI halo exchange, domain decomposition, and buffer packing/unpacking for the simulation solver.
One refined level: its own grid and metadata (host-only). Fields live in the flat stores amr_cons_st/...
Integer bounds for variables.
Derived type annexing a scalar field (SF).
An index-space rectangle in global cell indices. In v1, one t_box = one rank's subdomain....