MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_amr_frame.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
2!>
3!!@file
4!!@brief Contains module m_amr_frame
5
6# 7 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
7# 8 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
8# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
9# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
10# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
11# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
15# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
16
17# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
19# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
20
21# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
22# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
23! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
24! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
25# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
26# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
27# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
28
29# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
30# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
31
32# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
33
34# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
35
36# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
37
38# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
39
40# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
41
42# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
43
44# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
45
46# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
47! New line at end of file is required for FYPP
48# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
49# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
50# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
51# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
55# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
56
57# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
59# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
60
61# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
62# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
63! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
64! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
65# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
66# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
67# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
68
69# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
70# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75
76# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
77
78# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
79
80# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
81
82# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
83
84# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
85
86# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
87! New line at end of file is required for FYPP
88# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
89
90# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
94# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129
130# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
131
132# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
133
134# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
135
136# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
137
138# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
139
140# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
141# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154
155# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
156
157# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
158
159# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
160
161# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
162
163# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
164
165# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
166! New line at end of file is required for FYPP
167# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
168# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
169# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
170# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
174# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175
176# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
178# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179
180# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
181# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
182! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
183! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
184# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
185# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
186# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
187
188# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
189# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
190
191# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
192
193# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
194
195# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
196
197# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
198
199# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
200
201# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
202
203# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
204
205# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
206! New line at end of file is required for FYPP
207# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
208
209# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
210
211# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
212
213# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
214
215# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
216
217# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
218
219# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
220
221# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
222
223# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
224
225# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
226
227# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
228
229# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
230
231# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
232
233# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
234
235# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
236
237# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
238
239# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
240
241# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
242
243# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
244
245# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
246
247# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
248
249# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
250
251# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
252
253# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
254
255# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
256
257# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
258
259# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
260
261# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
262
263# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
264! New line at end of file is required for FYPP
265# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
266
267! GPU parallel region (scalar reductions, maxval/minval)
268# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
269
270! GPU parallel loop over threads (most common GPU macro)
271# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
272
273! Required closing for GPU_PARALLEL_LOOP
274# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
275
276! Mark routine for device compilation
277# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
278
279! Declare device-resident data
280# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
281
282! Inner loop within a GPU parallel region
283# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
284
285! Scoped GPU data region
286# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
287
288! Host code with device pointers (for MPI with GPU buffers)
289# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
290
291! Allocate device memory (unscoped)
292# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
293
294! Free device memory
295# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
296
297! Atomic operation on device
298# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
299
300! End atomic capture block
301# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
302
303! Copy data between host and device
304# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
305
306! Synchronization barrier
307# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
308
309! Import GPU library module (openacc or omp_lib)
310# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
311
312! Emit code only for AMD compiler
313# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
314
315! Emit code for non-Cray compilers
316# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
317
318! Emit code only for Cray compiler
319# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
320
321! Cray compiler directive, emitted from Fypp so the formatter cannot turn it into a comment
322# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
323
324! Emit code for non-NVIDIA compilers
325# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
326
327# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
328# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
329! New line at end of file is required for FYPP
330# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
331
332# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
333
334! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
335! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
336! example see misc/nvidia_uvm/bind.sh.
337# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
338
339! Allocate and create GPU device memory
340# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
341
342! Free GPU device memory and deallocate
343# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
344
345! Cray-specific GPU pointer setup for vector fields
346# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
347
348! Cray-specific GPU pointer setup for scalar fields
349# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
350
351! Cray-specific GPU pointer teardown for scalar fields - exact inverse of ACC_SETUP_SFs. Removes the
352! present-table entries ACC_SETUP_SFs added (the scalar_field descriptor and its %sf copyin). REQUIRED when
353! an ACC_SETUP'd field is freed MID-RUN (e.g. AMR slot free on regrid/restart): Cray 'exit data delete'
354! decrements the reference counter, so the lone @:DEALLOCATE(arg%sf) only undoes the @:ALLOCATE create ref
355! and leaves the descriptor + the ACC_SETUP %sf copyin dangling. Call BEFORE @:DEALLOCATE(arg%sf).
356# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
357
358! Cray-specific GPU pointer setup for acoustic source spatials
359# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
360
361! Whether stress index idx is a shear component. Spelled out because NVHPC 26.9 at -O2 vectorizes any(idx == shear_indices)
362! inside a loop to always true, which doubled the normal-stress elastic energy.
363# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
364
365# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
366
367# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
368! New line at end of file is required for FYPP
369# 9 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp" 2
370
371!> @brief Block-frame swap (fine grid state in/out of the shared solver).
373
374 use m_derived_types ! scalar_field, t_box, int_bounds_info
376 use m_mpi_proxy, only: s_mpi_abort ! @:ASSERT expands to it
378 use m_igr, only: jac, jac_old
379 use m_amr_state
381 use m_amr_store
383
384 implicit none
385
386 private
388
389contains
390
391 !> Swap to the current fine block and load its solver frame amr_cons_br; paired with s_amr_fine_leave.
392 impure subroutine s_amr_fine_enter()
393
394 integer :: loc_cur !< plain scalar for device routines (NVHPC -Minline)
395
396 call s_amr_swap_to_fine()
397 loc_cur = amr_loc_of(amr_cur)
398 call s_amr_br_load(loc_cur)
399
400 end subroutine s_amr_fine_enter
401
402 !> Store the solver frame back to the fine block and restore the coarse grid.
403 impure subroutine s_amr_fine_leave()
404
405 integer :: loc_cur
406
407 loc_cur = amr_loc_of(amr_cur)
408 call s_amr_br_store(loc_cur)
410
411 end subroutine s_amr_fine_leave
412
413 !> Swap the global grid state to the fine block. Must be paired with s_amr_restore_coarse.
414 impure subroutine s_amr_swap_to_fine(widen)
415
416 logical, intent(in), optional :: widen !< also widen idwint to the ghost shells (the RHS conversion range)
417 integer :: ibm, o, e
418
419 ! Saves are depth-guarded: a nested save would overwrite the sw_* bounce buffers with fine state, and the outermost restore
420 ! would install fine extents as the coarse grid. Installs are idempotent, so unguarded.
421
423 if (amr_swap_depth == 1) then
424 sw_m = m; sw_n = n; sw_p = p
426 end if
427 ! the acoustic source's spatials are coarse-grid cell indices; its support never overlaps a block (checked at startup), so
428 ! the fine RHS skips it
429 if (amr_swap_depth == 1) sw_acoustic_source = acoustic_source
430 acoustic_source = .false.
431 m = amr_slots(amr_cur)%m; n = amr_slots(amr_cur)%n; p = amr_slots(amr_cur)%p
432 idwint%beg = 0; idwint%end = [m, n, p]
433 idwbuff = amr_slots(amr_cur)%idwbuff
434 ! save coarse coords, then install the fine coords and extend them into the ghost shell (s_build_level_coords fills 0:m)
435# 75 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
436 if (amr_dim(1)) then
437# 76 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
438 if (amr_swap_depth == 1) then; sw_x_cb = x_cb; sw_x_cc = x_cc; sw_dx = dx; end if
439# 77 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
440 x_cb(-1:m) = amr_slots(amr_cur)%x_cb(-1:m)
441# 78 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
442 x_cc(0:m) = amr_slots(amr_cur)%x_cc(0:m)
443# 79 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
444 dx(0:m) = amr_slots(amr_cur)%dx(0:m)
445# 80 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
446 call s_amr_extend_ghost_coords(1, x_cb, x_cc, dx, m, lbound(amr_gxcb, 1), amr_gxcb)
447# 81 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
448 end if
449# 75 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
450 if (amr_dim(2)) then
451# 76 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
452 if (amr_swap_depth == 1) then; sw_y_cb = y_cb; sw_y_cc = y_cc; sw_dy = dy; end if
453# 77 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
454 y_cb(-1:n) = amr_slots(amr_cur)%y_cb(-1:n)
455# 78 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
456 y_cc(0:n) = amr_slots(amr_cur)%y_cc(0:n)
457# 79 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
458 dy(0:n) = amr_slots(amr_cur)%dy(0:n)
459# 80 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
460 call s_amr_extend_ghost_coords(2, y_cb, y_cc, dy, n, lbound(amr_gycb, 1), amr_gycb)
461# 81 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
462 end if
463# 75 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
464 if (amr_dim(3)) then
465# 76 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
466 if (amr_swap_depth == 1) then; sw_z_cb = z_cb; sw_z_cc = z_cc; sw_dz = dz; end if
467# 77 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
468 z_cb(-1:p) = amr_slots(amr_cur)%z_cb(-1:p)
469# 78 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
470 z_cc(0:p) = amr_slots(amr_cur)%z_cc(0:p)
471# 79 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
472 dz(0:p) = amr_slots(amr_cur)%dz(0:p)
473# 80 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
474 call s_amr_extend_ghost_coords(3, z_cb, z_cc, dz, p, lbound(amr_gzcb, 1), amr_gzcb)
475# 81 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
476 end if
477# 83 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
478 ! batched advance: extend the leader's grid into the slab of amr_bat_n stacked blocks (stride amr_bat_w along amr_bat_sd),
479 ! since the flux divergence reads dx/dy/dz at every slab cell. Cell boundaries are not replicated: the batched path does not
480 ! read them.
481 if (amr_bat_n > 1) then
482 do ibm = 2, amr_bat_n
483# 89 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
484 o = amr_bat_off(1, ibm); e = amr_bat_ext(1)
485# 90 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
486 if (o > 0) then
487 x_cc(o - buff_size:o + e + buff_size) = x_cc(-buff_size:e + buff_size)
488# 92 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
489 dx(o - buff_size:o + e + buff_size) = dx(-buff_size:e + buff_size)
490# 93 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
491 end if
492# 89 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
493 o = amr_bat_off(2, ibm); e = amr_bat_ext(2)
494# 90 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
495 if (o > 0) then
496 y_cc(o - buff_size:o + e + buff_size) = y_cc(-buff_size:e + buff_size)
497# 92 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
498 dy(o - buff_size:o + e + buff_size) = dy(-buff_size:e + buff_size)
499# 93 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
500 end if
501# 89 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
502 o = amr_bat_off(3, ibm); e = amr_bat_ext(3)
503# 90 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
504 if (o > 0) then
505 z_cc(o - buff_size:o + e + buff_size) = z_cc(-buff_size:e + buff_size)
506# 92 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
507 dz(o - buff_size:o + e + buff_size) = dz(-buff_size:e + buff_size)
508# 93 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
509 end if
510# 95 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
511 end do
512# 97 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
513 if (1 == amr_bat_sd) then
514# 98 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
515 e = maxval(amr_bat_off(1,1:amr_bat_n)) + amr_bat_ext(1)
516# 99 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
517 m = e; idwint(1)%end = e; idwbuff(1)%end = e + buff_size
518# 100 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
519 end if
520# 97 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
521 if (2 == amr_bat_sd) then
522# 98 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
523 e = maxval(amr_bat_off(2,1:amr_bat_n)) + amr_bat_ext(2)
524# 99 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
525 n = e; idwint(2)%end = e; idwbuff(2)%end = e + buff_size
526# 100 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
527 end if
528# 97 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
529 if (3 == amr_bat_sd) then
530# 98 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
531 e = maxval(amr_bat_off(3,1:amr_bat_n)) + amr_bat_ext(3)
532# 99 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
533 p = e; idwint(3)%end = e; idwbuff(3)%end = e + buff_size
534# 100 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
535 end if
536# 102 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
537 end if
538 if (present(widen)) then
539 if (widen) idwint = idwbuff
540 end if
541 ! sync extents/bounds/coordinates to the GPU_DECLARE'd device copies (stale coarse bounds = OOB kernels)
543 ! hypoelastic FD coefficients depend on grid spacing: recompute them for the fine grid
544 if (hypoelasticity) call s_hypoelastic_update_fd_coeffs()
545 ! IGR: save the coarse sigma and seed the fine solve with its parent values (best initial guess, and the frozen Dirichlet
546 ! ghost data for the block-local Jacobi solve)
547 if (igr) call s_amr_igr_swap_sigma()
548
549 end subroutine s_amr_swap_to_fine
550
551 !> Extend one axis of the installed fine grid into the ghost shell with the same parent-cell bisection the interior uses:
552 !! boundary jg belongs to parent cell cl = isect_lo + floor(jg/rr), and k = modulo(jg, rr) picks the rr-way split. The parent
553 !! boundaries are gcb for a level-1 block; a level>=2 block's parent grid is rebuilt from replicated metadata (the parent's own
554 !! arrays exist only on its owner). Blocks stay buff_size inside the domain, so every ghost parent is an in-domain cell.
555 impure subroutine s_amr_extend_ghost_coords(d, cb, cc, dd, ext, glb, gcb)
556
557 integer, intent(in) :: d, ext, glb
558 real(wp), intent(inout) :: cb(-1 - buff_size:), cc(-buff_size:), dd(-buff_size:)
559 real(wp), intent(in) :: gcb(glb:)
560 real(wp), allocatable :: pcb(:), tcc(:), tdx(:)
561 integer :: jg, cl, k, rr, pblk, pnf
562
563 rr = amr_slots(amr_cur)%amr_ref_ratio
564 if (amr_block_level(amr_cur) >= 2) then
566 pnf = amr_ref_ratio**amr_block_level(pblk)*(amr_region_hi_all(d, pblk) - amr_region_lo_all(d, pblk) + 1) - 1
567 allocate (pcb(-1:pnf), tcc(0:pnf), tdx(0:pnf))
568 call s_amr_build_block_coords(pblk, gcb, glb, pcb, tcc, tdx, d)
569 else
570 allocate (pcb(glb:ubound(gcb, 1))); pcb = gcb
571 end if
572 do jg = -1 - buff_size, ext + buff_size
573 if (jg >= 0 .and. jg <= ext) cycle
574 cl = amr_isect_lo(d) + floor(real(jg, wp)/real(rr, wp))
575 k = modulo(jg, rr)
576 if (k == rr - 1) then
577 cb(jg) = pcb(cl)
578 else
579 cb(jg) = (real(rr - 1 - k, wp)*pcb(cl - 1) + real(k + 1, wp)*pcb(cl))/real(rr, wp)
580 end if
581 end do
582 do jg = -buff_size, ext + buff_size
583 if (jg >= 0 .and. jg <= ext) cycle
584 dd(jg) = cb(jg) - cb(jg - 1); cc(jg) = 0.5_wp*(cb(jg - 1) + cb(jg))
585 end do
586
587 end subroutine s_amr_extend_ghost_coords
588
589 !> Restore the global grid state saved by s_amr_swap_to_fine.
590 impure subroutine s_amr_restore_coarse(sync_device)
591
592 !> .false. only from the batched stage when another batch follows at once (its swap re-pushes the grid state)
593 logical, intent(in), optional :: sync_device
594 logical :: sync
595
596 if (.not. (amr_swap_depth > 0)) then
597# 161 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
598 call s_mpi_abort("m_amr_frame.fpp:161: " // "Assertion failed: amr_swap_depth > 0. " &
599# 161 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
600 & // "s_amr_restore_coarse without a matching s_amr_swap_to_fine")
601# 161 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
602 end if
603# 162 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
605 ! inner restore: the enclosing swap frame still needs its slot installed and the sw_* buffers still hold the coarse state
606 if (amr_swap_depth > 0) return
607 m = sw_m; n = sw_n; p = sw_p
609 acoustic_source = sw_acoustic_source
610 ! restore full coarse coords
611# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
612 if (amr_dim(1)) then; x_cb = sw_x_cb; x_cc = sw_x_cc; dx = sw_dx; end if
613# 171 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
614# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
615 if (amr_dim(2)) then; y_cb = sw_y_cb; y_cc = sw_y_cc; dy = sw_dy; end if
616# 171 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
617# 170 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
618 if (amr_dim(3)) then; z_cb = sw_z_cb; z_cc = sw_z_cc; dz = sw_dz; end if
619# 171 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
620# 172 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
621 sync = .true.; if (present(sync_device)) sync = sync_device
622 if (sync) call s_amr_sync_grid_state_to_device()
623 if (hypoelasticity) call s_hypoelastic_update_fd_coeffs()
624 if (igr) call s_amr_igr_restore_sigma()
625
626 end subroutine s_amr_restore_coarse
627
628 !> Save the coarse jac/jac_old and seed the (already swapped-in) fine block's sigma by piecewise-constant parent injection from
629 !! the saved coarse sigma: interior = initial guess, ghost shell = frozen Dirichlet coupling data.
630 impure subroutine s_amr_igr_swap_sigma()
631
632 integer :: j, k, l, ci, cj, ck, ibm, g, o1, o2, o3, mb1, me1, mb2, me2, mb3, me3, lo1, lo2, lo3, ox, oy, oz
633 integer :: cb1, ce1, cb2, ce2, cb3, ce3, lo(3), o(3), mb(3), me(3)
634
635 ! bounds hoisted to scalars: sw_idwbuff etc. are host-only, so naming them in a kernel fails OpenACC's present lookup
636
637 cb1 = sw_idwbuff(1)%beg; ce1 = sw_idwbuff(1)%end
638 cb2 = sw_idwbuff(2)%beg; ce2 = sw_idwbuff(2)%end
639 cb3 = sw_idwbuff(3)%beg; ce3 = sw_idwbuff(3)%end
640 ox = amr_sidx(1); oy = amr_sidx(2); oz = amr_sidx(3)
641 ! Save the coarse sigma, outermost swap only (a nested save would overwrite sw_jac with fine state). The seed below is
642 ! unguarded: sw_jac still holds the coarse state. jac_old exists only for the Jacobi solver (igr_iter_solver == 1), so it
643 ! gets its own kernel: naming an unallocated array in a launched kernel aborts under amdflang.
644 if (amr_swap_depth == 1) then
645# 197 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
646# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
647
648# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
649
650# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
651#if defined(MFC_OpenACC)
652# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
653!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l)
654# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
655#elif defined(MFC_OpenMP)
656# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
657
658# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
659
660# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
661
662# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
663!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) private(j, k, l)
664# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
665#endif
666# 201 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
667 do l = cb3, ce3
668 do k = cb2, ce2
669 do j = cb1, ce1
670 sw_jac(j, k, l) = jac(j, k, l)
671# 205 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
672 end do
673 end do
674 end do
675
676# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
677#if defined(MFC_OpenACC)
678# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
679!$acc end parallel loop
680# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
681#elif defined(MFC_OpenMP)
682# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
683
684# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
685!$omp end target teams loop
686# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
687#endif
688# 209 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
689# 212 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
690# 197 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
691# 198 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
692 if (igr_iter_solver == 1) then
693# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
694
695# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
696
697# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
698#if defined(MFC_OpenACC)
699# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
700!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l)
701# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
702#elif defined(MFC_OpenMP)
703# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
704
705# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
706
707# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
708
709# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
710!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) private(j, k, l)
711# 200 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
712#endif
713# 201 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
714 do l = cb3, ce3
715 do k = cb2, ce2
716 do j = cb1, ce1
717 sw_jac_old(j, k, l) = jac_old(j, k, l)
718# 205 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
719 end do
720 end do
721 end do
722
723# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
724#if defined(MFC_OpenACC)
725# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
726!$acc end parallel loop
727# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
728#elif defined(MFC_OpenMP)
729# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
730
731# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
732!$omp end target teams loop
733# 208 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
734#endif
735# 209 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
736# 210 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
737 end if
738# 212 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
739# 213 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
740 end if
741 ! seed each block's buffered range from its own parent; in a batched slab member ibm sits at offset o along amr_bat_sd and
742 ! its own extents bound the seed
743 do ibm = 1, max(1, amr_bat_n)
744 if (amr_bat_n > 1) then
745 g = amr_bat_blk(ibm); lo = amr_isect_lo_all(:,g)
746 o = amr_bat_off(:,ibm)
747 mb = merge(-buff_size, 0, amr_dim); me = merge(amr_bat_mext(:,ibm) + buff_size, 0, amr_dim)
748 else
749 lo = amr_isect_lo; o = 0; mb = idwbuff%beg; me = idwbuff%end
750 end if
751 lo1 = lo(1); lo2 = lo(2); lo3 = lo(3); o1 = o(1); o2 = o(2); o3 = o(3)
752 mb1 = mb(1); mb2 = mb(2); mb3 = mb(3); me1 = me(1); me2 = me(2); me3 = me(3)
753# 227 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
754# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
755
756# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
757
758# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
759#if defined(MFC_OpenACC)
760# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
761!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l, ci, cj, ck) copyin(lo1, lo2, lo3, o1, o2, o3, mb1, me1, mb2, me2, mb3, me3)
762# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
763#elif defined(MFC_OpenMP)
764# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
765
766# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
767
768# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
769
770# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
771!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) &
772# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
773!$omp& private(j, k, l, ci, cj, ck) map(to:lo1, lo2, lo3, o1, o2, o3, mb1, me1, mb2, me2, mb3, me3)
774# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
775#endif
776# 232 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
777 do l = mb3, me3
778 do k = mb2, me2
779 do j = mb1, me1
780 ci = lo1 + floor(real(j, wp)/real(amr_ref_ratio, wp)) - ox
781 cj = 0; ck = 0
782 if (n_glb > 0) cj = lo2 + floor(real(k, wp)/real(amr_ref_ratio, wp)) - oy
783 if (p_glb > 0) ck = lo3 + floor(real(l, wp)/real(amr_ref_ratio, wp)) - oz
784 ci = min(max(ci, cb1), ce1)
785 cj = min(max(cj, cb2), ce2)
786 ck = min(max(ck, cb3), ce3)
787 jac(j + o1, k + o2, l + o3) = sw_jac(ci, cj, ck)
788# 243 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
789 end do
790 end do
791 end do
792
793# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
794#if defined(MFC_OpenACC)
795# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
796!$acc end parallel loop
797# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
798#elif defined(MFC_OpenMP)
799# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
800
801# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
802!$omp end target teams loop
803# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
804#endif
805# 247 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
806# 250 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
807# 227 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
808# 228 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
809 if (igr_iter_solver == 1) then
810# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
811
812# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
813
814# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
815#if defined(MFC_OpenACC)
816# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
817!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l, ci, cj, ck) copyin(lo1, lo2, lo3, o1, o2, o3, mb1, me1, mb2, me2, mb3, me3)
818# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
819#elif defined(MFC_OpenMP)
820# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
821
822# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
823
824# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
825
826# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
827!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) &
828# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
829!$omp& private(j, k, l, ci, cj, ck) map(to:lo1, lo2, lo3, o1, o2, o3, mb1, me1, mb2, me2, mb3, me3)
830# 230 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
831#endif
832# 232 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
833 do l = mb3, me3
834 do k = mb2, me2
835 do j = mb1, me1
836 ci = lo1 + floor(real(j, wp)/real(amr_ref_ratio, wp)) - ox
837 cj = 0; ck = 0
838 if (n_glb > 0) cj = lo2 + floor(real(k, wp)/real(amr_ref_ratio, wp)) - oy
839 if (p_glb > 0) ck = lo3 + floor(real(l, wp)/real(amr_ref_ratio, wp)) - oz
840 ci = min(max(ci, cb1), ce1)
841 cj = min(max(cj, cb2), ce2)
842 ck = min(max(ck, cb3), ce3)
843 jac_old(j + o1, k + o2, l + o3) = sw_jac(ci, cj, ck)
844# 243 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
845 end do
846 end do
847 end do
848
849# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
850#if defined(MFC_OpenACC)
851# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
852!$acc end parallel loop
853# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
854#elif defined(MFC_OpenMP)
855# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
856
857# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
858!$omp end target teams loop
859# 246 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
860#endif
861# 247 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
862# 248 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
863 end if
864# 250 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
865# 251 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
866 end do
867
868 end subroutine s_amr_igr_swap_sigma
869
870 !> Restore the coarse jac/jac_old saved by s_amr_igr_swap_sigma (bounds already restored).
871 impure subroutine s_amr_igr_restore_sigma()
872
873 integer :: j, k, l, b1, e1, b2, e2, b3, e3
874
875 b1 = idwbuff(1)%beg; e1 = idwbuff(1)%end
876 b2 = idwbuff(2)%beg; e2 = idwbuff(2)%end
877 b3 = idwbuff(3)%beg; e3 = idwbuff(3)%end
878# 264 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
879# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
880
881# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
882
883# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
884#if defined(MFC_OpenACC)
885# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
886!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l)
887# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
888#elif defined(MFC_OpenMP)
889# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
890
891# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
892
893# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
894
895# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
896!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) private(j, k, l)
897# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
898#endif
899# 268 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
900 do l = b3, e3
901 do k = b2, e2
902 do j = b1, e1
903 jac(j, k, l) = sw_jac(j, k, l)
904# 272 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
905 end do
906 end do
907 end do
908
909# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
910#if defined(MFC_OpenACC)
911# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
912!$acc end parallel loop
913# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
914#elif defined(MFC_OpenMP)
915# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
916
917# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
918!$omp end target teams loop
919# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
920#endif
921# 276 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
922# 279 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
923# 264 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
924# 265 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
925 if (igr_iter_solver == 1) then
926# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
927
928# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
929
930# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
931#if defined(MFC_OpenACC)
932# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
933!$acc parallel loop collapse(3) gang vector default(present) private(j, k, l)
934# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
935#elif defined(MFC_OpenMP)
936# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
937
938# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
939
940# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
941
942# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
943!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) collapse(3) defaultmap(tofrom:aggregate) defaultmap(tofrom:allocatable) defaultmap(tofrom:pointer) private(j, k, l)
944# 267 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
945#endif
946# 268 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
947 do l = b3, e3
948 do k = b2, e2
949 do j = b1, e1
950 jac_old(j, k, l) = sw_jac_old(j, k, l)
951# 272 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
952 end do
953 end do
954 end do
955
956# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
957#if defined(MFC_OpenACC)
958# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
959!$acc end parallel loop
960# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
961#elif defined(MFC_OpenMP)
962# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
963
964# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
965!$omp end target teams loop
966# 275 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
967#endif
968# 276 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
969# 277 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
970 end if
971# 279 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
972# 280 "/home/runner/work/MFC/MFC/src/simulation/m_amr_frame.fpp"
973
974 end subroutine s_amr_igr_restore_sigma
975
976end module m_amr_frame
Rank decomposition, box arithmetic and block ownership (SFC cut, owner lookups, owned-block lists).
Parent/child data exchange: gather plans, pack/unpack and the fill waves.
Block-frame swap (fine grid state in/out of the shared solver).
impure subroutine, public s_amr_fine_enter()
Swap to the current fine block and load its solver frame amr_cons_br; paired with s_amr_fine_leave.
impure subroutine, public s_amr_swap_to_fine(widen)
Swap the global grid state to the fine block. Must be paired with s_amr_restore_coarse.
impure subroutine s_amr_igr_restore_sigma()
Restore the coarse jac/jac_old saved by s_amr_igr_swap_sigma (bounds already restored).
impure subroutine, public s_amr_restore_coarse(sync_device)
Restore the global grid state saved by s_amr_swap_to_fine.
impure subroutine s_amr_extend_ghost_coords(d, cb, cc, dd, ext, glb, gcb)
Extend one axis of the installed fine grid into the ghost shell with the same parent-cell bisection t...
impure subroutine, public s_amr_fine_leave()
Store the solver frame back to the fine block and restore the coarse grid.
impure subroutine s_amr_igr_swap_sigma()
Save the coarse jac/jac_old and seed the (already swapped-in) fine block's sigma by piecewise-constan...
Module-scope AMR state (block pool, flat field store, wave buffers) and the leaf helpers every AMR mo...
real(wp), dimension(:), allocatable sw_y_cc
real(wp), dimension(:), allocatable sw_z_cb
real(wp), dimension(:), allocatable sw_dx
impure subroutine s_amr_build_block_coords(k, gcb, glb, fcb, fcc, fdx, d)
Fine cell coordinates of block k in dimension d, rebuilt from the global coarse boundaries gcb by rep...
integer amr_swap_depth
Swap nesting depth: only the outermost swap saves the coarse state and only its restore puts it back;...
real(wp), dimension(:), allocatable amr_gycb
type(t_level), dimension(:), allocatable amr_slots
Fixed pool of refined-block slots; the working slot amr_cur selects which one per-block routines oper...
real(wp), dimension(:), allocatable sw_x_cb
integer sw_m
Swap contract (s_amr_swap_to_fine / s_amr_restore_coarse): the fine advance swaps these coarse global...
real(wp), dimension(:), allocatable sw_x_cc
real(wp), dimension(:), allocatable sw_y_cb
pure integer function f_amr_parent_block(k)
Index of the level-(level(k)-1) block that block k refines, or 0 for level 1 (parent is the L0 grid)....
logical sw_acoustic_source
real(wp), dimension(:), allocatable sw_dz
integer, dimension(:), allocatable amr_loc_of
Dense local index for live slots: amr_slots is sparse over 1:amr_max_blocks, the field store is dense...
real(wp), dimension(:,:,:), allocatable sw_jac_old
type(int_bounds_info), dimension(3) sw_idwbuff
real(wp), dimension(:), allocatable sw_dy
real(wp), dimension(:), allocatable amr_gxcb
Global coarse cell-boundary arrays (-1:X_glb), assembled once at init; the fine-distribution owner re...
real(wp), dimension(:,:,:), allocatable sw_jac
IGR sigma-state bounce: the fine solve reuses jac/jac_old at fine indices, so the coarse contents (ja...
real(wp), dimension(:), allocatable amr_gzcb
type(int_bounds_info), dimension(3) sw_idwint
real(wp), dimension(:), allocatable sw_z_cc
Flat per-block field store: dense slot indexing, reservation, alloc/free and the prim/bridge loaders.
impure subroutine, public s_amr_br_store(loc)
impure subroutine, public s_amr_sync_grid_state_to_device()
Push the host global grid state (m/n/p, idwint/idwbuff, coordinate arrays) to its GPU_DECLARE'd devic...
impure subroutine, public s_amr_br_load(loc)
Move block loc's conserved state between the flat store and the bridge over the whole buffered box (a...
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the computational domain, fluid properties, and simulation algorithm configurat...
integer, dimension(:,:), allocatable amr_isect_lo_all
type(int_bounds_info), dimension(1:3) idwint
real(wp), dimension(:), allocatable, target z_cb
integer, dimension(3) amr_sidx
integer, dimension(:,:), allocatable amr_region_lo_all
Per-slot mirrors (allocated 1:amr_max_blocks by m_amr): region box, rank intersection and ownership f...
integer, dimension(amr_bat_max) amr_bat_blk
integer, dimension(3) amr_isect_lo
The block's coarse footprint driving the coarse<->fine gather/scatter, per dim (collapsed dims 0:0): ...
integer, dimension(:,:), allocatable amr_region_hi_all
type(int_bounds_info), dimension(1:3) idwbuff
integer, dimension(:), allocatable amr_block_level
Refinement level of each active block (1..amr_max_level) and the deepest level populated....
integer buff_size
Number of ghost cells for boundary condition storage.
real(wp), dimension(:), allocatable, target y_cc
integer, dimension(3) amr_bat_ext
logical, dimension(3) amr_dim
The active dimensions, and this rank's coarse subdomain origin and extent padded to 3-vectors (0 in c...
real(wp), dimension(:), allocatable, target z_cc
integer, dimension(3, amr_bat_max) amr_bat_mext
real(wp), dimension(:), allocatable, target x_cc
real(wp), dimension(:), allocatable, target y_cb
real(wp), dimension(:), allocatable, target dy
integer, dimension(3, amr_bat_max) amr_bat_off
real(wp), dimension(:), allocatable, target dz
real(wp), dimension(:), allocatable, target dx
real(wp), dimension(:), allocatable, target x_cb
Computes hypoelastic stress-rate source terms and damage-state evolution.
impure subroutine, public s_hypoelastic_update_fd_coeffs()
(Re)compute the centered finite-difference coefficients from the current grid globals (m/n/p,...
Iterative ghost rasterization (IGR) for sharp immersed boundary treatment.
integer(kind=8) j
real(wp), dimension(:,:,:), allocatable, public jac_old
integer(kind=8) l
real(wp), dimension(:,:,:), allocatable, target, public jac
integer(kind=8) k
MPI halo exchange, domain decomposition, and buffer packing/unpacking for the simulation solver.