1# 1 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
6# 7 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
7# 8 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
8# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
9# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
10# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
11# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
15# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
17# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
19# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
21# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
22# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
25# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
26# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
27# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
29# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
30# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
32# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
34# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
36# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
38# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
40# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
42# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
44# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
46# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
48# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
49# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
50# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
51# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
55# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
57# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
59# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
61# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
62# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
65# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
66# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
67# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
69# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
70# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
72# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
74# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
76# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
78# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
80# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
82# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
84# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
86# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
88# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
90# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
94# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
96# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
98# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
100# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
102# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
104# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
106# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
108# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
110# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
112# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
114# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
116# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
118# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
120# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
122# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
124# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
126# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
128# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
130# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
132# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
134# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
136# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
138# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
140# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
141# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
143# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
145# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
147# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
149# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
151# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
153# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
155# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
157# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
159# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
161# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
163# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
165# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
167# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
168# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
169# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
170# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
174# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
176# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
178# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
180# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
181# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
184# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
185# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
186# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
188# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
189# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
191# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
193# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
195# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
197# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
199# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
201# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
203# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
205# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
207# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
209# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
211# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
213# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
215# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
217# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
219# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
221# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
223# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
225# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
227# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
229# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
231# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
233# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
235# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
237# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
239# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
241# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
243# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
245# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
247# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
249# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
251# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
253# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
255# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
257# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
259# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
261# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
263# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
265# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
268# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
271# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
274# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
277# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
280# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
283# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
286# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
289# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
292# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
295# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
298# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
301# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
304# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
307# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
310# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
313# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
316# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
319# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
322# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
325# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
327# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
328# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
330# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
332# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
337# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
340# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
343# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
346# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
349# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
356# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
359# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
363# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
365# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
367# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
369# 9 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp" 2
375 use mpi !< mpi-io for the parallel_io amr restart file
401 integer,
intent(in) :: r
402 integer,
intent(out) :: sidx(3), ext(3)
403 integer :: coords(3), gd(3), pd(3), base, rem, d
406 pd(1) = num_procs_x; pd(2) = num_procs_y; pd(3) = num_procs_z
408 coords(1) = r/(pd(2)*pd(3))
409 coords(2) = mod(r, pd(2)*pd(3))/pd(3)
410 coords(3) = mod(r, pd(3))
413 base = (gd(d) + 1)/pd(d)
414 rem = mod(gd(d) + 1, pd(d))
415 ext(d) = base - 1 + merge(1, 0, coords(d) < rem)
416 sidx(d) = coords(d)*base + min(coords(d), rem)
424 integer :: sidx(3), ext(3)
428 call s_mpi_abort(
's_amr_rank_decomp does not reproduce this rank''s decomposition - computed split disagrees with ' &
429 & //
's_mpi_decompose_computational_domain')
438 integer,
intent(in) :: g, base, rem
440 if (g < rem*(base + 1))
then
443 c = rem + (g - rem*(base + 1))/base
453 integer,
intent(in) :: blo(3), bhi(3)
454 integer,
intent(out) :: clo(3), chi(3)
455 integer :: gd(3), pd(3), base, rem, d
458 pd(1) = num_procs_x; pd(2) = num_procs_y; pd(3) = num_procs_z
461 base = (gd(d) + 1)/pd(d)
462 rem = mod(gd(d) + 1, pd(d))
473 integer,
intent(in) :: blo(3), bhi(3)
474 integer,
intent(out) :: ranks(:)
475 integer,
intent(out) :: nr
476 integer :: clo(3), chi(3), cx, cy, cz, pyz
479 pyz = num_procs_y*num_procs_z
481 do cx = clo(1), chi(1)
482 do cy = clo(2), chi(2)
483 do cz = clo(3), chi(3)
485 ranks(nr) = cx*pyz + cy*num_procs_z + cz
496 integer,
intent(in) :: blo(3), bhi(3), r
497 integer :: clo(3), chi(3), c(3)
500 c(1) = r/(num_procs_y*num_procs_z)
501 c(2) = mod(r/num_procs_z, num_procs_y)
502 c(3) = mod(r, num_procs_z)
503 hit = all(c >= clo) .and. all(c <= chi)
510 integer,
intent(in) :: blo(3), bhi(3)
511 integer :: clo(3), chi(3)
514 nr = (chi(1) - clo(1) + 1)*(chi(2) - clo(2) + 1)*(chi(3) - clo(3) + 1)
522 integer,
intent(in) :: r
523 integer,
intent(out) :: crlo(3), crhi(3)
524 integer :: sidx(3), ext(3)
535 integer,
intent(in) :: alo(3), ahi(3), blo(3), bhi(3)
536 integer,
intent(out) :: olo(3), ohi(3)
538 olo = max(alo, blo); ohi = min(ahi, bhi)
546 real(wp),
intent(out) :: cost(:)
547 integer ::
k,
j, kk,
l, lo(3), hi(3)
556# 194 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
557#if defined(MFC_OpenACC)
558# 194 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
560# 194 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
561#elif defined(MFC_OpenMP)
562# 194 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
564# 194 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
566# 195 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
586 call mpi_allreduce(mpi_in_place, cost,
amr_num_blocks, mpi_p, mpi_sum, mpi_comm_world, ierr)
597 integer,
intent(in) :: n
598 integer(kind=8),
intent(in) :: keys(n)
599 real(wp),
intent(in) :: wt(n)
600 integer(kind=8),
intent(out) :: cut(0:
num_procs - 1)
601 integer,
intent(out) :: owner(n)
602 integer :: ord(n),
k, r
603 real(wp) :: total, cum, tgt, tol
612 total = total + wt(
k)
618 tgt = real(r + 1, wp)*total/real(
num_procs, wp)
621 tol = spacing(tgt)*real(n, wp)
622 if (cum >= tgt - tol .and. r <
num_procs - 1) r = r + 1
624 cut(r) = keys(ord(
k))
625 cum = cum + wt(ord(
k))
629 if (cut(r) < cut(r - 1)) cut(r) = cut(r - 1)
638 integer,
intent(in) :: n
639 integer(kind=8),
intent(in) :: keys(n)
640 integer,
intent(out) :: ord(n)
641 integer :: mrg(n),
k, width, lo_m, mid_m, hi_m, i_m, j_m, t_m
649 do while (lo_m <= n - width)
650 mid_m = lo_m + width - 1
651 hi_m = min(lo_m + 2*width - 1, n)
652 i_m = lo_m; j_m = mid_m + 1; t_m = lo_m
653 do while (i_m <= mid_m .and. j_m <= hi_m)
655 if (keys(ord(i_m)) <= keys(ord(j_m)))
then
656 mrg(t_m) = ord(i_m); i_m = i_m + 1
658 mrg(t_m) = ord(j_m); j_m = j_m + 1
662 do while (i_m <= mid_m); mrg(t_m) = ord(i_m); i_m = i_m + 1; t_m = t_m + 1;
end do
663 do while (j_m <= hi_m); mrg(t_m) = ord(j_m); j_m = j_m + 1; t_m = t_m + 1;
end do
664 ord(lo_m:hi_m) = mrg(lo_m:hi_m)
665 lo_m = lo_m + 2*width
698 integer :: b, rlo(3), rhi(3), milo(3), mihi(3), bl(3), bh(3)
699 integer :: crlo(3), crhi(3), plo(3), phi(3), pl(3), ph(3), mar, pblk, acc, nxt
702# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
704# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
706# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
709# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
710# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
712# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
714# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
717# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
718# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
720# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
722# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
725# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
726# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
728# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
730# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
733# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
734# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
736# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
738# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
741# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
742# 331 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
744# 332 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
746# 333 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
749# 335 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
750# 336 "/home/runner/work/MFC/MFC/src/simulation/m_amr_distribution.fpp"
786 if (all(bl <= bh))
then
792 if (all(pl <= ph))
then
819 integer,
intent(in) ::
k, g
820 real(wp),
intent(inout) :: vol(0:)
821 integer,
intent(out) :: rk(:), nr
822 integer :: lo(3), hi(3), ilo(3), ihi(3), ol(3), oh(3),
j, q
831 vol(rk(
j)) = product(real(max(oh - ol + 1, 0), wp))
838 if (any(ol > oh)) cycle
840 if (vol(q) <= 0._wp)
then
841 nr = nr + 1; rk(nr) = q
843 vol(q) = vol(q) + product(real(oh - ol + 1, wp))
854 integer,
intent(in) :: na, aidx(na)
855 integer(kind=8),
intent(in) :: key(na)
856 real(wp),
intent(in) :: wt(na)
857 integer,
intent(inout) :: own(na)
858 real(wp),
parameter :: place_tol = 0.05_wp
859 real(wp),
allocatable :: vol(:), load(:)
860 integer,
allocatable :: rk(:), ord(:)
861 integer :: a, b,
j, r, m, nr, pass
865 vol = 0._wp; load = 0._wp
869 if (
j == 1) own(a) = rk(1)
870 if (vol(rk(
j)) > vol(own(a)) .or. (vol(rk(
j)) == vol(own(a)) .and. rk(
j) < own(a))) own(a) = rk(
j)
872 vol(rk(1:nr)) = 0._wp
873 load(own(a)) = load(own(a)) + wt(a)
876 cap = (1._wp + place_tol)*sum(load)/real(
num_procs, wp)
881 if (load(r) <= cap)
exit
883 if (own(a) /= r) cycle
886 vol(rk(1:nr)) = 0._wp
889 if (rk(
j) == r) cycle
892 else if (load(rk(
j)) < load(m) .or. (load(rk(
j)) == load(m) .and. rk(
j) < m))
then
897 m = minloc(load, 1) - 1
900 if (load(m) + wt(a) > cap) cycle
901 own(a) = m; load(r) = load(r) - wt(a); load(m) = load(m) + wt(a)
905 deallocate (vol, load, rk, ord)
911 integer ::
k, a, lev, maxlev, na
913 integer,
allocatable :: aidx(:), aown(:)
914 integer(kind=8),
allocatable :: key(:), akey(:)
915 real(wp),
allocatable :: wt(:), cost(:), awt(:)
941 akey(na) = key(
k); awt(na) = wt(
k); aidx(na) =
k
954 deallocate (aidx, aown, key, akey, wt, cost, awt)
963 integer,
intent(in) :: own(:)
964 integer ::
k, r, lev, maxlev
965 integer(kind=8) :: key
988 real(wp),
intent(in) :: wt(:)
989 integer,
intent(in) :: maxlev
990 real(wp),
allocatable :: rw(:),
tw(:), rc(:)
991 real(wp) :: mx, mean, cmx, cmean
992 integer ::
k, lev,
nb, empty
994 if (.not. rank_time_wrt .or.
proc_rank /= 0)
return
1000 rw = 0._wp; rc = 0._wp
1010 mx = maxval(rw); mean = sum(rw)/real(
num_procs, wp)
1011 empty = count(rw <= 0._wp)
1013 cmx = maxval(rc); cmean = sum(rc)/real(
num_procs, wp)
1014 if (mean > 0._wp) print
'(A,I0,A,I0,A,I0,A,F8.3,A,F8.3,A,I0,A,I0)',
' [amr-balance] level ', lev,
': boxes ',
nb, &
1015 &
'/ranks ',
num_procs,
' max/mean ', mx/mean,
' boxes_max/mean ', cmx/cmean,
' no_blocks_ranks ', empty,
' of ', &
1019 write (*,
'(A)', advance=
'no')
' [amr-balance] per-rank fine_work :'
1020 do k = 0,
num_procs - 1;
write (*,
'(I12)', advance=
'no') nint(
tw(
k), kind=8);
end do
1024 if (mean > 0._wp) print
'(A,F8.3,A,I0,A,I0)',
' [amr-balance] TOTAL : max/mean ', maxval(
tw)/mean, &
1025 &
' ranks_with_no_fine_block ', count(
tw <= 0._wp),
' fine_work ', nint(sum(
tw), kind=8)
1026 deallocate (rw,
tw, rc)
1034 integer,
intent(in) ::
k
1035 integer(kind=8) :: mk, cut(0:
num_procs - 1)
1036 integer :: lo, hi, mid
1043 if (mk <= cut(mid))
then
1059 if (amr_place_owner) cycle
1061 &
call s_mpi_abort(
'SFC cut-point owner disagrees with amr_block_owner - cut capture or search is wrong')
1070 integer,
intent(in) :: lo(3), hi(3)
1081 integer,
intent(in) :: r
1082 integer,
intent(out) :: ilo(3), ihi(3)
1083 integer :: sidx(3), ext(3)
1086 ilo = merge(sidx, 0,
amr_dim); ihi = merge(sidx + ext, 0,
amr_dim)
1094 integer,
intent(in) :: i, mrg
1095 integer,
intent(out) :: blo(3), bhi(3)
1096 real(wp) :: c(3), half(3)
1098 c = [patch_ib(i)%x_centroid, patch_ib(i)%y_centroid, patch_ib(i)%z_centroid]
1099 select case (patch_ib(i)%geometry)
1101 half = patch_ib(i)%radius
1103 if (patch_ib(i)%geometry == 10)
then
1104 half = [0.5_wp*patch_ib(i)%length_x, patch_ib(i)%radius, patch_ib(i)%radius]
1106 half = 0.5_wp*[patch_ib(i)%length_x, patch_ib(i)%length_y, patch_ib(i)%length_z]
1108 half = matmul(abs(patch_ib(i)%rotation_matrix), half)
1110 call s_mpi_abort(
'amr dynamic regrid with ib: unsupported body geometry for the ' &
1111 & //
'containment bounding box (supported: circle/rectangle/sphere/box/cylinder)')
1121 integer,
intent(inout) :: lo(3), hi(3)
1122 integer :: i, blo(3), bhi(3), mrg
1127 mrg = max(amr_buf, 4) + max(0, amr_max_level - 1)*
amr_cpat_mar
1133 call s_mpi_abort(
'amr dynamic regrid with ib: the immersed body plus its containment ' &
1134 & //
'margin does not fit inside the refinable domain interior (blocks stay buff_size off the edges)')
1137 lo = min(lo, blo); hi = max(hi, bhi)
1139 call s_mpi_abort(
'amr dynamic regrid with ib: containing the immersed body plus margin ' &
1140 & //
'exceeds the per-rank block size cap; use fewer ranks or a larger amr_maxc_fit')
1150 integer,
intent(in) :: lo(3), hi(3), cap
1151 type(
t_box),
intent(inout) :: out(:)
1152 integer,
intent(inout) :: nt, capped
1154 integer,
intent(in),
optional :: tsz(3)
1155 integer :: ntl(3), s(3), t1, t2, t3, qlo(3), qhi(3), tc(3)
1159 ntl = (hi - lo + tc)/tc; s = (hi - lo + ntl)/ntl
1160 do t3 = 0, ntl(3) - 1
1161 qlo(3) = lo(3) + t3*s(3); qhi(3) = min(lo(3) + (t3 + 1)*s(3) - 1, hi(3))
1162 do t2 = 0, ntl(2) - 1
1163 qlo(2) = lo(2) + t2*s(2); qhi(2) = min(lo(2) + (t2 + 1)*s(2) - 1, hi(2))
1164 do t1 = 0, ntl(1) - 1
1165 if (nt >= cap) then; capped = 1; return;
end if
1166 qlo(1) = lo(1) + t1*s(1); qhi(1) = min(lo(1) + (t1 + 1)*s(1) - 1, hi(1))
1167 nt = nt + 1; out(nt)%lo = qlo; out(nt)%hi = qhi
Rank decomposition, box arithmetic and block ownership (SFC cut, owner lookups, owned-block lists).
impure subroutine, public s_amr_refresh_my_blocks()
Refresh the owned-block list if any owner write has happened since the last refresh....
pure integer function, public f_amr_overlap_count(blo, bhi)
Overlap count only (allocation sizing), = product of the per-dim coord-range widths.
impure subroutine, public s_amr_body_bbox(i, mrg, blo, bhi)
Margin-padded global coarse-index bounding box of immersed body i (supported analytic geometries only...
pure integer function f_amr_owner(k)
Owner rank of block k by binary search of its Morton key in its level's cut amr_fine_cut(:,...
impure subroutine s_amr_block_cost(cost)
Per-block cost over each block's level-0 footprint (1 per cell + K_ib per IB-marked cell),...
pure subroutine s_amr_coord_range(blo, bhi, clo, chi)
Per-dim contiguous rank-coord range [clo:chi] whose owned coarse slab meets box [blo:bhi],...
subroutine, public s_amr_sfc_cut(keys, wt, n, cut, owner)
Cost-weighted SFC partition of n items into num_procs contiguous Morton-key ranges: walk the key-sort...
impure subroutine, public s_amr_adopt_block_owners(own)
Install a recorded owner table instead of recomputing it (restart at the writer's rank count): the co...
impure subroutine s_amr_place_overlap(k, g, vol, rk, nr)
Coarse-side overlap per rank of block k's region grown by g coarse cells: level 1 against the level-0...
pure subroutine, public s_amr_tile_box(lo, hi, out, nt, cap, capped, tsz)
Split box [lo:hi] into an even grid of adjacent sub-boxes each <= amr_maxc_fit per dim (the max a ran...
pure subroutine, public s_amr_sort_by_key(keys, n, ord)
ord(1:n) = item indices in ascending key order. Stable iterative bottom-up merge sort,...
impure subroutine, public s_amr_compute_isect(lo, hi)
Set this rank's intersection of box lo:hi with its subdomain (amr_isect_lo/hi) and whether it holds f...
impure subroutine, public s_amr_refresh_lists()
Rebuild the epoch-keyed block lists when the mesh epoch has moved (one O(global blocks) walk per regr...
impure subroutine, public s_amr_validate_decomp()
Guard: the computed decomposition must reproduce this rank's actual one (each rank checks itself,...
pure subroutine, public s_amr_rank_decomp(r, sidx, ext)
Rank r's coarse-grid decomposition (start_idx + local extent m/n/p), computed O(1) from its cartesian...
impure subroutine, public s_amr_assign_block_owners()
impure subroutine s_amr_place_owner(key, wt, aidx, na, own)
Owner-aware placement (amr_place_owner) of one level's na blocks (slots aidx, weights wt,...
pure subroutine, public s_amr_rank_interior(r, ilo, ihi)
Rank r's coarse interior box (global, no ghosts) from s_amr_rank_decomp; restriction targets are iden...
pure logical function, public f_amr_rank_overlaps(blo, bhi, r)
Does rank r's subdomain overlap coarse box [blo:bhi]? Answers membership without enumerating the over...
impure subroutine, public s_amr_validate_owner()
Guard: the SFC cut-point accessor must reproduce the stored owner table exactly.
pure subroutine, public s_amr_ranks_overlapping(blo, bhi, ranks, nr)
Ascending rank list overlapping coarse box [blo:bhi]: the coord brick is enumerated cx->cy->cz so r =...
pure subroutine, public s_amr_rank_coarse_range(r, crlo, crhi)
Rank r's contiguous owned coarse-cell range per dim: the interior plus buff_size ghosts where the sub...
pure subroutine, public s_amr_box_isect(alo, ahi, blo, bhi, olo, ohi)
Per-dim intersection of two global boxes [alo:ahi] and [blo:bhi] -> [olo:ohi] (empty when olo > ohi i...
pure integer function f_amr_cell_coord(g, base, rem)
Closed-form inverse of the per-dim split in s_amr_rank_decomp: the cart coord owning global coarse ce...
impure subroutine s_amr_report_balance(wt, maxlev)
Per-level and total load-balance report (rank 0, rank_time_wrt): max/mean assigned block weight over ...
impure subroutine, public s_amr_expand_box_over_bodies(lo, hi)
Expand a candidate regrid box (global indices) to fully contain every immersed body it overlaps,...
Module-scope AMR state (block pool, flat field store, wave buffers) and the leaf helpers every AMR mo...
integer amr_cpat_mar
coarse-cell stencil reach (f_amr_cpat_mar)
logical amr_myblk_dirty
Set wherever amr_block_owner is written (assigner, inheritance, restart/migration),...
pure subroutine s_amr_phys_to_cells(pmin, pmax, pad_cells, blo, bhi)
Convert a physical-space bbox to a global coarse-index bbox padded by pad_cells (uniform spacing; the...
integer amr_l1r_nblk
second key half, matching s_amr_reg_prepare's (epoch, num_blocks) pair
integer, dimension(:), allocatable amr_child_ptr
children of p = amr_child_idx(amr_child_ptr(p-1)+1 : amr_child_ptr(p)), ascending
pure subroutine s_amr_patch_box(k, plo, phi)
Block k's padded coarse patch box [plo, phi] (region +/- amr_cpat_mar; collapsed dims 0:0).
integer, dimension(:), allocatable amr_child_idx
(amr_child_ptr(0) = 0)
integer(kind=8), dimension(:,:), allocatable amr_fine_cut
(0:num_procs-1, 1:amr_max_level) SFC Morton-key upper bound per rank, one independent cost-weighted c...
integer, dimension(:), allocatable amr_fch_blk
foreign children of my owned parents (level >= 2, powner == me, cowner /= me)
integer, dimension(:), allocatable amr_parent_blk
cached f_amr_parent_block (0 for level <= 1), avoiding an O(global blocks) scan per block per stage
pure integer function f_amr_parent_block(k)
Index of the level-(level(k)-1) block that block k refines, or 0 for level 1 (parent is the L0 grid)....
pure subroutine s_amr_region_box(k, rlo, rhi)
Block k's region box in global coarse cells; collapsed dimensions read 0:0.
pure logical function f_amr_boxes_overlap(alo, ahi, blo, bhi)
Do two coarse-index boxes [alo:ahi] and [blo:bhi] overlap? Collapsed dims never disqualify.
integer, dimension(:), allocatable amr_own_blk
blocks this rank intersects (amr_owns_all, the multi-owner notion, all levels)
integer, dimension(:), allocatable amr_l1r_blk
Level-1 blocks this rank receives restriction for (not its own, overlapping its interior),...
integer, dimension(:), allocatable amr_l1p_blk
padded variant (region +/- amr_cpat_mar vs my coarse range): stage-fill gather
integer, dimension(:), allocatable amr_my_blk
Block indices this rank owns, ascending (the paired MPI_SENDRECVs depend on the order); built in s_am...
integer, dimension(3) amr_maxc_fit
Regrid box size cap per dim (fixed for the run, identical on all ranks; 1 in collapsed dims): a box o...
Box type, per-rank partition arithmetic (one box per rank) and the Morton key.
pure integer(kind=8) function, public f_morton(ix, iy, iz)
3D Morton (Z-order) key of (ix, iy, iz), 21 bits/dim; negative coords clamp to 0.
Compile-time constant parameters: default values, tolerances, and physical constants.
real(wp), parameter k_ib
AMR block-owner cost per IB cell, relative to a fluid cell.
Shared derived types for field data, patch geometry, bubble dynamics, and MPI I/O structures.
Global parameters for the computational domain, fluid properties, and simulation algorithm configurat...
integer, dimension(3) amr_sidx
integer, dimension(:,:), allocatable amr_region_lo_all
Per-slot mirrors (allocated 1:amr_max_blocks by m_amr): region box, rank intersection and ownership f...
integer proc_rank
Rank of the local processor.
integer(8) amr_mesh_epoch
Monotone mesh epoch, incremented wherever amr_seam_pairs_dirty is set and at the end of each slot rec...
integer, dimension(3) amr_isect_lo
The block's coarse footprint driving the coarse<->fine gather/scatter, per dim (collapsed dims 0:0): ...
integer, dimension(:,:), allocatable amr_region_hi_all
logical, dimension(:), allocatable amr_owns_all
integer, dimension(:), allocatable amr_block_level
Refinement level of each active block (1..amr_max_level) and the deepest level populated....
integer buff_size
Number of ghost cells for boundary condition storage.
logical, dimension(3) amr_dim
The active dimensions, and this rank's coarse subdomain origin and extent padded to 3-vectors (0 in c...
integer, dimension(3) amr_isect_hi
integer num_procs
Number of processors.
integer, dimension(:), allocatable amr_block_owner
Fine-level distribution map: SFC/work-balanced single-owner rank per active block,...
integer amr_num_blocks
Number of active AMR fine-block slots (>= 1) and the working slot index selecting which one the per-b...
integer, dimension(3) amr_ext
logical amr_rank_owns_block
true on the current block's single owner rank: amr_block_owner(amr_cur) == proc_rank; kept by s_set_a...
Ghost-node immersed boundary method: locates ghost/image points, computes interpolation coefficients,...
type(integer_field), public ib_markers
MPI halo exchange, domain decomposition, and buffer packing/unpacking for the simulation solver.
An index-space rectangle in global cell indices. In v1, one t_box = one rank's subdomain....