MFC
Exascale flow solver
Loading...
Searching...
No Matches
m_phase_timing.fpp.f90
Go to the documentation of this file.
1# 1 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
2!>
3!!@file
4!!@brief Contains module m_phase_timing
5
6# 1 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 1
7# 1 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 1
8# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
9# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
10# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
11# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
12# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
13# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
14
15# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
16# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
17# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
18
19# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
20# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
21! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
22! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
23# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
24# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
25# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
26
27# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
28# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
29
30# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
31
32# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
33
34# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
35
36# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
37
38# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
39
40# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
41
42# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
43
44# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
45! New line at end of file is required for FYPP
46# 2 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
47# 1 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 1
48# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
49# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
50# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
51# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
52# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
53# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
54
55# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
56# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
57# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
58
59# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
60# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
61! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
62! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
63# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
64# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
65# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
66
67# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
68# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
69
70# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
71
72# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
73
74# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
75
76# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
77
78# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
79
80# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
81
82# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
83
84# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
85! New line at end of file is required for FYPP
86# 2 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp" 2
87
88# 4 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
89# 5 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
90# 6 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
91# 7 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
92# 8 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
93
94# 20 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
95
96# 49 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
97
98# 54 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
99
100# 59 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
101
102# 64 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
103
104# 69 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
105
106# 74 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
107
108# 87 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
109
110# 101 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
111
112# 106 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
113
114# 111 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
115
116# 116 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
117
118# 121 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
119
120# 126 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
121
122# 131 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
123
124# 136 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
125
126# 141 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
127
128# 171 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
129
130# 231 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
131
132# 245 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
133
134# 270 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
135
136# 281 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
137
138# 283 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
139# 294 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
140
141# 343 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
142
143# 353 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
144
145# 363 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
146
147# 372 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
148
149# 389 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
150
151# 399 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
152
153# 406 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
154
155# 412 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
156
157# 418 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
158
159# 424 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
160
161# 430 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
162
163# 436 "/home/runner/work/MFC/MFC/src/common/include/omp_macros.fpp"
164! New line at end of file is required for FYPP
165# 3 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
166# 1 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 1
167# 1 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 1
168# 2 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
169# 3 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
170# 4 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
171# 5 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
172# 6 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
173
174# 8 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
175# 9 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
176# 10 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
177
178# 12 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
179# 1 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp" 1
180! Default for Fypp runs outside the toolchain (e.g. documentation). Searched last, so the
181! build/include/<target>/thermochem.fpp the toolchain writes from m_thermochem takes precedence.
182# 4 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
183# 5 "/home/runner/work/MFC/MFC/src/common/include/defaults/thermochem.fpp"
184# 13 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp" 2
185
186# 16 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
187# 17 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
188
189# 24 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
190
191# 53 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
192
193# 65 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
194
195# 75 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
196
197# 105 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
198
199# 117 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
200
201# 127 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
202
203# 174 "/home/runner/work/MFC/MFC/src/common/include/shared_parallel_macros.fpp"
204! New line at end of file is required for FYPP
205# 2 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp" 2
206
207# 7 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
208
209# 17 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
210
211# 22 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
212
213# 27 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
214
215# 32 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
216
217# 37 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
218
219# 42 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
220
221# 47 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
222
223# 52 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
224
225# 57 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
226
227# 62 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
228
229# 73 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
230
231# 78 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
232
233# 83 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
234
235# 88 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
236
237# 103 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
238
239# 131 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
240
241# 161 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
242
243# 176 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
244
245# 194 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
246
247# 216 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
248
249# 245 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
250
251# 260 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
252
253# 270 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
254
255# 279 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
256
257# 295 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
258
259# 305 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
260
261# 312 "/home/runner/work/MFC/MFC/src/common/include/acc_macros.fpp"
262! New line at end of file is required for FYPP
263# 4 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp" 2
264
265! GPU parallel region (scalar reductions, maxval/minval)
266# 23 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
267
268! GPU parallel loop over threads (most common GPU macro)
269# 44 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
270
271! Required closing for GPU_PARALLEL_LOOP
272# 56 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
273
274! Mark routine for device compilation
275# 113 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
276
277! Declare device-resident data
278# 131 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
279
280! Inner loop within a GPU parallel region
281# 146 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
282
283! Scoped GPU data region
284# 165 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
285
286! Host code with device pointers (for MPI with GPU buffers)
287# 191 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
288
289! Allocate device memory (unscoped)
290# 205 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
291
292! Free device memory
293# 217 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
294
295! Atomic operation on device
296# 229 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
297
298! End atomic capture block
299# 240 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
300
301! Copy data between host and device
302# 252 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
303
304! Synchronization barrier
305# 264 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
306
307! Import GPU library module (openacc or omp_lib)
308# 273 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
309
310! Emit code only for AMD compiler
311# 280 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
312
313! Emit code for non-Cray compilers
314# 287 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
315
316! Emit code only for Cray compiler
317# 294 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
318
319! Cray compiler directive, emitted from Fypp so the formatter cannot turn it into a comment
320# 301 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
321
322! Emit code for non-NVIDIA compilers
323# 308 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
324
325# 310 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
326# 311 "/home/runner/work/MFC/MFC/src/common/include/parallel_macros.fpp"
327! New line at end of file is required for FYPP
328# 2 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp" 2
329
330# 14 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
331
332! Caution: This macro requires the use of a binding script to set CUDA_VISIBLE_DEVICES, such that we have one GPU device per MPI
333! rank. That's because for both cudaMemAdvise (preferred location) and cudaMemPrefetchAsync we use location = device_id = 0. For an
334! example see misc/nvidia_uvm/bind.sh.
335# 52 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
336
337! Allocate and create GPU device memory
338# 72 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
339
340! Free GPU device memory and deallocate
341# 80 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
342
343! Cray-specific GPU pointer setup for vector fields
344# 104 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
345
346! Cray-specific GPU pointer setup for scalar fields
347# 120 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
348
349! Cray-specific GPU pointer teardown for scalar fields - exact inverse of ACC_SETUP_SFs. Removes the
350! present-table entries ACC_SETUP_SFs added (the scalar_field descriptor and its %sf copyin). REQUIRED when
351! an ACC_SETUP'd field is freed MID-RUN (e.g. AMR slot free on regrid/restart): Cray 'exit data delete'
352! decrements the reference counter, so the lone @:DEALLOCATE(arg%sf) only undoes the @:ALLOCATE create ref
353! and leaves the descriptor + the ACC_SETUP %sf copyin dangling. Call BEFORE @:DEALLOCATE(arg%sf).
354# 140 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
355
356! Cray-specific GPU pointer setup for acoustic source spatials
357# 165 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
358
359! Whether stress index idx is a shear component. Spelled out because NVHPC 26.9 at -O2 vectorizes any(idx == shear_indices)
360! inside a loop to always true, which doubled the normal-stress elastic energy.
361# 171 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
362
363# 177 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
364
365# 184 "/home/runner/work/MFC/MFC/src/common/include/macros.fpp"
366! New line at end of file is required for FYPP
367# 6 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp" 2
368
369!> @brief Per-phase wall-time budget of the AMR step, reported when rank_time_wrt is on. Brackets are disjoint (regrid sub-phases
370!! nest inside regrid) and drain the device first, so a phase's time includes the GPU work it launched.
372
374 use m_mpi_common
375
376 implicit none
377
378 private
383
384 integer, parameter :: ph_halo = 1 !< coarse cons halo exchange (once per stage)
385 integer, parameter :: ph_gather = 2 !< fill waves: coarse-patch gather into every block
386 integer, parameter :: ph_gfill = 3 !< ghost prolongation from the gathered patch
387 integer, parameter :: ph_seam = 4 !< fine-fine seam halo
388 integer, parameter :: ph_rhs = 5 !< fine-block RHS
389 integer, parameter :: ph_rk = 6 !< fine-block RK update + IB
390 integer, parameter :: ph_reflux = 7 !< level-1 reflux (faces wave + apply)
391 integer, parameter :: ph_regrid = 8 !< regrid
392 integer, parameter :: ph_coarse = 9 !< coarse (base-grid) solver work
393 integer, parameter :: ph_swap = 10 !< per-batch grid-state swap and restore
394 integer, parameter :: ph_restr = 11 !< post-stage restrict + reflux-to-parent fold
395 integer, parameter :: ph_lerp = 12 !< subcycled coarse time interpolation for the level-1 fills
396 integer, parameter :: ph_ntop = 12 !< the disjoint brackets above; the regrid sub-phases below nest in PH_REGRID
397 integer, parameter :: ph_rg_halo = 13 !< coarse halo + host copy for tagging
398 integer, parameter :: ph_rg_tag = 14 !< tagging
399 integer, parameter :: ph_rg_clus = 15 !< clustering, shaping, nesting and box checks
400 integer, parameter :: ph_rg_own = 16 !< stash and owner assignment
401 integer, parameter :: ph_rg_mig = 17 !< fine-state migration
402 integer, parameter :: ph_rg_build = 18 !< new-block build (fills, prolongation, overlap copy)
403 integer, parameter :: ph_rg_rec = 19 !< slot reconcile, IB setup, seam check
404 integer, parameter :: ph_n = 19
405 character(len=8), parameter :: ph_name(ph_n) = [character(len=8)::'halo','gather', 'gfill', 'seam', 'rhs', 'rk', 'reflux', &
406 & 'regrid', 'coarse', 'swap', 'restr', 'lerp', ' rg.halo', ' rg.tag', ' rg.clus', ' rg.own', ' rg.mig', &
407 & ' rg.build', ' rg.rec']
408
409 real(wp) :: acc(ph_n) = 0._wp, wall = 0._wp !< wall is the accumulated step-loop time the phases are budgeted against
410 integer(8) :: ncall(ph_n) = 0, tic_c(ph_n) = 0, wall_c = 0
411 integer :: depth(ph_n) = 0 !< a re-entered bracket (shared routine) measures only its outermost pair
412
413 !> AMR allocations by group, noted by each owner before it frees them (s_mem_note) and printed with the phase table.
414 integer, parameter :: mem_store = 1 !< flat block store (cons + RK stage copy)
415 integer, parameter :: mem_scratch = 2 !< batched-advance scratch (prim, rhs) and the block bridge
416 integer, parameter :: mem_patch = 3 !< coarse patch pool and its subcycling time cache
417 integer, parameter :: mem_reg = 4 !< flux registers
418 integer, parameter :: mem_ib = 5 !< fine IB ghost-point park
419 integer, parameter :: mem_ibhost = 6 !< per-slot fine IB marker copies (host only)
420 integer, parameter :: mem_n = 6
421 character(len=8), parameter :: mem_name(mem_n + 1) = [character(len=8)::'store','scratch', 'patch', 'register', 'ib park', &
422 & 'ib marks', 'total']
423 real(wp) :: mem_mib(mem_n) = 0._wp
424
425contains
426
427 impure subroutine s_phase_tic(id)
428
429 integer, intent(in) :: id
430
431 if (.not. rank_time_wrt) return
432 depth(id) = depth(id) + 1
433 if (depth(id) > 1) return
434
435# 72 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
436#if defined(MFC_OpenACC)
437# 72 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
438!$acc wait
439# 72 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
440#elif defined(MFC_OpenMP)
441# 72 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
442!$omp barrier
443# 72 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
444#endif
445# 73 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
446 call system_clock(tic_c(id))
447
448 end subroutine s_phase_tic
449
450 impure subroutine s_phase_toc(id)
451
452 integer, intent(in) :: id
453 integer(8) :: c, rate
454
455 if (.not. rank_time_wrt) return
456 depth(id) = depth(id) - 1
457 if (depth(id) > 0) return
458
459# 85 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
460#if defined(MFC_OpenACC)
461# 85 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
462!$acc wait
463# 85 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
464#elif defined(MFC_OpenMP)
465# 85 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
466!$omp barrier
467# 85 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
468#endif
469# 86 "/home/runner/work/MFC/MFC/src/simulation/m_phase_timing.fpp"
470 call system_clock(c, rate)
471 acc(id) = acc(id) + real(c - tic_c(id), wp)/real(rate, wp)
472 ncall(id) = ncall(id) + 1
473
474 end subroutine s_phase_toc
475
476 impure subroutine s_phase_step_tic()
477
478 if (rank_time_wrt) call system_clock(wall_c)
479
480 end subroutine s_phase_step_tic
481
482 impure subroutine s_phase_step_toc()
483
484 integer(8) :: c, rate
485
486 if (.not. rank_time_wrt) return
487 call system_clock(c, rate)
488 wall = wall + real(c - wall_c, wp)/real(rate, wp)
489
490 end subroutine s_phase_step_toc
491
492 !> Mean and max over ranks per phase, share of the step-loop wall, imbalance (max/mean) and calls; the residual is the wall
493 !! minus the sum of the means.
494 impure subroutine s_phase_report()
495
496 real(wp) :: gmax(ph_n), gsum(ph_n), mean, mmax(mem_n + 1), msum(mem_n + 1)
497 integer(8) :: gcall(ph_n)
498 integer :: i, ierr
499
500 if (.not. rank_time_wrt) return
501#ifdef MFC_MPI
502 gmax = acc
504 call mpi_allreduce(acc, gsum, ph_n, mpi_p, mpi_sum, mpi_comm_world, ierr)
505 call mpi_allreduce(ncall, gcall, ph_n, mpi_integer8, mpi_sum, mpi_comm_world, ierr)
506 mmax = [mem_mib, sum(mem_mib)] ! last entry: the rank's total
507 call mpi_allreduce(mmax, msum, mem_n + 1, mpi_p, mpi_sum, mpi_comm_world, ierr)
508 call s_mpi_allreduce_array_max(mmax, mem_n + 1)
509#else
510 gmax = acc; gsum = acc; gcall = ncall; mmax = [mem_mib, sum(mem_mib)]; msum = mmax
511#endif
512 if (proc_rank /= 0) return
513 print '(A,F10.3,A)', '[phase] step-loop wall = ', wall, ' s'
514 print '(A)', '[phase] name mean s max s % wall imbalance calls/rank ms/call'
515 do i = 1, ph_n
516 if (gsum(i) <= 0._wp) cycle
517 mean = gsum(i)/real(num_procs, wp)
518 print '(A,A8,F10.3,F9.3,F9.1,A,F8.3,I12,F11.4)', '[phase] ', ph_name(i), mean, gmax(i), 100._wp*mean/wall, '%', &
519 & gmax(i)/mean, gcall(i)/int(num_procs, 8), 1000._wp*mean/max(real(gcall(i)/int(num_procs, 8), wp), 1._wp)
520 end do
521 print '(A,F10.3,F19.1,A)', '[phase] RESIDUAL', wall - sum(gsum(1:ph_ntop))/real(num_procs, wp), &
522 & 100._wp*(wall - sum(gsum(1:ph_ntop))/real(num_procs, wp))/wall, '%'
523 if (msum(mem_n + 1) <= 0._wp) return
524 print '(A)', '[amr-mem] group mean MiB max MiB (per rank, at the end of the run)'
525 do i = 1, mem_n + 1
526 print '(A,A8,2F12.1)', '[amr-mem] ', mem_name(i), msum(i)/real(num_procs, wp), mmax(i)
527 end do
528 print '(A,F8.2,A)', '[amr-mem] solver work arrays are allocated ', &
529 & product(real(idwbuff_alloc%end - idwbuff_alloc%beg + 1, wp))/product(real(idwbuff%end - idwbuff%beg + 1, wp)), &
530 & 'x the coarse extent (largest block or batch slab)'
531
532 end subroutine s_phase_report
533
534 !> Add bytes to group id of the end-of-run AMR memory table.
535 impure subroutine s_mem_note(id, bytes)
536
537 integer, intent(in) :: id
538 integer(kind=8), intent(in) :: bytes
539
540 mem_mib(id) = mem_mib(id) + real(bytes, wp)/1024._wp**2
541
542 end subroutine s_mem_note
543
544 !> s_mem_note for the fields of f that are allocated.
545 impure subroutine s_mem_note_fields(id, f)
546
547 integer, intent(in) :: id
548 type(scalar_field), intent(in) :: f(:)
549 integer :: i
550
551 do i = 1, size(f)
552 if (associated(f(i)%sf)) call s_mem_note(id, storage_size(f(i)%sf, kind=8)/8_8*size(f(i)%sf, kind=8))
553 end do
554
555 end subroutine s_mem_note_fields
556
557end module m_phase_timing
Global parameters for the computational domain, fluid properties, and simulation algorithm configurat...
integer proc_rank
Rank of the local processor.
type(int_bounds_info), dimension(1:3) idwbuff_alloc
ALLOCATION bounds for the solver working set, as distinct from the RUNTIME bounds in idwbuff: the AMR...
type(int_bounds_info), dimension(1:3) idwbuff
integer num_procs
Number of processors.
MPI communication layer: domain decomposition, halo exchange, reductions, and parallel I/O setup.
impure subroutine s_mpi_allreduce_array_max(arr, arr_len)
In-place elementwise max-reduction of a real array across all ranks: each element is written (identic...
Per-phase wall-time budget of the AMR step, reported when rank_time_wrt is on. Brackets are disjoint ...
integer, parameter, public ph_regrid
regrid
integer, parameter, public ph_rk
fine-block RK update + IB
real(wp) wall
wall is the accumulated step-loop time the phases are budgeted against
impure subroutine, public s_phase_step_toc()
integer, parameter, public ph_rg_rec
slot reconcile, IB setup, seam check
character(len=8), dimension(mem_n+1), parameter mem_name
integer, parameter, public ph_gather
fill waves: coarse-patch gather into every block
impure subroutine, public s_phase_step_tic()
impure subroutine, public s_mem_note(id, bytes)
Add bytes to group id of the end-of-run AMR memory table.
integer, parameter, public ph_halo
coarse cons halo exchange (once per stage)
integer, parameter, public ph_gfill
ghost prolongation from the gathered patch
integer, parameter, public ph_lerp
subcycled coarse time interpolation for the level-1 fills
integer, parameter, public ph_rg_mig
fine-state migration
impure subroutine, public s_phase_report()
Mean and max over ranks per phase, share of the step-loop wall, imbalance (max/mean) and calls; the r...
integer, parameter, public ph_restr
post-stage restrict + reflux-to-parent fold
integer, parameter, public mem_ib
fine IB ghost-point park
integer, parameter ph_n
integer, parameter, public ph_rg_halo
coarse halo + host copy for tagging
integer(8), dimension(ph_n) tic_c
integer, parameter, public ph_rhs
fine-block RHS
integer, parameter, public ph_seam
fine-fine seam halo
integer, parameter, public ph_coarse
coarse (base-grid) solver work
integer, parameter, public mem_ibhost
per-slot fine IB marker copies (host only)
real(wp), dimension(ph_n) acc
integer, parameter ph_ntop
the disjoint brackets above; the regrid sub-phases below nest in PH_REGRID
character(len=8), dimension(ph_n), parameter ph_name
integer, dimension(ph_n) depth
a re-entered bracket (shared routine) measures only its outermost pair
impure subroutine, public s_mem_note_fields(id, f)
s_mem_note for the fields of f that are allocated.
integer, parameter, public ph_rg_tag
tagging
integer, parameter, public ph_rg_build
new-block build (fills, prolongation, overlap copy)
impure subroutine, public s_phase_tic(id)
integer, parameter, public mem_scratch
batched-advance scratch (prim, rhs) and the block bridge
integer, parameter, public ph_reflux
level-1 reflux (faces wave + apply)
integer, parameter mem_n
integer(8), dimension(ph_n) ncall
integer, parameter, public ph_swap
per-batch grid-state swap and restore
impure subroutine, public s_phase_toc(id)
integer, parameter, public ph_rg_clus
clustering, shaping, nesting and box checks
integer, parameter, public ph_rg_own
stash and owner assignment
integer, parameter, public mem_patch
coarse patch pool and its subcycling time cache
integer, parameter, public mem_store
AMR allocations by group, noted by each owner before it frees them (s_mem_note) and printed with the ...
real(wp), dimension(mem_n) mem_mib
integer, parameter, public mem_reg
flux registers