diff --git a/.codespell-ignore-words b/.codespell-ignore-words new file mode 100644 index 00000000000..4069bf5a026 --- /dev/null +++ b/.codespell-ignore-words @@ -0,0 +1,34 @@ +abot +alo +apoints +asend +ba +bloc +blocs +boxs +cant +ccache +clen +compex +couldnt +fromm +frop +geometrys +hist +hsi +infor +inout +ist +lsit +nd +nineth +parm +parms +pres +ptd +recuse +shft +siz +structed +te +thi diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000000..7fa3560731f --- /dev/null +++ b/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +skip = .git,*.ipynb,*.bib,*.ps,*.patch,*~,CHANGES,*/Extern/SWFFT,*/Extern/hpgmg,./tmp_install_dir,./installdir,*/build,*/tmp_build_dir +ignore-words = .codespell-ignore-words diff --git a/.github/workflows/bittree.yml b/.github/workflows/bittree.yml new file mode 100644 index 00000000000..c12fbedc58f --- /dev/null +++ b/.github/workflows/bittree.yml @@ -0,0 +1,127 @@ +name: bittree + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-bittree + cancel-in-progress: true + +jobs: + bittree-2d: + name: Bittree 2D + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dependencies + run: | + .github/workflows/dependencies/dependencies.sh + .github/workflows/dependencies/dependencies_clang-tidy.sh 15 + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Check out Bittree + uses: actions/checkout@v4 + with: + repository: Flash-X/Bittree + path: bittree + - name: Build Bittree + run: | + cd ${{ github.workspace }}/bittree + python setup.py library --dim 2 --prefix ${{ github.workspace }}/libbittree + cd build + make -j2 + make install + - name: Build and Run Test + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=80M + export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy + export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt + ccache -z + + export AMREX_BITTREE_HOME=${{ github.workspace }}/libbittree + cd ${{ github.workspace }}/Tests/Amr/Advection_AmrCore/Exec + make -j2 USE_MPI=TRUE USE_BITTREE=TRUE DIM=2 TEST=TRUE \ + CCACHE=ccache + mpiexec -n 2 ./main2d.gnu.TEST.MPI.ex inputs_bittree amr.plot_int=1000 + + ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt + make -j2 -f clang-tidy-ccache-misses.mak \ + CLANG_TIDY=clang-tidy-15 \ + CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" + + ccache -s + du -hs ~/.cache/ccache + + bittree-3d: + name: Bittree 3D + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dependencies + run: | + .github/workflows/dependencies/dependencies.sh + .github/workflows/dependencies/dependencies_clang-tidy.sh 15 + .github/workflows/dependencies/dependencies_ccache.sh + - name: Set Up Cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Check out Bittree + uses: actions/checkout@v4 + with: + repository: Flash-X/Bittree + path: bittree + - name: Build Bittree + run: | + cd ${{ github.workspace }}/bittree + python setup.py library --dim 3 --prefix ${{ github.workspace }}/libbittree + cd build + make -j2 + make install + - name: Build and Run Test + run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=80M + export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy + export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt + ccache -z + + export AMREX_BITTREE_HOME=${{ github.workspace }}/libbittree + cd ${{ github.workspace }}/Tests/Amr/Advection_AmrCore/Exec + make -j2 USE_MPI=TRUE USE_BITTREE=TRUE DIM=3 TEST=TRUE BL_NO_FORT=TRUE\ + CCACHE=ccache + mpiexec -n 2 ./main3d.gnu.TEST.MPI.ex inputs_bittree max_step=10 + + ${{github.workspace}}/Tools/C_scripts/mmclt.py --input ${{github.workspace}}/ccache.log.txt + make -j2 -f clang-tidy-ccache-misses.mak \ + CLANG_TIDY=clang-tidy-15 \ + CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index a43b89ee8b0..73d6eaf0903 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -8,7 +8,7 @@ on: jobs: CleanUpCcacheCachePostPR: - name: Clean Up Ccahe Cache Post PR + name: Clean Up Ccache Cache Post PR runs-on: ubuntu-latest permissions: actions: write @@ -17,7 +17,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v3 - - name: Clean up ccahe + - name: Clean up ccache run: | gh extension install actions/gh-actions-cache diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index 4b0f444885b..f224ace0ef3 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -2,13 +2,13 @@ name: CleanUpCache on: workflow_run: - workflows: [LinuxClang, cuda, LinuxGcc, hip, Hypre, intel, macos, PETSc, SUNDIALS, windows, CodeQL, smoke, apps] + workflows: [bittree, LinuxClang, cuda, LinuxGcc, hip, Hypre, intel, macos, PETSc, SUNDIALS, CodeQL, smoke, apps] types: - completed jobs: CleanUpCcacheCache: - name: Clean Up Ccahe Cache for ${{ github.event.workflow_run.name }} + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} runs-on: ubuntu-latest permissions: actions: write @@ -17,7 +17,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v3 - - name: Clean up ccahe + - name: Clean up ccache run: | gh extension install actions/gh-actions-cache diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000000..491b839c00e --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,23 @@ +name: codespell + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-codespell + cancel-in-progress: true + +jobs: + codespell: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install codespell + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends python3-pip + pip3 install --user codespell + + - name: Run codespell + run: codespell diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b4abc1c0a05..3fe50a5a02d 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -117,6 +117,7 @@ jobs: - uses: actions/checkout@v3 - name: Dependencies run: | + .github/workflows/dependencies/ubuntu_free_disk_space.sh .github/workflows/dependencies/dependencies_nvhpc.sh .github/workflows/dependencies/dependencies_ccache.sh - name: Set Up Cache diff --git a/.github/workflows/dependencies/ubuntu_free_disk_space.sh b/.github/workflows/dependencies/ubuntu_free_disk_space.sh new file mode 100755 index 00000000000..6b3e5b2f55e --- /dev/null +++ b/.github/workflows/dependencies/ubuntu_free_disk_space.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# Copyright 2023 The AMReX Community +# +# License: BSD-3-Clause-LBNL + +# Don't want to use the following line because apt-get remove may fail if +# the package specfied does not exist. +# set -eu -o pipefail + +# Large packages +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 + +echo 'Removing some packages we do not need' + +df -h + +apt list --installed + +sudo apt-get remove -y '^apache.*' +sudo apt-get remove -y '^aspnetcore.*' +sudo apt-get remove -y '^azure.*' +sudo apt-get remove -y '^dotnet.*' +sudo apt-get remove -y '^firebird.*' +sudo apt-get remove -y '^firefox.*' +sudo apt-get remove -y '^google.*' +sudo apt-get remove -y '^hhvm.*' +sudo apt-get remove -y '^microsoft.*' +sudo apt-get remove -y '^mongodb.*' +sudo apt-get remove -y '^mono-.*' +sudo apt-get remove -y '^monodoc-.*' +sudo apt-get remove -y '^mysql.*' +sudo apt-get remove -y '^php.*' +sudo apt-get remove -y '^powershell.*' +sudo apt-get remove -y '^snapd.*' +sudo apt-get remove -y '^temurin.*' + +sudo apt-get autoremove -y + +df -h diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 031f7d331bf..6133e666fad 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -152,6 +152,7 @@ jobs: - name: Dependencies if: ${{ env.CODEPLAYTOKEN != '' }} run: | + .github/workflows/dependencies/ubuntu_free_disk_space.sh .github/workflows/dependencies/dependencies_hip.sh 5.4.6 .github/workflows/dependencies/dependencies_dpcpp.sh .github/workflows/dependencies/dependencies_codeplay.sh ${{ env.CODEPLAYTOKEN }} @@ -199,6 +200,7 @@ jobs: - uses: actions/checkout@v3 - name: Dependencies run: | + .github/workflows/dependencies/ubuntu_free_disk_space.sh .github/workflows/dependencies/dependencies_dpcpp.sh sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic .github/workflows/dependencies/dependencies_ccache.sh diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 985e143108f..18dbce32b5b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -14,28 +14,28 @@ jobs: steps: - uses: actions/checkout@v3 - uses: seanmiddleditch/gha-setup-ninja@master - - name: Set Up Cache - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: Install Ccache - run: | - Invoke-WebRequest https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip -OutFile ccache-4.8-windows-x86_64.zip - Expand-Archive ccache-4.8-windows-x86_64.zip + #- name: Set Up Cache + # uses: actions/cache@v3 + # with: + # path: ~/.ccache + # key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + # restore-keys: | + # ccache-${{ github.workflow }}-${{ github.job }}-git- + #- name: Install Ccache + # run: | + # Invoke-WebRequest https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip -OutFile ccache-4.8-windows-x86_64.zip + # Expand-Archive ccache-4.8-windows-x86_64.zip - name: Build & Install run: | - $ccachepath = Join-Path $pwd "ccache-4.8-windows-x86_64" - $Env:PATH += ";$ccachepath" - $ccachecachedir = Join-Path $HOME ".ccache" - $Env:CCACHE_DIR="$ccachecachedir" - $Env:CCACHE_DIR - $Env:CCACHE_COMPRESS='1' - $Env:CCACHE_COMPRESSLEVEL='10' - $Env:CCACHE_MAXSIZE='105M' - ccache -z + #$ccachepath = Join-Path $pwd "ccache-4.8-windows-x86_64" + #$Env:PATH += ";$ccachepath" + #$ccachecachedir = Join-Path $HOME ".ccache" + #$Env:CCACHE_DIR="$ccachecachedir" + #$Env:CCACHE_DIR + #$Env:CCACHE_COMPRESS='1' + #$Env:CCACHE_COMPRESSLEVEL='10' + #$Env:CCACHE_MAXSIZE='105M' + #ccache -z cmake -S . -B build ` -G "Ninja" ` @@ -45,11 +45,11 @@ jobs: -DAMReX_EB=OFF ` -DAMReX_ENABLE_TESTS=ON ` -DAMReX_FORTRAN=OFF ` - -DAMReX_MPI=OFF ` - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DAMReX_MPI=OFF + #-DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build --config Debug -j 2 - ccache -s + #ccache -s # Build libamrex and all test (static) test_msvc_static: @@ -58,27 +58,27 @@ jobs: steps: - uses: actions/checkout@v3 - uses: seanmiddleditch/gha-setup-ninja@master - - name: Set Up Cache - uses: actions/cache@v3 - with: - path: ~/.ccache - key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} - restore-keys: | - ccache-${{ github.workflow }}-${{ github.job }}-git- - - name: Install Ccache - run: | - Invoke-WebRequest https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip -OutFile ccache-4.8-windows-x86_64.zip - Expand-Archive ccache-4.8-windows-x86_64.zip + #- name: Set Up Cache + # uses: actions/cache@v3 + # with: + # path: ~/.ccache + # key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + # restore-keys: | + # ccache-${{ github.workflow }}-${{ github.job }}-git- + #- name: Install Ccache + # run: | + # Invoke-WebRequest https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-windows-x86_64.zip -OutFile ccache-4.8-windows-x86_64.zip + # Expand-Archive ccache-4.8-windows-x86_64.zip - name: Build & Install run: | - $ccachepath = Join-Path $pwd "ccache-4.8-windows-x86_64" - $Env:PATH += ";$ccachepath" - $ccachecachedir = Join-Path $HOME ".ccache" - $Env:CCACHE_DIR="$ccachecachedir" - $Env:CCACHE_COMPRESS='1' - $Env:CCACHE_COMPRESSLEVEL='10' - $Env:CCACHE_MAXSIZE='135M' - ccache -z + #$ccachepath = Join-Path $pwd "ccache-4.8-windows-x86_64" + #$Env:PATH += ";$ccachepath" + #$ccachecachedir = Join-Path $HOME ".ccache" + #$Env:CCACHE_DIR="$ccachecachedir" + #$Env:CCACHE_COMPRESS='1' + #$Env:CCACHE_COMPRESSLEVEL='10' + #$Env:CCACHE_MAXSIZE='135M' + #ccache -z cmake -S . -B build ` -G "Ninja" ` @@ -87,11 +87,11 @@ jobs: -DAMReX_EB=ON ` -DAMReX_ENABLE_TESTS=ON ` -DAMReX_FORTRAN=OFF ` - -DAMReX_MPI=OFF ` - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DAMReX_MPI=OFF + #-DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build --config RelWithDebInfo -j 2 - ccache -s + #ccache -s # Build libamrex and all tests tests_clang: @@ -116,17 +116,18 @@ jobs: -DAMReX_OMP=ON cmake --build build --config Release -j 2 - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + # If we add ccache back, don't forget to update cleanup-cache.yml + #save_pr_number: + # if: github.event_name == 'pull_request' + # runs-on: ubuntu-latest + # steps: + # - name: Save PR number + # env: + # PR_NUMBER: ${{ github.event.number }} + # run: | + # echo $PR_NUMBER > pr_number.txt + # - uses: actions/upload-artifact@v3 + # with: + # name: pr_number + # path: pr_number.txt + # retention-days: 1 diff --git a/CHANGES b/CHANGES index e27552e7cb4..ae56cdd5ffe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,48 @@ +# 23.10 + + -- Bugfix typo in AMReX_SundialsIntegrator.H Nvar vs NVar, the + declared/used variable is NVar (#3573) + + -- Code Spell (#3563) + + -- Add Fortran interface for average_down_faces (#3553) + + -- PureSoA: Disable AoS Access (#3290) + + -- Another terrain fix for MPI (#3557) + Fix Increment, OK, and EnforcePeriodicWhere for terrain-fitted particles. (#3556) + + -- Added cvode functionality to SUNDIALS integrator (#3436) + + -- ParmParse::addfile needs Init (#3440) + + -- Make the same changes to ApplyInitialRedistribution as to ApplyMLRedistribution (#3554) + + -- Reset EB Fab Type (#3552) + EB Data outside domain (#3549) + + -- We weren't defining cent_hat out far enough (#3548) + + -- Add Fortran inteface for FillCoarsePatch for face variables (#3542) + + -- print_state/printCell: Make it work without managed memory (#3543) + + -- FillPatch Fortran Interface: Fix incorrect size of Vector (#3546) + + -- ReduceOps: reset result readiness flag (#3545) + + -- Fix Fortran interface for FillPatch for face variables (#3541) + + -- Support multiple CUDA architectures at compilation (#3535) + + -- Add Kestrel machine and remove Rhodes machine from Make.nrel (#3533) + + -- Explicitly flush when writing to the terse run log (#3532) + + -- Missing header in AMReX_GpuComplex.H (#3531) + + -- Add global domain id offset to conduit wrapper (#3524) + # 23.09 -- Fix InitRandomPerBox for 1D & 2D (#3527) diff --git a/Docs/sphinx_documentation/source/AmrCore.rst b/Docs/sphinx_documentation/source/AmrCore.rst index b444efffab9..6aadd62250e 100644 --- a/Docs/sphinx_documentation/source/AmrCore.rst +++ b/Docs/sphinx_documentation/source/AmrCore.rst @@ -238,7 +238,7 @@ Within AMReX_Interpolater.cpp/H are the derived classes: - :cpp:`FaceLinear` -- :cpp:`FaceDivFree`: This is more accurately a divergence-preserving interpolation on face centered data, i.e., it ensures the divergence of the fine ghost cells match the value of the divergence of the underlying coarse cell. All fine cells overlying a given coarse cell will have the same divergence, even when the coarse grid divergence is spatially varying. Note that when using this with :cpp:`FillPatch` for time sub-cycling, the coarse grid times may not match the fine grid time, in which case :cpp:`FillPatch` will create coarse values at the fine time before calling this interpolation and the result of the :cpp:`FillPatch` is *not* garanteed to preserve the original divergence. +- :cpp:`FaceDivFree`: This is more accurately a divergence-preserving interpolation on face centered data, i.e., it ensures the divergence of the fine ghost cells match the value of the divergence of the underlying coarse cell. All fine cells overlying a given coarse cell will have the same divergence, even when the coarse grid divergence is spatially varying. Note that when using this with :cpp:`FillPatch` for time sub-cycling, the coarse grid times may not match the fine grid time, in which case :cpp:`FillPatch` will create coarse values at the fine time before calling this interpolation and the result of the :cpp:`FillPatch` is *not* guaranteed to preserve the original divergence. These Interpolaters can be executed on CPU or GPU, with certain limitations: diff --git a/Docs/sphinx_documentation/source/Basics.rst b/Docs/sphinx_documentation/source/Basics.rst index fd2d12ee0b8..1227b5f7b71 100644 --- a/Docs/sphinx_documentation/source/Basics.rst +++ b/Docs/sphinx_documentation/source/Basics.rst @@ -2537,12 +2537,27 @@ The basic idea behind physical boundary conditions is as follows: ext_dir "External Dirichlet". It is the user's responsibility to write a routine - to fill ghost cells (more details below). + to fill ghost cells (more details below). The boundary location + is on the domain face even when the data inside the domain are + cell-centered. + + ext_dir_cc + "External Dirichlet". It is the user's responsibility to write a routine + to fill ghost cells (more details below). The boundary location + is at the cell center of ghost cells outside the domain. foextrap "First Order Extrapolation" First order extrapolation from last cell in interior. + hoextrap + "High Order Extrapolation". The boundary location is on the domain + face even when the data inside the domain are cell-centered. + + hoextrapcc + "High Order Extrapolation" The boundary location is at the cell + center of ghost cells outside the domain. + reflect_even Reflection from interior cells with sign unchanged, :math:`q(-i) = q(i)`. diff --git a/Docs/sphinx_documentation/source/GPU.rst b/Docs/sphinx_documentation/source/GPU.rst index 1391015f31e..56d540e63c4 100644 --- a/Docs/sphinx_documentation/source/GPU.rst +++ b/Docs/sphinx_documentation/source/GPU.rst @@ -1738,14 +1738,14 @@ by "amrex" in your :cpp:`inputs` file. +----------------------------+-----------------------------------------------------------------------+-------------+----------+ | | Description | Type | Default | +============================+=======================================================================+=============+==========+ -| use_gpu_aware_mpi | Whether to use GPU memory for communication buffers during MPI calls. | Bool | False | -| | If true, the buffers will use device memory. If false, they will use | | | -| | pinned memory. In practice, we find it is usually not worth it to use | | | -| | GPU aware MPI. | | | +| use_gpu_aware_mpi | Whether to use GPU memory for communication buffers during MPI calls. | Bool | 0 | +| | If true, the buffers will use device memory. If false (i.e., 0), they | | | +| | will use pinned memory. In practice, we find it is not always worth | | | +| | it to use GPU aware MPI. | | | +----------------------------+-----------------------------------------------------------------------+-------------+----------+ -| abort_on_out_of_gpu_memory | If the size of free memory on the GPU is less than the size of a | Bool | False | +| abort_on_out_of_gpu_memory | If the size of free memory on the GPU is less than the size of a | Bool | 0 | | | requested allocation, AMReX will call AMReX::Abort() with an error | | | | | describing how much free memory there is and what was requested. | | | +----------------------------+-----------------------------------------------------------------------+-------------+----------+ -| the_arena_is_managed | Whether :cpp:`The_Arena()` allocates managed memory. | Bool | False | +| the_arena_is_managed | Whether :cpp:`The_Arena()` allocates managed memory. | Bool | 0 | +----------------------------+-----------------------------------------------------------------------+-------------+----------+ diff --git a/Docs/sphinx_documentation/source/InputsPlotFiles.rst b/Docs/sphinx_documentation/source/InputsPlotFiles.rst index 64b202ebf5c..9e8789a90ac 100644 --- a/Docs/sphinx_documentation/source/InputsPlotFiles.rst +++ b/Docs/sphinx_documentation/source/InputsPlotFiles.rst @@ -12,7 +12,7 @@ as whether a plotfile should be written out immediately after restarting a simul | plot_int | Frequency of plotfile output; | Int | -1 | | | if -1 then no plotfiles will be written | | | +---------------------+-----------------------------------------------------------------------+-------------+-----------+ -| plotfile_on_restart | Should we write a plotfile when we restart (only used if plot_int>0) | Bool | False | +| plotfile_on_restart | Should we write a plotfile when we restart (only used if plot_int>0) | Bool | 0 (false) | +---------------------+-----------------------------------------------------------------------+-------------+-----------+ | plot_file | Prefix to use for plotfile output | String | plt | +---------------------+-----------------------------------------------------------------------+-------------+-----------+ diff --git a/Docs/sphinx_documentation/source/LinearSolvers.rst b/Docs/sphinx_documentation/source/LinearSolvers.rst index 87048bc8195..e7266f02243 100644 --- a/Docs/sphinx_documentation/source/LinearSolvers.rst +++ b/Docs/sphinx_documentation/source/LinearSolvers.rst @@ -565,7 +565,7 @@ The following parameter should be set to True if the problem to be solved has a In this case, the solution is only defined to within a constant. Setting this parameter to True replaces one row in the matrix sent to hypre from AMReX by a row that sets the value at one cell to 0. -- :cpp:`hypre.adjust_singular_matrix`: Default is False. +- :cpp:`hypre.adjust_singular_matrix`: Default is false. The following parameters can be set in the inputs file to control the choice of preconditioner and smoother: diff --git a/Docs/sphinx_documentation/source/Particle.rst b/Docs/sphinx_documentation/source/Particle.rst index da5fabb02b2..be8292c772b 100644 --- a/Docs/sphinx_documentation/source/Particle.rst +++ b/Docs/sphinx_documentation/source/Particle.rst @@ -713,7 +713,7 @@ with OpenMP, the first thing to look at is whether there are enough tiles availa +-------------------+-----------------------------------------------------------------------+-------------+-------------+ | | Description | Type | Default | +===================+=======================================================================+=============+=============+ -| do_tiling | Whether to use tiling for particles. Should be on when using OpenMP, | Bool | False | +| do_tiling | Whether to use tiling for particles. Should be on when using OpenMP, | Bool | false | | | and off when running on GPUs. | | | +-------------------+-----------------------------------------------------------------------+-------------+-------------+ | tile_size | If tiling is on, the maximum tile_size to in each direction | Ints | 1024000,8,8 | @@ -739,7 +739,7 @@ problems with particle IO, you could try varying some / all of these parameters. | datadigits_read | This for backwards compatibility, don't use unless you need to read | Int | 5 | | | and old (pre mid 2017) AMReX dataset. | | | +-------------------+-----------------------------------------------------------------------+-------------+-------------+ -| use_prepost | This is an optimization for large particle datasets that groups MPI | Bool | False | +| use_prepost | This is an optimization for large particle datasets that groups MPI | Bool | false | | | calls needed during the IO together. Try it seeing poor IO speeds | | | | | on large problems. | | | +-------------------+-----------------------------------------------------------------------+-------------+-------------+ diff --git a/Docs/sphinx_documentation/source/Python_Chapter.rst b/Docs/sphinx_documentation/source/Python_Chapter.rst new file mode 100644 index 00000000000..76061ea2ebe --- /dev/null +++ b/Docs/sphinx_documentation/source/Python_Chapter.rst @@ -0,0 +1,13 @@ +.. role:: cpp(code) + :language: c++ + +.. _Chap:Python: + +Python Interface +================ + + +The core of AMReX is written in C++. +For users who want to write all of their programs in Python, or C++ application developers that like to add Python interfaces to their applications for scripting, rapid prototyping, code coupling and/or AI/ML workflows, many AMReX classes, functions and all data containers are now also available. + +Please see `pyAMReX `__ (`manual `__) for further details. diff --git a/Docs/sphinx_documentation/source/index.rst b/Docs/sphinx_documentation/source/index.rst index 8cdced26a19..b748a6394a5 100644 --- a/Docs/sphinx_documentation/source/index.rst +++ b/Docs/sphinx_documentation/source/index.rst @@ -51,6 +51,7 @@ Documentation on migration from BoxLib is available in the AMReX repository at D LinearSolvers_Chapter Particle_Chapter Fortran_Chapter + Python_Chapter EB_Chapter TimeIntegration_Chapter GPU_Chapter diff --git a/GNUmakefile.in b/GNUmakefile.in index 21185a705f1..b85c2e0c35e 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -45,6 +45,9 @@ endif ifeq ($(USE_SUNDIALS),TRUE) Pdirs += Extern/SUNDIALS endif +ifeq ($(USE_BITTREE),TRUE) + Pdirs += Extern/Bittree +endif Ppack := $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) include $(Ppack) diff --git a/Src/Amr/AMReX_Amr.cpp b/Src/Amr/AMReX_Amr.cpp index 0b1dfb6f886..c3e023e04a8 100644 --- a/Src/Amr/AMReX_Amr.cpp +++ b/Src/Amr/AMReX_Amr.cpp @@ -3085,7 +3085,7 @@ Amr::bldFineLevels (Real strt_time) { bool grids_the_same; - const int MaxCnt = 4; + const int MaxCnt = std::max(4, max_level+1); int count = 0; diff --git a/Src/Amr/AMReX_StateDescriptor.cpp b/Src/Amr/AMReX_StateDescriptor.cpp index 161090d1a7f..4f1e3c3fe1b 100644 --- a/Src/Amr/AMReX_StateDescriptor.cpp +++ b/Src/Amr/AMReX_StateDescriptor.cpp @@ -19,7 +19,7 @@ StateDescriptor::bf_thread_safety (const int* /*lo*/,const int* /*hi*/, if (!bf_ext_dir_threadsafe) { bool has_ext_dir = false; for (int i=0; i<2*AMREX_SPACEDIM*ng && !has_ext_dir; ++i) { - has_ext_dir = bc[i]==BCType::ext_dir; + has_ext_dir = ((bc[i]==BCType::ext_dir) || (bc[i]==BCType::ext_dir_cc)); } if (has_ext_dir) { thread_safe = false; } } diff --git a/Src/AmrCore/AMReX_AmrCore.cpp b/Src/AmrCore/AMReX_AmrCore.cpp index 502b3f5cb23..ed0cd5d1020 100644 --- a/Src/AmrCore/AMReX_AmrCore.cpp +++ b/Src/AmrCore/AMReX_AmrCore.cpp @@ -104,7 +104,7 @@ AmrCore::regrid (int lbase, Real time, bool) DistributionMapping level_dmap = dmap[lev]; if (ba_changed) { level_grids = new_grids[lev]; - level_dmap = DistributionMapping(level_grids); + level_dmap = MakeDistributionMap(lev, level_grids); } const auto old_num_setdm = num_setdm; RemakeLevel(lev, time, level_grids, level_dmap); @@ -117,7 +117,7 @@ AmrCore::regrid (int lbase, Real time, bool) } else // a new level { - DistributionMapping new_dmap(new_grids[lev]); + DistributionMapping new_dmap = MakeDistributionMap(lev, new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromCoarse(lev, time, new_grids[lev], new_dmap); SetBoxArray(lev, new_grids[lev]); diff --git a/Src/AmrCore/AMReX_AmrMesh.H b/Src/AmrCore/AMReX_AmrMesh.H index a3c6fbc8f62..f5d49f5c5da 100644 --- a/Src/AmrCore/AMReX_AmrMesh.H +++ b/Src/AmrCore/AMReX_AmrMesh.H @@ -11,6 +11,10 @@ #include #include +#ifdef AMREX_USE_BITTREE +#include +#endif + namespace amrex { struct AmrInfo { @@ -166,7 +170,7 @@ public: void SetGridEff (Real eff) noexcept { grid_eff = eff; } void SetNProper (int n) noexcept { n_proper = n; } - //! Set ref_ratio would require rebuiling Geometry objects. + //! Set ref_ratio would require rebuilding Geometry objects. void SetFinestLevel (int new_finest_level) noexcept { finest_level = new_finest_level; } void SetDistributionMap (int lev, const DistributionMapping& dmap_in) noexcept; @@ -253,6 +257,8 @@ public: [[nodiscard]] long CountCells (int lev) noexcept; + [[nodiscard]] virtual DistributionMapping MakeDistributionMap (int lev, BoxArray const& ba); + protected: int finest_level; //!< Current finest level. @@ -260,6 +266,11 @@ protected: Vector dmap; Vector grids; +#ifdef AMREX_USE_BITTREE + bool use_bittree = false; + std::unique_ptr btmesh; +#endif + unsigned int num_setdm = 0; unsigned int num_setba = 0; diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 70a8df1dabf..efccf318315 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -5,6 +5,13 @@ #include #include #include +#include + +#ifdef AMREX_USE_BITTREE +#include +#endif + +#include namespace amrex { @@ -376,6 +383,10 @@ AmrMesh::InitAmrMesh (int max_level_in, const Vector& n_cell_in, finest_level = -1; +#ifdef AMREX_USE_BITTREE + pp.queryAdd("use_bittree",use_bittree); +#endif + if (check_input) { checkInput(); } } @@ -437,6 +448,26 @@ AmrMesh::LevelDefined (int lev) noexcept return lev <= max_level && !grids[lev].empty() && !dmap[lev].empty(); } +DistributionMapping +AmrMesh::MakeDistributionMap (int lev, BoxArray const& ba) +{ + + BL_PROFILE("AmrMesh::MakeDistributionMap()"); + + if (verbose) { + amrex::Print() << "Creating new distribution map on level: " << lev << "\n"; + } + +#ifdef AMREX_USE_BITTREE + // if (use_bittree) { + // return DistributionMapping(ba); + // } else +#endif + { + return DistributionMapping(ba); + } +} + void AmrMesh::ChopGrids (int lev, BoxArray& ba, int target_size) const { @@ -514,6 +545,10 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& if (new_grids.size() < max_crse+2) { new_grids.resize(max_crse+2); } +#ifdef AMREX_USE_BITTREE + if(!use_bittree) { +#endif + // // Construct problem domain at each level. // @@ -774,6 +809,72 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& } } } + +#ifdef AMREX_USE_BITTREE + } +#endif + +#ifdef AMREX_USE_BITTREE + // Bittree version + if(use_bittree) { + // Initialize BT refinement + btmesh->refine_init(); + + // ------------------------------------------------------------------- + // Use tagging data to mark BT for refinement, then use the new bitmap + // to calculate the new grids. + auto tree0 = btmesh->getTree(); + + // [1] Error Estimation and tagging + // btTags is indexed by bitid, Bittree's internal indexing scheme. + // For any id, btTags = 1 if should be parent, -1 if should not be parent (or not exist). + std::vector btTags(tree0->id_upper_bound(),0); + + for (int lev=max_crse; lev>=lbase; --lev) { + + TagBoxArray tags(grids[lev],dmap[lev], n_error_buf[lev]); + ErrorEst(lev, tags, time, 0); + tags.buffer(n_error_buf[lev]); + + for (MFIter mfi(tags); mfi.isValid(); ++mfi) { + auto const& tagbox = tags.const_array(mfi); + bool has_set_tags = amrex::Reduce::AnyOf(mfi.validbox(), + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + return tagbox(i,j,k)!=TagBox::CLEAR; + }); + + // Set the values of btTags. + int bitid = btUnit::getBitid(btmesh.get(),false,lev,mfi.index()); + // TODO Check lev == tree0->block_level(bitid) + if(has_set_tags) { + btTags[bitid] = 1; + } + else { + btTags[bitid] = -1; + } + } + } + + // [2] btRefine - check for proper octree nesting and update bitmap + MPI_Comm comm = ParallelContext::CommunicatorSub(); + int changed = btUnit::btRefine(btmesh.get(), btTags, max_crse, lbase, grids, dmap, comm); + + // [3] btCalculateGrids - use new bitmap to generate new grids + if (changed>0) { + btUnit::btCalculateGrids(btmesh.get(),lbase,new_finest,new_grids,max_grid_size); + } else { + new_finest = finest_level; + for(int i=0; i<=finest_level; ++i) { + new_grids[i] = grids[i]; + } + } + + // Finalize BT refinement + btmesh->refine_apply(); + } +#endif + } void @@ -783,11 +884,48 @@ AmrMesh::MakeNewGrids (Real time) { finest_level = 0; - const BoxArray& ba = MakeBaseGrids(); - DistributionMapping dm(ba); + BoxArray ba; + DistributionMapping dm; const auto old_num_setdm = num_setdm; const auto old_num_setba = num_setba; +#ifdef AMREX_USE_BITTREE + if(!use_bittree) { +#endif + ba = MakeBaseGrids(); + dm = MakeDistributionMap(0, ba); + +#ifdef AMREX_USE_BITTREE + } + else { + //Initialize Bittree + + // top = number of grids on coarsest level in each direction + std::vector top(AMREX_SPACEDIM,0); + IntVect ncells = geom[0].Domain().length(); + for(int i=0; i includes(ngrids,1); + + btmesh = std::make_unique(top.data(),includes.data()); + + // Set BCs + for(int d=0; d::storeRKCoarseData (Real /*time*/, Real dt, MF const& S_old auto const& fpc = getFPinfo(); for (auto& tmf : m_cf_crse_data) { - tmf.first = std::numeric_limits::lowest(); // because we dont' need it + tmf.first = std::numeric_limits::lowest(); // because we don't need it tmf.second = std::make_unique(make_mf_crse_patch(fpc, m_ncomp)); } m_cf_crse_data[0].second->ParallelCopy(S_old, m_cgeom.periodicity()); diff --git a/Src/AmrCore/AMReX_FluxRegister.H b/Src/AmrCore/AMReX_FluxRegister.H index 95f3cefde26..4178eb289ba 100644 --- a/Src/AmrCore/AMReX_FluxRegister.H +++ b/Src/AmrCore/AMReX_FluxRegister.H @@ -171,7 +171,7 @@ public: const Geometry& geom); /** - * \brief /in this version the area is assumed to muliplied into the flux (if not, use scale to fix) + * \brief /in this version the area is assumed to multiplied into the flux (if not, use scale to fix) * * \param mflx * \param dir @@ -192,7 +192,7 @@ public: /** * \brief Increment flux correction with fine data. * - * /in this version the area is assumed to muliplied into the flux (if not, use scale to fix) + * /in this version the area is assumed to multiplied into the flux (if not, use scale to fix) * * \param mflx * \param dir @@ -230,7 +230,7 @@ public: /** * \brief Increment flux correction with fine data. * - * in this version the area is assumed to muliplied into the flux (if not, use scale to fix) + * in this version the area is assumed to multiplied into the flux (if not, use scale to fix) * * \param flux * \param dir diff --git a/Src/AmrCore/Make.package b/Src/AmrCore/Make.package index df3c2e83d40..bd0bddd6704 100644 --- a/Src/AmrCore/Make.package +++ b/Src/AmrCore/Make.package @@ -1,5 +1,6 @@ -CEXE_headers += AMReX_AmrCore.H AMReX_Cluster.H AMReX_ErrorList.H AMReX_FillPatchUtil.H AMReX_FillPatchUtil_I.H AMReX_FluxRegister.H \ +CEXE_headers += AMReX_AmrCore.H AMReX_Cluster.H AMReX_ErrorList.H AMReX_FillPatchUtil.H \ + AMReX_FillPatchUtil_I.H AMReX_FluxRegister.H \ AMReX_Interpolater.H AMReX_MFInterpolater.H AMReX_TagBox.H AMReX_AmrMesh.H \ AMReX_InterpBase.H CEXE_sources += AMReX_AmrCore.cpp AMReX_Cluster.cpp AMReX_ErrorList.cpp AMReX_FillPatchUtil.cpp AMReX_FluxRegister.cpp \ diff --git a/Src/Base/AMReX.cpp b/Src/Base/AMReX.cpp index b43add7c9d1..f0e56952de2 100644 --- a/Src/Base/AMReX.cpp +++ b/Src/Base/AMReX.cpp @@ -408,8 +408,8 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse, else { // This counts command line arguments before a "--" - // and only sends the preceeding arguments to ParmParse; - // the rest get ingored. + // and only sends the preceding arguments to ParmParse; + // the rest get ignored. int ppargc = 1; for (; ppargc < argc; ++ppargc) { if (std::strcmp(argv[ppargc], "--") == 0) { break; } @@ -514,7 +514,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse, pp.queryAdd("handle_sigfpe" , system::handle_sigfpe ); pp.queryAdd("handle_sigill" , system::handle_sigill ); - // We save the singal handlers and restore them in Finalize. + // We save the signal handlers and restore them in Finalize. if (system::handle_sigsegv) { prev_handler_sigsegv = std::signal(SIGSEGV, BLBackTrace::handler); } else { diff --git a/Src/Base/AMReX_ANSIEscCode.H b/Src/Base/AMReX_ANSIEscCode.H index 02f4bbc059f..1c348deabf1 100644 --- a/Src/Base/AMReX_ANSIEscCode.H +++ b/Src/Base/AMReX_ANSIEscCode.H @@ -16,7 +16,7 @@ namespace Font { constexpr char RapidBlink [] = "\033[6m"; } -namespace FGColor { // Forground colors +namespace FGColor { // Foreground colors constexpr char Black [] = "\033[30m"; constexpr char Red [] = "\033[31m"; constexpr char Green [] = "\033[32m"; diff --git a/Src/Base/AMReX_Arena.H b/Src/Base/AMReX_Arena.H index c3dad6ac2b4..e42ebdc9cd2 100644 --- a/Src/Base/AMReX_Arena.H +++ b/Src/Base/AMReX_Arena.H @@ -146,7 +146,7 @@ public: /** * \brief Does the device have enough free memory for allocating this * much memory? For CPU builds, this always return true. This is not a - * const funciton because it may attempt to release memory back to the + * const function because it may attempt to release memory back to the * system. */ [[nodiscard]] virtual bool hasFreeDeviceMemory (std::size_t sz); diff --git a/Src/Base/AMReX_BCUtil.H b/Src/Base/AMReX_BCUtil.H index dd4e814610c..3ebc53a0576 100644 --- a/Src/Base/AMReX_BCUtil.H +++ b/Src/Base/AMReX_BCUtil.H @@ -9,11 +9,12 @@ namespace amrex { // This is for filling cell-centered data outside physical domain - // (excluding periodic boundaries). It only fills - // BCType::foextrap, BCType::hoextrap, BCType::reflect_even, and - // BCType::reflect::odd. It does not fill BCType::ext_dir (i.e., - // external Dirichlet). If you have BCType::ext_dir, you can - // copy, paste and rename this function, and then modify it. + // (excluding periodic boundaries). It only fills BCType::foextrap, + // BCType::hoextrap, BCType::hoextrapcc, BCType::reflect_even, and + // BCType::reflect::odd. It does not fill BCType::ext_dir and + // BCType::ext_dir_cc (i.e., external Dirichlet). If you have + // BCType::ext_dir or BCType::ext_dir_cc, you can copy, paste and rename + // this function, and then modify it. void FillDomainBoundary (MultiFab& phi, const Geometry& geom, const Vector& bc); } diff --git a/Src/Base/AMReX_BCUtil.cpp b/Src/Base/AMReX_BCUtil.cpp index c0a645ba05c..dc5b4f4f10c 100644 --- a/Src/Base/AMReX_BCUtil.cpp +++ b/Src/Base/AMReX_BCUtil.cpp @@ -15,7 +15,7 @@ void dummy_cpu_fill_extdir (Box const& /*bx*/, Array4 const& /*dest*/, const BCRec* /*bcr*/, const int /*bcomp*/, const int /*orig_comp*/) { - // do something for external Dirichlet (BCType::ext_dir) if there are + // do something for external Dirichlet (BCType::ext_dir or BCType::ext_dir_cc) if there are } struct dummy_gpu_fill_extdir @@ -27,7 +27,7 @@ struct dummy_gpu_fill_extdir const BCRec* /*bcr*/, const int /*bcomp*/, const int /*orig_comp*/) const { - // do something for external Dirichlet (BCType::ext_dir) if there are + // do something for external Dirichlet (BCType::ext_dir or BCType::ext_dir_cc) if there are } }; diff --git a/Src/Base/AMReX_BC_TYPES.H b/Src/Base/AMReX_BC_TYPES.H index 872ac1eff5d..00fadd5e6d0 100644 --- a/Src/Base/AMReX_BC_TYPES.H +++ b/Src/Base/AMReX_BC_TYPES.H @@ -44,10 +44,8 @@ SYMMETRY | Un REFLECT_ODD | REFLECT_EVEN | REFLECT_EVEN | INT_DIR : data taken from other grids or interpolated -EXT_DIR : data specified on EDGE (FACE) of bndry for the linear solvers - for application codes, the location of the EXT_DIR data will - depend on how they do reconstruction, and may be edge or - cell-centered. +EXT_DIR : data specified on EDGE (FACE) +EXT_DIR_CC : data specified at cell center HOEXTRAP : higher order extrapolation to EDGE of bndry HOEXTRAPCC : linear extrapolation to CELL of bndry FOEXTRAP : first order extrapolation from last cell in interior @@ -74,6 +72,7 @@ enum mathematicalBndryTypes : int { ext_dir = 3, hoextrap = 4, hoextrapcc = 5, + ext_dir_cc = 6, user_1 = 1001, user_2 = 1002, user_3 = 1003 @@ -94,6 +93,7 @@ enum mathematicalBndryTypes : int { #define EXT_DIR 3 #define HOEXTRAP 4 #define HOEXTRAPCC 5 +#define EXT_DIR_CC 6 #define Interior 0 #define Inflow 1 diff --git a/Src/Base/AMReX_BLProfiler.cpp b/Src/Base/AMReX_BLProfiler.cpp index 9d12b23708b..fc94104b7d3 100644 --- a/Src/Base/AMReX_BLProfiler.cpp +++ b/Src/Base/AMReX_BLProfiler.cpp @@ -1281,7 +1281,7 @@ void BLProfiler::WriteCommStats(bool bFlushing, bool memCheck) nfiHeader.Stream() << "CommProfProc " << myProc << " nCommStats " << vCommStats.size() << " datafile " << localDFileName - << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek posotion + << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek position << " " << procName << '\n'; for(int ib(0); ib < CommStats::barrierNames.size(); ++ib) { int seekindex(CommStats::barrierNames[ib].second); diff --git a/Src/Base/AMReX_CArena.H b/Src/Base/AMReX_CArena.H index 8f79c844da8..d68285bc878 100644 --- a/Src/Base/AMReX_CArena.H +++ b/Src/Base/AMReX_CArena.H @@ -70,7 +70,7 @@ public: /** * \brief Does the device have enough free memory for allocating this * much memory? For CPU builds, this always return true. This is not a - * const funciton because it may attempt to release memory back to the + * const function because it may attempt to release memory back to the * system. */ [[nodiscard]] bool hasFreeDeviceMemory (std::size_t sz) final; diff --git a/Src/Base/AMReX_CTOParallelForImpl.H b/Src/Base/AMReX_CTOParallelForImpl.H index 80663728359..73ca3a25e87 100644 --- a/Src/Base/AMReX_CTOParallelForImpl.H +++ b/Src/Base/AMReX_CTOParallelForImpl.H @@ -180,7 +180,7 @@ ParallelFor (TypeList /*list_of_compile_time_options*/, * * \param ctos list of all possible values of the parameters. * \param option the run time parameters. - * \param N an interger specifying the 1D for loop's range. + * \param N an integer specifying the 1D for loop's range. * \param f a callable object taking an integer and working on that iteration. */ template diff --git a/Src/Base/AMReX_CoordSys.H b/Src/Base/AMReX_CoordSys.H index d558ba053db..ab946ffa3d0 100644 --- a/Src/Base/AMReX_CoordSys.H +++ b/Src/Base/AMReX_CoordSys.H @@ -54,7 +54,7 @@ public: BL_ASSERT(c_sys != undef); return (c_sys == RZ); } - //! Is CoordType == cartesion? + //! Is CoordType == cartesian? [[nodiscard]] bool IsCartesian () const noexcept { BL_ASSERT(c_sys != undef); return (c_sys == cartesian); } diff --git a/Src/Base/AMReX_DistributionMapping.H b/Src/Base/AMReX_DistributionMapping.H index 2cbd038b5d9..0707532a0fc 100644 --- a/Src/Base/AMReX_DistributionMapping.H +++ b/Src/Base/AMReX_DistributionMapping.H @@ -405,7 +405,7 @@ DistributionMapping MakeSimilarDM (const BoxArray& ba, const MultiFab& mf, const * of overlap with. * * @param[in] ba The BoxArray we want to generate a DistributionMapping for. - * @param[in] src_ba The BoxArray associatied with the src DistributionMapping. + * @param[in] src_ba The BoxArray associated with the src DistributionMapping. * @param[in] src_dm The input DistributionMapping we want the output to be similar to. * @param[in] ng The number of grow cells to use when computing intersection / overlap * @return The computed DistributionMapping. diff --git a/Src/Base/AMReX_FACopyDescriptor.H b/Src/Base/AMReX_FACopyDescriptor.H index 5194ca7f876..7e1e383d237 100644 --- a/Src/Base/AMReX_FACopyDescriptor.H +++ b/Src/Base/AMReX_FACopyDescriptor.H @@ -698,7 +698,7 @@ FabArrayCopyDescriptor::CollectData () amrex::The_Arena()->free(md_recv_data); } - // Wait and upack data + // Wait and unpack data if (N_rcvs > 0) { Vector stats(N_rcvs); diff --git a/Src/Base/AMReX_FArrayBox.H b/Src/Base/AMReX_FArrayBox.H index 45e49ebab93..084b38ce46b 100644 --- a/Src/Base/AMReX_FArrayBox.H +++ b/Src/Base/AMReX_FArrayBox.H @@ -23,7 +23,7 @@ class FArrayBox; * only want to write out 32 bit FABs. * * With the exception of the enumeration constants, this class is -* primarily for FArrayBox implementors; i.e. user's shouldn't +* primarily for FArrayBox implementers; i.e. user's shouldn't * call any of the member functions in this class directly. */ @@ -241,7 +241,7 @@ public: /** * \brief Construct an initial FAB with the data space allocated but - * not inititialized. ncomp is the number of components + * not initialized. ncomp is the number of components * (variables) at each data point in the Box. */ explicit FArrayBox (const Box& b, @@ -409,7 +409,7 @@ public: /** * \brief Set the FABio::Format in the program. * This is the preferred way to set the output format - * in "new" FABs. When desiging new programs, this should + * in "new" FABs. When designing new programs, this should * be the only function that needs to be called in order * to set the format. */ diff --git a/Src/Base/AMReX_FILCC_1D.F90 b/Src/Base/AMReX_FILCC_1D.F90 index 873f67f5941..76e90a8d7a1 100644 --- a/Src/Base/AMReX_FILCC_1D.F90 +++ b/Src/Base/AMReX_FILCC_1D.F90 @@ -14,7 +14,7 @@ !! corner of q array !! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) !! -!! NOTE: all corner as well as edge data is filled if not EXT_DIR +!! NOTE: all corner as well as edge data is filled if not EXT_DIR/EXT_DIR_CC ! ----------------------------------------------------------- #ifndef AMREX_XSDK diff --git a/Src/Base/AMReX_FILCC_2D.F90 b/Src/Base/AMReX_FILCC_2D.F90 index 89ef77d8384..c2ca8f7ddaf 100644 --- a/Src/Base/AMReX_FILCC_2D.F90 +++ b/Src/Base/AMReX_FILCC_2D.F90 @@ -16,7 +16,7 @@ !! corner of q array !! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) !! -!! NOTE: all corner as well as edge data is filled if not EXT_DIR +!! NOTE: all corner as well as edge data is filled if not EXT_DIR/EXT_DIR_CC ! ----------------------------------------------------------- subroutine filcc(q,q_l1,q_l2,q_h1,q_h2,domlo,domhi,dx,xlo,bc) diff --git a/Src/Base/AMReX_FILCC_3D.F90 b/Src/Base/AMReX_FILCC_3D.F90 index aa3fec74ab0..59ce83d469d 100644 --- a/Src/Base/AMReX_FILCC_3D.F90 +++ b/Src/Base/AMReX_FILCC_3D.F90 @@ -6,7 +6,7 @@ !> This routine is intended to be a generic fill function !! for cell centered data. It knows how to exrapolate, !! and reflect data and can be used to supplement problem -!! specific fill functions (ie. EXT_DIR). +!! specific fill functions (ie. EXT_DIR/EXT_DIR_CC). !! !! \param q <= array to fill !! \param q_l1,q_l2,q_l3,q_h1,q_h2,q_h3 => index extent of q array diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H index 59bb6b73807..a8839a4bcc0 100644 --- a/Src/Base/AMReX_FabArray.H +++ b/Src/Base/AMReX_FabArray.H @@ -723,7 +723,7 @@ public: * * \param comp component * \param nghost number of ghost cells - * \param local If true, MPI communciation is skipped. + * \param local If true, MPI communication is skipped. */ template ::value,int> = 0> typename F::value_type @@ -1205,7 +1205,7 @@ public: * \param comp starting component * \param ncomp number of components * \param nghost number of ghost cells - * \param local If true, MPI communciation is skipped. + * \param local If true, MPI communication is skipped. * \param ignore_covered ignore covered cells. Only relevant for cell-centered EB data. */ template ::value,int> = 0> @@ -1220,7 +1220,7 @@ public: * \param comp starting component * \param ncomp number of components * \param nghost number of ghost cells - * \param local If true, MPI communciation is skipped. + * \param local If true, MPI communication is skipped. */ template ::value,int> = 0> typename F::value_type diff --git a/Src/Base/AMReX_FabArrayUtility.H b/Src/Base/AMReX_FabArrayUtility.H index e1e62f08d04..78f3355d34a 100644 --- a/Src/Base/AMReX_FabArrayUtility.H +++ b/Src/Base/AMReX_FabArrayUtility.H @@ -1547,7 +1547,7 @@ indexFromValue (FabArray const& mf, int comp, IntVect const& nghost, * \param ycomp starting component of y * \param ncomp number of components * \param nghost number of ghost cells - * \param local If true, MPI communciation is skipped. + * \param local If true, MPI communication is skipped. */ template ::value,int> FOO = 0> typename FAB::value_type diff --git a/Src/Base/AMReX_FilCC_C.cpp b/Src/Base/AMReX_FilCC_C.cpp index 7cdba486957..e2d8c6129fd 100644 --- a/Src/Base/AMReX_FilCC_C.cpp +++ b/Src/Base/AMReX_FilCC_C.cpp @@ -41,7 +41,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, if (lo.x < ilo) { const int imin = lo.x; const int imax = ilo-1; - if (bc.lo(0) == BCType::ext_dir) { + if (bc.lo(0) == BCType::ext_dir || bc.lo(0) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.lo(0) == BCType::foextrap) { for (int k = lo.z; k <= hi.z; ++k) { @@ -88,7 +88,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, const int imin = ihi+1; const int imax = hi.x; - if (bc.hi(0) == BCType::ext_dir) { + if (bc.hi(0) == BCType::ext_dir || bc.hi(0) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.hi(0) == BCType::foextrap) { for (int k = lo.z; k <= hi.z; ++k) { @@ -136,7 +136,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, if (lo.y < jlo) { const int jmin = lo.y; const int jmax = jlo-1; - if (bc.lo(1) == BCType::ext_dir) { + if (bc.lo(1) == BCType::ext_dir || bc.lo(1) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.lo(1) == BCType::foextrap) { for (int k = lo.z; k <= hi.z; ++k) { @@ -182,7 +182,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, if (hi.y > jhi) { const int jmin = jhi+1; const int jmax = hi.y; - if (bc.hi(1) == BCType::ext_dir) { + if (bc.hi(1) == BCType::ext_dir || bc.hi(1) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.hi(1) == BCType::foextrap) { for (int k = lo.z; k <= hi.z; ++k) { @@ -231,7 +231,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, if (lo.z < klo) { const int kmin = lo.z; const int kmax = klo-1; - if (bc.lo(2) == BCType::ext_dir) { + if (bc.lo(2) == BCType::ext_dir || bc.lo(2) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.lo(2) == BCType::foextrap) { for (int k = kmin; k <= kmax; ++k) { @@ -277,7 +277,7 @@ void fab_filcc (Box const& bx, Array4 const& qn, int ncomp, if (hi.z > khi) { const int kmin = khi+1; const int kmax = hi.z; - if (bc.hi(2) == BCType::ext_dir) { + if (bc.hi(2) == BCType::ext_dir || bc.hi(2) == BCType::ext_dir_cc) { // Do nothing. } else if (bc.hi(2) == BCType::foextrap) { for (int k = kmin; k <= kmax; ++k) { diff --git a/Src/Base/AMReX_GpuControl.H b/Src/Base/AMReX_GpuControl.H index d6f7a55ffaf..0f21213a86d 100644 --- a/Src/Base/AMReX_GpuControl.H +++ b/Src/Base/AMReX_GpuControl.H @@ -150,7 +150,7 @@ namespace Gpu { } /** - * This struct provides a RAII-style mechansim for changing the number + * This struct provides a RAII-style mechanism for changing the number * of streams returned by Gpu::numStreams() to a single stream. */ struct [[nodiscard]] SingleStreamRegion @@ -166,7 +166,7 @@ namespace Gpu { }; /** - * This struct provides a RAII-style mechansim for disabling GPU + * This struct provides a RAII-style mechanism for disabling GPU * synchronization in MFITer by default. Note that explicit calls to * Gpu::steramSynchronize and Gpu::deviceSynchronize still work. */ diff --git a/Src/Base/AMReX_IArrayBox.H b/Src/Base/AMReX_IArrayBox.H index 4d39ace1012..b5240395f02 100644 --- a/Src/Base/AMReX_IArrayBox.H +++ b/Src/Base/AMReX_IArrayBox.H @@ -58,7 +58,7 @@ public: /** * \brief Construct an initial FAB with the data space allocated but - * not inititialized. ncomp is the number of components + * not initialized. ncomp is the number of components * (variables) at each data point in the Box. */ explicit IArrayBox (const Box& b, diff --git a/Src/Base/AMReX_IntVect.H b/Src/Base/AMReX_IntVect.H index 2b8900248a8..fd71c93ae87 100644 --- a/Src/Base/AMReX_IntVect.H +++ b/Src/Base/AMReX_IntVect.H @@ -108,7 +108,7 @@ public: /** * \brief Construct an IntVect from an Vector. It is an error if - * the Vector doesn' t have the same dimension as this + * the Vector doesn't have the same dimension as this * IntVect. */ explicit IntVect (const Vector& a) noexcept : vect{AMREX_D_DECL(a[0],a[1],a[2])} { diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp index 05881353ac4..93ba453cc07 100644 --- a/Src/Base/AMReX_MultiFabUtil.cpp +++ b/Src/Base/AMReX_MultiFabUtil.cpp @@ -335,7 +335,8 @@ namespace amrex amrex::average_down(S_fine, S_crse, scomp, ncomp, ratio); #else - AMREX_ASSERT(S_crse.nComp() == S_fine.nComp()); + AMREX_ASSERT(S_crse.nComp() == S_fine.nComp() && + S_fine.is_cell_centered() && S_crse.is_cell_centered()); // // Coarsen() the fine stuff on processors owning the fine data. diff --git a/Src/Base/AMReX_MultiFabUtilI.H b/Src/Base/AMReX_MultiFabUtilI.H index cb7e8892ee1..cae4b86aa3a 100644 --- a/Src/Base/AMReX_MultiFabUtilI.H +++ b/Src/Base/AMReX_MultiFabUtilI.H @@ -64,7 +64,7 @@ namespace amrex::MFUtil { //! Regrid (by duplication) class T (which must be either of type - //! MultiFab or iMultiFab) usign the new BoxArray and + //! MultiFab or iMultiFab) using the new BoxArray and //! DistributionMapping. The boolean flag `regrid_ghost` switched //! between the SymmetricGhost and AsymmetricGhost copy functions. template diff --git a/Src/Base/AMReX_NonLocalBC.H b/Src/Base/AMReX_NonLocalBC.H index fc7f4cb6583..f7f22a67198 100644 --- a/Src/Base/AMReX_NonLocalBC.H +++ b/Src/Base/AMReX_NonLocalBC.H @@ -264,7 +264,7 @@ struct MultiBlockCommMetaData : FabArrayBase::CommMetaData { // [concept.FabProjection] // -//! \brief This type trait tests if a type P is a projetion for FAB. +//! \brief This type trait tests if a type P is a projection for FAB. template struct IsFabProjection : IsCallableR, Dim3, int> diff --git a/Src/Base/AMReX_PODVector.H b/Src/Base/AMReX_PODVector.H index 8f71e1e0fd2..0f10dfb94ef 100644 --- a/Src/Base/AMReX_PODVector.H +++ b/Src/Base/AMReX_PODVector.H @@ -422,7 +422,7 @@ namespace amrex iterator insert (const_iterator a_pos, T&& a_item) { - // This is *POD* vector afterall + // This is *POD* vector after all return insert(a_pos, 1, a_item); } diff --git a/Src/Base/AMReX_ParReduce.H b/Src/Base/AMReX_ParReduce.H index 8cbe69345f9..ae5059b2399 100644 --- a/Src/Base/AMReX_ParReduce.H +++ b/Src/Base/AMReX_ParReduce.H @@ -14,7 +14,7 @@ namespace amrex { * the second MultiFab. \verbatim auto const& ma1 = mf1.const_arrays(); - auot const& ma2 = mf2.const_arrays(); + auto const& ma2 = mf2.const_arrays(); GpuTuple mm = ParReduce(TypeList{}, TypeList{}, mf1, mf1.nGrowVect(), @@ -29,7 +29,7 @@ namespace amrex { * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam Ts... data types (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list list of reduce operators * \param type_list list of data types @@ -79,7 +79,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam T data type (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list a reduce operator stored in TypeList * \param type_list a data type stored in TypeList @@ -113,7 +113,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * and the maximum of the second MultiFab. \verbatim auto const& ma1 = mf1.const_arrays(); - auot const& ma2 = mf2.const_arrays(); + auto const& ma2 = mf2.const_arrays(); GpuTuple mm = ParReduce(TypeList{}, TypeList{}, mf1, mf1.nGrowVect(), mf1.nComp(), @@ -128,7 +128,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam Ts... data types (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list list of reduce operators * \param type_list list of data types @@ -175,7 +175,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam T data type (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list a reduce operator stored in TypeList * \param type_list a data type stored in TypeList @@ -210,7 +210,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * the second MultiFab. \verbatim auto const& ma1 = mf1.const_arrays(); - auot const& ma2 = mf2.const_arrays(); + auto const& ma2 = mf2.const_arrays(); GpuTuple mm = ParReduce(TypeList{}, TypeList{}, mf1, [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept @@ -224,7 +224,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam Ts... data types (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list list of reduce operators * \param type_list list of data types @@ -268,7 +268,7 @@ ParReduce (TypeList operation_list, TypeList type_list, * ReduceOpLogicalAnd, and ReduceOpLogicalOr) * \tparam T data type (e.g., Real, int, etc.) * \tparam FAB MultiFab/FabArray type - * \tparam F callable type like a lambda funcion + * \tparam F callable type like a lambda function * * \param operation_list a reduce operator stored in TypeList * \param type_list a data type stored in TypeList diff --git a/Src/Base/AMReX_RealVect.H b/Src/Base/AMReX_RealVect.H index 83cc747b030..635d21927f6 100644 --- a/Src/Base/AMReX_RealVect.H +++ b/Src/Base/AMReX_RealVect.H @@ -521,13 +521,13 @@ public: /** This is a RealVect all of whose components are equal to zero. */ - static const RealVect Zero; + static AMREX_EXPORT const RealVect Zero; /// /** This is a RealVect all of whose components are equal to one. */ - static const RealVect Unit; + static AMREX_EXPORT const RealVect Unit; /*@}*/ diff --git a/Src/Base/AMReX_Utility.cpp b/Src/Base/AMReX_Utility.cpp index 5b84bb3ae6d..16f2f20b554 100644 --- a/Src/Base/AMReX_Utility.cpp +++ b/Src/Base/AMReX_Utility.cpp @@ -193,7 +193,7 @@ amrex::UniqueString() tempstring << std::setprecision(n) << std::fixed << amrex::second(); auto const ts = tempstring.str(); auto const tsl = ts.length(); - return ts.substr(tsl-len,tsl); // tsl-len >= 0 becaues n >= len + return ts.substr(tsl-len,tsl); // tsl-len >= 0 because n >= len } void diff --git a/Src/Base/AMReX_VisMF.H b/Src/Base/AMReX_VisMF.H index fd1473a052d..f0b146f6a9d 100644 --- a/Src/Base/AMReX_VisMF.H +++ b/Src/Base/AMReX_VisMF.H @@ -545,7 +545,7 @@ void read_fab (FAB& fab, VisMF::FabOnDisk const& fod, std::string const& name) * fully defined, the BoxArray on the disk must match the BoxArray in the * given iMultiFab/FabArray object. If it is only constructed * with the default constructor, the BoxArray on the disk will be used and a - * new DistributionMapping will be made. When this funciton is used to + * new DistributionMapping will be made. When this function is used to * restart a calculation from checkpoint files, one should use a fully * defined iMultiFab/FabArray except for the first one in a * series of iMultiFab/MultiFab objects that share the same diff --git a/Src/Base/AMReX_VisMF.cpp b/Src/Base/AMReX_VisMF.cpp index 52efe827460..912be0b9b99 100644 --- a/Src/Base/AMReX_VisMF.cpp +++ b/Src/Base/AMReX_VisMF.cpp @@ -2202,7 +2202,7 @@ void VisMF::CloseStream(const std::string &fileName, bool forceClose) pifs.pstr = nullptr; pifs.isOpen = false; } - pifs.ioBuffer.clear(); + VisMFBuffer::ClearBuffer(pifs.ioBuffer); } diff --git a/Src/Base/AMReX_VisMFBuffer.H b/Src/Base/AMReX_VisMFBuffer.H index 843e5713f34..f095dbbebf2 100644 --- a/Src/Base/AMReX_VisMFBuffer.H +++ b/Src/Base/AMReX_VisMFBuffer.H @@ -29,6 +29,10 @@ public: ioBufferSize = iobuffersize; } + static void ClearBuffer (IO_Buffer& buf) { + IO_Buffer().swap(buf); + } + protected: static AMREX_EXPORT Long ioBufferSize; //!< ---- the settable buffer size diff --git a/Src/Base/AMReX_bc_types.fi b/Src/Base/AMReX_bc_types.fi index 2f3340ccfc4..d929b589b3d 100644 --- a/Src/Base/AMReX_bc_types.fi +++ b/Src/Base/AMReX_bc_types.fi @@ -17,3 +17,5 @@ PARAMETER (HOEXTRAP=4) INTEGER HOEXTRAPCC PARAMETER (HOEXTRAPCC=5) + INTEGER EXT_DIR_CC + PARAMETER (EXT_DIR_CC=6) diff --git a/Src/Base/AMReX_bc_types_mod.F90 b/Src/Base/AMReX_bc_types_mod.F90 index c1c6f237ba8..5b8c58c255a 100644 --- a/Src/Base/AMReX_bc_types_mod.F90 +++ b/Src/Base/AMReX_bc_types_mod.F90 @@ -15,6 +15,7 @@ module amrex_bc_types_module integer, parameter, public :: amrex_bc_ext_dir = 3 integer, parameter, public :: amrex_bc_hoextrap = 4 integer, parameter, public :: amrex_bc_hoextrapcc = 5 + integer, parameter, public :: amrex_bc_ext_dir_cc = 6 integer, parameter, public :: amrex_bc_user_1 = 1001 integer, parameter, public :: amrex_bc_user_2 = 1002 integer, parameter, public :: amrex_bc_user_3 = 1003 diff --git a/Src/Base/AMReX_filcc_mod.F90 b/Src/Base/AMReX_filcc_mod.F90 index c1a2b2951f4..1cca9b84090 100644 --- a/Src/Base/AMReX_filcc_mod.F90 +++ b/Src/Base/AMReX_filcc_mod.F90 @@ -165,7 +165,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) imin = lo(1) imax = ilo-1 - if (bc(1,1,n) .eq. amrex_bc_ext_dir) then + if (bc(1,1,n) .eq. amrex_bc_ext_dir .or. bc(1,1,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. @@ -237,7 +237,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) imin = ihi+1 imax = hi(1) - if (bc(1,2,n) .eq. amrex_bc_ext_dir) then + if (bc(1,2,n) .eq. amrex_bc_ext_dir .or. bc(1,2,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. @@ -311,7 +311,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) jmin = lo(2) jmax = jlo-1 - if (bc(2,1,n) .eq. amrex_bc_ext_dir) then + if (bc(2,1,n) .eq. amrex_bc_ext_dir .or. bc(2,1,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. @@ -383,7 +383,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) jmin = jhi+1 jmax = hi(2) - if (bc(2,2,n) .eq. amrex_bc_ext_dir) then + if (bc(2,2,n) .eq. amrex_bc_ext_dir .or. bc(2,2,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. @@ -461,7 +461,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) kmin = lo(3) kmax = klo-1 - if (bc(3,1,n) .eq. amrex_bc_ext_dir) then + if (bc(3,1,n) .eq. amrex_bc_ext_dir .or. bc(3,1,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. @@ -533,7 +533,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) kmin = khi+1 kmax = hi(3) - if (bc(3,2,n) .eq. amrex_bc_ext_dir) then + if (bc(3,2,n) .eq. amrex_bc_ext_dir .or. bc(3,2,n) .eq. amrex_bc_ext_dir_cc) then ! Do nothing. diff --git a/Src/Boundary/AMReX_InterpBndryData.H b/Src/Boundary/AMReX_InterpBndryData.H index c2b57ba6114..6dabc8ac481 100644 --- a/Src/Boundary/AMReX_InterpBndryData.H +++ b/Src/Boundary/AMReX_InterpBndryData.H @@ -82,7 +82,7 @@ public: * * \param crse BndryRegister storing coarse level data * \param c_start starting component of the coarse data - * \param fine MF containing physical bounadry values + * \param fine MF containing physical boundary values * \param f_start starting component of the fine data * \param bnd_start starting component in this InterpBndryData * \param num_comp number of component diff --git a/Src/Boundary/AMReX_YAFluxRegister.H b/Src/Boundary/AMReX_YAFluxRegister.H index 6f17c4de556..075a630a2f7 100644 --- a/Src/Boundary/AMReX_YAFluxRegister.H +++ b/Src/Boundary/AMReX_YAFluxRegister.H @@ -84,7 +84,7 @@ public: crse_cell = 0, crse_fine_boundary_cell, fine_cell }; - //! For curvilinear cooordinates only. In that case, the flux passed to + //! For curvilinear coordinates only. In that case, the flux passed to //! YAFluxRegister is assumed to have been multiplied by area. Note that //! YAFluxRegister does NOT make a copy of the volume data. So the //! coarse volume MF must be alive during the life time of diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index b77b2ebd00e..b2bbde200c5 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -9,7 +9,7 @@ void set_eb_data (const int i, const int j, GpuArray const& dx, Array4 const& vfrac, Array4 const& vcent, Array4 const& barea, Array4 const& bcent, - Array4 const& bnorm) noexcept + Array4 const& bnorm, Array4 const& levset) noexcept { #ifdef AMREX_USE_FLOAT constexpr Real almostone = 1.0_rt-1.e-6_rt; @@ -37,7 +37,20 @@ void set_eb_data (const int i, const int j, const Real nyabs = std::abs(ny); Real x_ym, x_yp, y_xm, y_xp; - if (nx > 0.0_rt) { + if (nx == 0.0_rt) { + if (apx(i,j,0) == 1.0_rt && apx(i+1,j,0) == 1.0_rt) { + if (levset(i,j,0) > 0.0_rt || levset(i,j+1,0) > 0.0_rt) { + x_ym = 0.5_rt*dx[0] - aym; + x_yp = 0.5_rt*dx[0] - ayp; + } else { + x_ym = -0.5_rt*dx[0] + aym; + x_yp = -0.5_rt*dx[0] + ayp; + } + } else { + x_ym = 0.0_rt; + x_yp = 0.0_rt; + } + } else if (nx > 0.0_rt) { x_ym = -0.5_rt*dx[0] + aym; x_yp = -0.5_rt*dx[0] + ayp; } else { @@ -45,7 +58,20 @@ void set_eb_data (const int i, const int j, x_yp = 0.5_rt*dx[0] - ayp; } - if (ny > 0.0_rt) { + if (ny == 0.0_rt) { + if (apy(i,j,0) == 1.0_rt && apy(i,j+1,0) == 1.0_rt) { + if (levset(i,j,0) > 0.0_rt || levset(i+1,j,0) > 0.0_rt) { + y_xm = 0.5_rt*dx[1] - axm; + y_xp = 0.5_rt*dx[1] - axp; + } else { + y_xm = -0.5_rt*dx[1] + axm; + y_xp = -0.5_rt*dx[1] + axp; + } + } else { + y_xm = 0.0_rt; + y_xp = 0.0_rt; + } + } else if (ny > 0.0_rt) { y_xm = -0.5_rt*dx[1] + axm; y_xp = -0.5_rt*dx[1] + axp; } else { @@ -135,7 +161,8 @@ bool set_eb_cell (int i, int j, Array4 const& cell, GpuArray const& dx, Array4 const& vfrac, Array4 const& vcent, Array4 const& barea, Array4 const& bcent, - Array4 const& bnorm, Real small_volfrac) noexcept + Array4 const& bnorm, Array4 const& levset, + Real small_volfrac) noexcept { bool is_small_cell = false; if (cell(i,j,0).isRegular()) { @@ -157,7 +184,7 @@ bool set_eb_cell (int i, int j, Array4 const& cell, bnorm(i,j,0,0) = 0.0_rt; bnorm(i,j,0,1) = 0.0_rt; } else { - set_eb_data(i,j,apx,apy,dx,vfrac,vcent,barea,bcent,bnorm); + set_eb_data(i,j,apx,apy,dx,vfrac,vcent,barea,bcent,bnorm,levset); // remove small cells if (vfrac(i,j,0) < small_volfrac) { set_covered(i,j,cell,vfrac,vcent,barea,bcent,bnorm); @@ -341,7 +368,7 @@ void build_cells (Box const& bx, Array4 const& cell, { amrex::ignore_unused(k); bool is_small = set_eb_cell(i, j, cell, apx, apy, dx, vfrac, vcent, barea, bcent, - bnorm, small_volfrac); + bnorm, levset, small_volfrac); if (is_small) { Gpu::Atomic::Add(dp, 1); } diff --git a/Src/EB/AMReX_EB2_GeometryShop.H b/Src/EB/AMReX_EB2_GeometryShop.H index 974738e94f1..ee353c13952 100644 --- a/Src/EB/AMReX_EB2_GeometryShop.H +++ b/Src/EB/AMReX_EB2_GeometryShop.H @@ -476,7 +476,7 @@ public: { // interp might still be quiet_nan because lst that // was set to zero has been changed by FillBoundary - // at periodic bounadries. + // at periodic boundaries. inter(i,j,k) = problo[0] + i*dx[0]; } else if (lst(i+1,j,k) == Real(0.0) || diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 6c6f75f784b..cd7f8fc6083 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -427,7 +427,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, small_volfrac, geom, extend_domain_face, cover_multiple_cuts, nsm, nmc); - // Becasue it is used in a synchronous reduction kernel in + // Because it is used in a synchronous reduction kernel in // build_cells, we do not need to worry about M2's lifetime. // But we still need to use Elixir to extend the life of // cellflagtmp. diff --git a/Src/EB/AMReX_EB_STL_utils.cpp b/Src/EB/AMReX_EB_STL_utils.cpp index 46a07fc9d9c..b45ef1cad30 100644 --- a/Src/EB/AMReX_EB_STL_utils.cpp +++ b/Src/EB/AMReX_EB_STL_utils.cpp @@ -699,7 +699,7 @@ STLtools::updateIntercept (Array,AMREX_SPACEDIM> const& inter_arr, { // interp might still be quiet_nan because lst that // was set to zero has been changed by FillBoundary - // at periodic bounadries. + // at periodic boundaries. inter(i,j,k) = problo[0] + static_cast(i)*dx[0]; } else if (lst(i+1,j,k) == Real(0.0) || diff --git a/Src/EB/AMReX_EB_StateRedistribute.cpp b/Src/EB/AMReX_EB_StateRedistribute.cpp index 79e661d2c90..faa77539f90 100644 --- a/Src/EB/AMReX_EB_StateRedistribute.cpp +++ b/Src/EB/AMReX_EB_StateRedistribute.cpp @@ -176,7 +176,7 @@ MLStateRedistribute ( Box const& bx, int ncomp, if (domain_per_grown.contains(IntVect(AMREX_D_DECL(r,s,t)))) { - // Initialize so that the slope stencil goes from -1:1 in each diretion + // Initialize so that the slope stencil goes from -1:1 in each direction int nx = 1; int ny = 1; int nz = 1; // Do we have enough extent in each coordinate direction to use the 3x3x3 stencil @@ -511,7 +511,7 @@ StateRedistribute ( Box const& bx, int ncomp, bool extdir_khi = (d_bcrec_ptr[n].hi(2) == amrex::BCType::ext_dir || d_bcrec_ptr[n].hi(2) == amrex::BCType::hoextrap); #endif - // Initialize so that the slope stencil goes from -1:1 in each diretion + // Initialize so that the slope stencil goes from -1:1 in each direction int nx = 1; int ny = 1; int nz = 1; // Do we have enough extent in each coordinate direction to use the 3x3x3 stencil diff --git a/Src/EB/AMReX_EB_utils.H b/Src/EB/AMReX_EB_utils.H index 10c5246c9e6..73c11ac95a3 100644 --- a/Src/EB/AMReX_EB_utils.H +++ b/Src/EB/AMReX_EB_utils.H @@ -50,9 +50,9 @@ namespace amrex { * that the distance is valid only if it's within a few cells to the EB. * * \param mf is a nodal MultiFab. - * \param ls_lev is an EB2::Level obejct with an implicit function. This + * \param ls_lev is an EB2::Level object with an implicit function. This * is at the same level as mf. - * \param eb_fac is an EBFArrayBoxFactory object containing EB informaiton. + * \param eb_fac is an EBFArrayBoxFactory object containing EB information. * \param refratio is the refinement ratio of mf to eb_fac. * \param fluid_has_positive_sign determines the sign of the fluid. */ diff --git a/Src/Extern/Bittree/AMReX_Bittree.H b/Src/Extern/Bittree/AMReX_Bittree.H new file mode 100644 index 00000000000..54a046be720 --- /dev/null +++ b/Src/Extern/Bittree/AMReX_Bittree.H @@ -0,0 +1,74 @@ +#ifndef BL_BITTREE_H_ +#define BL_BITTREE_H_ + +#include +#include +#include +#include + +namespace amrex { + +/* +Include in Make.local: +BITTREE_PATH = /path/to/bittree/installation +INCLUDE_LOCATIONS += $(BITTREE_PATH)/include +LIBRARY_LOCATIONS += $(BITTREE_PATH)/lib +LIBRARIES += -lbittree + +Include in inputs: +amr.use_bittree = true +*/ + +class btUnit { + // Functions used in AmrMesh + public: + static int btRefine (bittree::BittreeAmr* mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + static void btCalculateGrids (bittree::BittreeAmr* mesh, + int lbase, + int& new_finest, + Vector& new_grids, + Vector const& max_grid_size); + static void btCalculateLevel (bittree::BittreeAmr* mesh, + int lev, + BoxArray& ba, + IntVect const& max_grid_size); + // Utils + static int getBitid (bittree::BittreeAmr* mesh, bool updated, + int lev, int idx_on_lev); + static int getIndex (bittree::BittreeAmr* mesh, bool updated, + int lev, int bitid); + + // Functions to implement strict octree logic + private: + static void btCheckRefine (bittree::BittreeAmr* mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + + static void btCheckDerefine (bittree::BittreeAmr* mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + + // Utility Functions + static bool checkNeighborsRefine (bittree::BittreeAmr* mesh, + bittree::MortonTree::Block b); + static std::vector neighIntCoords (bittree::BittreeAmr* mesh, + unsigned lev, unsigned const* lcoord, + int const* gCell); + + public: + // Represents whether domain has periodic BC in each direction + // true = Periodic, false = Non-Periodic + static bool bcPeriodic[AMREX_SPACEDIM]; +}; + + +} +#endif diff --git a/Src/Extern/Bittree/AMReX_Bittree.cpp b/Src/Extern/Bittree/AMReX_Bittree.cpp new file mode 100644 index 00000000000..51bbe199278 --- /dev/null +++ b/Src/Extern/Bittree/AMReX_Bittree.cpp @@ -0,0 +1,388 @@ +#include +#include +#include +#include + +using namespace bittree; + +namespace amrex { +static constexpr auto K1D = int(AMREX_SPACEDIM>=1); +static constexpr auto K2D = int(AMREX_SPACEDIM>=2); +static constexpr auto K3D = int(AMREX_SPACEDIM>=3); + + +bool btUnit::bcPeriodic[AMREX_SPACEDIM]; + +/* +NOTE: Bittree object is created in AmrMesh::MakeNewGrids (Real time) + with + `mesh = std::make_shared(top,includes);` + +The functions here are called in the BT version of MakeNewGrids which has three steps: + 1. Error Estimation and tagging - btTagging + 2. Bitree's actual bitmap generated/updated - btRefine + 3. AMReX updates grids based on bitree - btCalculateGrids +*/ + + +/** New Bittree mesh is generated. + * + * This makes use of BT library functions and as well as routines adapted + * from Flash-X that enforce Octree nesting. + */ +int btUnit::btRefine (BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, MPI_Comm comm) +{ + BL_PROFILE("Bittree-btRefine"); + + // Tree before refinement. With only one rank, lnblocks = nblocks. + auto tree0 = mesh->getTree(); + + // Mark leaves to be refined + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if (btTags[id]==1) { + if(!tree0->block_is_parent(id)) { + mesh->refine_mark(id, true); + } + } + } + } + + mesh->refine_reduce(comm); + mesh->refine_update(); + + btCheckRefine(mesh, btTags, max_crse, lbase, grids, dmap, comm); + + // Mark derefinement (parents who will nodetype change to leaf) + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if (btTags[id]==-1) { + if(tree0->block_is_parent(id)) { + mesh->refine_mark(id, true); + } + } + } + } + + mesh->refine_reduce(comm); + mesh->refine_update(); + + btCheckDerefine(mesh, btTags, max_crse, lbase, grids, dmap, comm); + + // return delta count + return static_cast( mesh->delta_count() ); +} + +/** Creates new box arrays to match the new Bittree mesh. + */ +void btUnit::btCalculateGrids (BittreeAmr* const mesh, int lbase, + int& new_finest, + Vector& new_grids, + Vector const& max_grid_size) +{ + BL_PROFILE("Bittree-btCalculateGrids"); + + auto tree1 = mesh->getTree(true); + auto nlevs = tree1->levels(); + new_finest = int(nlevs - 1); + +//--Calculate the new grid layout and distribution map based on Bittree + for(int lev=lbase; lev<=new_finest; ++lev) { + btCalculateLevel(mesh, lev, new_grids[lev], + max_grid_size[lev]); + } +} + +/** Creates a box array based on Bittree. + */ +void btUnit::btCalculateLevel (BittreeAmr* const mesh, int lev, + BoxArray& ba, + IntVect const& max_grid_size) +{ + auto tree1 = mesh->getTree(true); + + //Bittree has its own indices for blocks which I call bitid; get + //the range of bitids for the level being made. Bitid range is + //contiguous for each level. + auto id0 = tree1->level_id0(lev); + auto id1 = tree1->level_id1(lev); + // int nblocks = tree1->level_blocks(lev); + + BoxList bl; + + for(auto i=id0; ilocate(i); + + if(b.level != lev) { + std::string msg = "Error identifying block in btCalculateGrids"; + //throw error? + } + + IntVect coordVec{AMREX_D_DECL(static_cast(b.coord[0]), + static_cast(b.coord[1]), + static_cast(b.coord[2]))}; + IntVect lo = max_grid_size*coordVec; + IntVect hi = max_grid_size*(coordVec+1) - 1; + bl.push_back( Box{lo,hi} ); + } + + ba = BoxArray(bl); +} + +int btUnit::getBitid (BittreeAmr* const mesh, bool updated, + int lev, int idx_on_lev) +{ + return idx_on_lev + int(mesh->getTree(updated)->level_id0(lev)); +} + +int btUnit::getIndex (BittreeAmr* const mesh, bool updated, + int lev, int bitid) +{ + return bitid - int(mesh->getTree(updated)->level_id0(lev)); +} + + + +//--------------------------------------------------------------------- +// Local Routines +//--------------------------------------------------------------------- + +/** Implements the logic which ensures the generated Bittree adheres + * to a strict octree structure with no more than one level difference + * between surrounding leaf blocks. + */ +void btUnit::btCheckRefine (BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, + Vector& dmap, MPI_Comm comm) +{ + BL_PROFILE("Bittree-btCheckRefine"); + + // Tree before refinement. + auto tree0 = mesh->getTree(); + + // Ref test is marked 1 if block needs a tag (and doesn't have one). + std::vector ref_test(tree0->id_upper_bound()); + + // Repeat is made true if another round is needed + bool repeat = false; + + do { + // Clear out ref_test + std::fill(ref_test.begin(),ref_test.end(),0); + + // Check neighbors - if any adjacent child of a neighbor is either a parent + // or marked for refinement, this block needs to be refined. + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + auto b = tree0->locate(id); + if( !b.is_parent && btTags[id]!=1 ) { + bool needsTag = checkNeighborsRefine( mesh, b); + //amrex::Print() << "needsTag for " << id << " : " << needsTag <=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if( ref_test[id]==1 && btTags[id]!=1 ) { + repeat = true; + btTags[id] = 1; + mesh->refine_mark(id,true); + } + } + } + + // If only processing local blocks, check all processors to see if + // a repeat is necessary, then reduce bittree to update on all ranks. + ParallelDescriptor::ReduceBoolOr(repeat); + + if(repeat) { + mesh->refine_reduce(comm); + mesh->refine_update(); + } + + } while(repeat); +} + + +/** Implements the logic which ensures the generated Bittree adheres + * to a strict octree structure with no more than one level difference + * between surrounding leaf blocks. + */ +void btUnit::btCheckDerefine (BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, + Vector& dmap, MPI_Comm comm) +{ + BL_PROFILE("Bittree-btCheckDerefine"); + + // Tree before refinement. With only one rank, lnblocks = nblocks. + auto tree0 = mesh->getTree(); + + std::vector deref_test(tree0->id_upper_bound()); + + // Repeat is made true if another round is needed + bool repeat = false; + + // Repeat is left true if another round is needed + do { + // Turn deref_test to default 0 if block can't be derefined + deref_test = btTags; + + // Check neighbors - if any adjacent child of neighbor is either a parent + // or marked for refinement, do not derefine. + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + auto b = tree0->locate(id); + if( btTags[id]==-1 ) { + bool cantDeref = checkNeighborsRefine( mesh, b); + if(cantDeref) { + deref_test[id] = 0; + } + } + } + } + + // Unmark any blocks who cannot derefine (as per above check). + repeat = false; + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if( deref_test[id]==0 && btTags[id]==-1 ) { + repeat = true; + btTags[id] = 0; + + // Unmark for derefinement + mesh->refine_mark(id, false); + } + } + } + + // If only processing local blocks, check all processors to see if + // a repeat is necessary, then reduce bittree to update on all ranks. + ParallelDescriptor::ReduceBoolOr(repeat); + + if(repeat) { + mesh->refine_reduce_and(comm); + mesh->refine_update(); + } + + } while(repeat); +} + + +// Check all neighbors to see if their adjacent children are parents or marked for refinement. +bool btUnit::checkNeighborsRefine (BittreeAmr* const mesh, MortonTree::Block b) +{ + BL_PROFILE("Bittree-checkNeighborsRefine"); + + auto tree0 = mesh->getTree(); + auto tree1 = mesh->getTree(true); + int nIdx[3], cIdx[3]; + unsigned childCoord_u[AMREX_SPACEDIM]; + + // Loop over neighbors + for(nIdx[2]= -1*K3D; nIdx[2]<= K3D; ++nIdx[2]) { + for(nIdx[1]= -1*K2D; nIdx[1]<= K2D; ++nIdx[1]) { + for(nIdx[0]= -1*K1D; nIdx[0]<= K1D; ++nIdx[0]) { + std::vector nCoord = neighIntCoords(mesh, b.level, b.coord, nIdx); + + // If neighbor is outside domain or otherwise invalid, continue. + if(AMREX_D_TERM(nCoord[0]<0, || nCoord[1]<0, || nCoord[2]<0 )) { + continue; + } + + // Identify neighbor from Bittree. + unsigned neighCoord_u[AMREX_SPACEDIM]; + for(unsigned d=0; d(nCoord[d]); + } + auto n = tree0->identify(b.level, neighCoord_u); + if(b.level==n.level && n.is_parent) { + // Loop over children of neighbor. + for(cIdx[2]= 0; cIdx[2]<= K3D; ++cIdx[2]) { + for(cIdx[1]= 0; cIdx[1]<= K2D; ++cIdx[1]) { + for(cIdx[0]= 0; cIdx[0]<= K1D; ++cIdx[0]) { + + // Only check adjacent children + if (( ((1-nIdx[0])/2)==cIdx[0] || nIdx[0] == 0 ) && + ( ((1-nIdx[1])/2)==cIdx[1] || nIdx[1] == 0 ) && + ( ((1-nIdx[2])/2)==cIdx[2] || nIdx[2] == 0 )) { + + // Identify child on updated tree + for(unsigned d=0; d(cIdx[d]); + } + auto c = tree1->identify(n.level+1, childCoord_u); + + // If child WILL be parent, return true + if( c.level==(b.level+1) && c.is_parent) { + return true; + } + } + }}} + } + }}} + + // Return false otherwise + return false; +} + +/** Calculate integer coordinates of neighbors, taking into account BCs. + * Currently assuming Periodic in all directions. + */ +std::vector btUnit::neighIntCoords (BittreeAmr* const mesh, + unsigned lev, unsigned const* lcoord, + int const* gCell) +{ + auto tree = mesh->getTree(); + + std::vector neighCoord(AMREX_SPACEDIM); + +//--Calculate integer coordinates of neighbor in direction + for(unsigned d=0;d(lcoord[d]) + gCell[d]; + } + +//--Make sure not out-of-bounds. If periodic BCs, apply modulo + std::vector maxcoord(AMREX_SPACEDIM); + for(unsigned d=0;d(tree->top_size(d)) << lev; + } + + for(unsigned d=0;d= maxcoord[d]) { + if ( bcPeriodic[d] == true ) { + neighCoord[d] = neighCoord[d] - maxcoord[d]; + } else { + neighCoord[d] = -1; + } + } + } + + return neighCoord; +} + +} diff --git a/Src/Extern/Bittree/CMakeLists.txt b/Src/Extern/Bittree/CMakeLists.txt new file mode 100644 index 00000000000..d51454b2172 --- /dev/null +++ b/Src/Extern/Bittree/CMakeLists.txt @@ -0,0 +1,11 @@ +target_include_directories( amrex + PUBLIC + $) + +add_amrex_define(AMREX_USE_BITTREE NO_LEGACY) + +target_sources( amrex + PRIVATE + AMReX_Bittree.H + AMReX_Bittree.cpp + ) diff --git a/Src/Extern/Bittree/Make.package b/Src/Extern/Bittree/Make.package new file mode 100644 index 00000000000..0aefb6f300a --- /dev/null +++ b/Src/Extern/Bittree/Make.package @@ -0,0 +1,5 @@ +CEXE_headers += AMReX_Bittree.H +CEXE_sources += AMReX_Bittree.cpp + +VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Bittree +INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Extern/Bittree diff --git a/Src/Extern/HDF5/AMReX_ParticlesHDF5.H b/Src/Extern/HDF5/AMReX_ParticlesHDF5.H index 0e5321f7ace..c72f6e22523 100644 --- a/Src/Extern/HDF5/AMReX_ParticlesHDF5.H +++ b/Src/Extern/HDF5/AMReX_ParticlesHDF5.H @@ -172,7 +172,7 @@ public: /** * \brief This version of WritePlotFile assigns component names, but allows the user to pass - * in a vector of ints that toggle on / off the writing of specfic components. + * in a vector of ints that toggle on / off the writing of specific components. * * \param dir The base directory into which to write (i.e. "plt00000") * \param file The name of the sub-directory for this particle type (i.e. "Tracer") @@ -187,7 +187,7 @@ public: /** * \brief This version of WritePlotFile assigns component names, but allows the user to pass - * in a vector of ints that toggle on / off the writing of specfic components. + * in a vector of ints that toggle on / off the writing of specific components. * * This version also lets you pass in a functor to toggle whether each particle gets output. * diff --git a/Src/Extern/HYPRE/AMReX_HypreNodeLap.cpp b/Src/Extern/HYPRE/AMReX_HypreNodeLap.cpp index 79b35a5d58f..448869ca359 100644 --- a/Src/Extern/HYPRE/AMReX_HypreNodeLap.cpp +++ b/Src/Extern/HYPRE/AMReX_HypreNodeLap.cpp @@ -47,7 +47,7 @@ HypreNodeLap::HypreNodeLap (const BoxArray& grids_, const DistributionMapping& d Int nnodes_proc = fill_local_node_id(); // At this point, local_node_id stores the ids local to each box. - // nnodes_grid stroes the number of nodes in each box. nnodes_proc is + // nnodes_grid stores the number of nodes in each box. nnodes_proc is // the number of nodes on this MPI process. If a nodal is invalid, its // id is invalid (i.e., a very negative number). Note that the data // type of local_node_id is int, not HYPRE_Int for performance on GPU. diff --git a/Src/Extern/HYPRE/AMReX_HypreSolver.H b/Src/Extern/HYPRE/AMReX_HypreSolver.H index 205518f1f16..039373d5ab1 100644 --- a/Src/Extern/HYPRE/AMReX_HypreSolver.H +++ b/Src/Extern/HYPRE/AMReX_HypreSolver.H @@ -220,7 +220,7 @@ HypreSolver::HypreSolver (Vector const& a_index_type, fill_local_id(a_marker); // At this point, m_local_id stores the ids local to each box. - // m_nrows_grid stroes the number of unique points in each box. + // m_nrows_grid stores the number of unique points in each box. // m_nrows_proc is the number of rowss for all variables on this MPI // process. If a point is invalid, its id is invalid (i.e., a very // negative number). Note that the data type of local_node_id is int, diff --git a/Src/Extern/SENSEI/AMReX_AmrMeshDataAdaptor.cpp b/Src/Extern/SENSEI/AMReX_AmrMeshDataAdaptor.cpp index 34b92c1d25d..48cd8012b30 100644 --- a/Src/Extern/SENSEI/AMReX_AmrMeshDataAdaptor.cpp +++ b/Src/Extern/SENSEI/AMReX_AmrMeshDataAdaptor.cpp @@ -405,7 +405,7 @@ int AmrMeshDataAdaptor::GetNumberOfArrays(const std::string &meshName, if (this->Internals->States.size() < 1) { - SENSEI_ERROR("No simualtion data, missing call to SetDataSource?") + SENSEI_ERROR("No simulation data, missing call to SetDataSource?") return -1; } @@ -462,7 +462,7 @@ int AmrMeshDataAdaptor::GetMeshHasGhostCells(const std::string &meshName, int &n if (this->Internals->States.size() < 1) { - SENSEI_ERROR("No simualtion data, missing call to SetDataSource?") + SENSEI_ERROR("No simulation data, missing call to SetDataSource?") return -1; } @@ -489,7 +489,7 @@ int AmrMeshDataAdaptor::GetMesh(const std::string &meshName, if (this->Internals->States.size() < 1) { - SENSEI_ERROR("No simualtion data, missing call to SetDataSource?") + SENSEI_ERROR("No simulation data, missing call to SetDataSource?") return -1; } @@ -631,7 +631,7 @@ int AmrMeshDataAdaptor::AddGhostCellsArray(svtkDataObject* mesh, if (this->Internals->States.size() < 1) { - SENSEI_ERROR("No simualtion data, missing call to SetDataSource?") + SENSEI_ERROR("No simulation data, missing call to SetDataSource?") return -1; } @@ -756,7 +756,7 @@ int AmrMeshDataAdaptor::AddArray(svtkDataObject* mesh, if (this->Internals->States.size() < 1) { - SENSEI_ERROR("No simualtion data, missing call to SetDataSource?") + SENSEI_ERROR("No simulation data, missing call to SetDataSource?") return -1; } diff --git a/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H b/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H index 0a85cb7efa9..b8d9dc112a7 100644 --- a/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H +++ b/Src/Extern/SENSEI/AMReX_AmrMeshParticleInSituBridge.H @@ -21,7 +21,7 @@ namespace amrex { -/// SENSEI bridge code for simulations proccessing both amrex::Amr and amrex::ParticleContainer +/// SENSEI bridge code for simulations processing both amrex::Amr and amrex::ParticleContainer class AmrMeshParticleInSituBridge : public InSituBridge { public: diff --git a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H index bb5a573c11d..262089694ab 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H @@ -581,14 +581,14 @@ int ParticleDataAdaptor::GetMeshMetadata( for (const auto& kv : pmap) { // loop over particles in ParticleTile - const auto& aos = kv.second.GetArrayOfStructs(); - const auto &parts = aos(); + auto& particle_tile = kv.second; + auto ptd = particle_tile.getConstParticleTileData(); // visit only the "real" particles, skip the "neighbor" particles. - long long numReal = aos.numRealParticles(); + long long numReal = particle_tile.numRealParticles(); for (long long i = 0; i < numReal; ++i) { - const auto &part = parts[i]; + const auto &part = ptd[i]; if (part.id() > 0) { #if (AMREX_SPACEDIM == 1) @@ -869,7 +869,7 @@ int ParticleDataAdaptor::AddParticlesSOARea for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order @@ -881,7 +881,7 @@ int ParticleDataAdaptor::AddParticlesSOARea for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = ptd[i]; if (part.id() > 0) { pData[i*nComps + j] = realData[i]; @@ -944,7 +944,7 @@ int ParticleDataAdaptor::AddParticlesSOAInt for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order @@ -953,7 +953,7 @@ int ParticleDataAdaptor::AddParticlesSOAInt for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = ptd[i]; if (part.id() > 0) { pData[i] = intData[i]; diff --git a/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H b/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H index 9f8cccedaa7..2623f832590 100644 --- a/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H +++ b/Src/Extern/SUNDIALS/AMReX_SundialsIntegrator.H @@ -354,7 +354,7 @@ public: } // Clean up allocated memory - for (int i = 0; i < Nvar; ++i) { + for (int i = 0; i < NVar; ++i) { N_VDestroy(nv_many_arr[i]); } delete[] nv_many_arr; @@ -638,7 +638,7 @@ public: } // Clean up allocated memory - for (int i = 0; i < Nvar; ++i) { + for (int i = 0; i < NVar; ++i) { N_VDestroy(nv_many_arr[i]); } delete[] nv_many_arr; @@ -758,7 +758,7 @@ public: } // Clean up allocated memory - for (int i = 0; i < Nvar; ++i) { + for (int i = 0; i < NVar; ++i) { N_VDestroy(nv_many_arr[i]); } delete[] nv_many_arr; diff --git a/Src/Extern/SUNDIALS/AMReX_Sundials_Core.cpp b/Src/Extern/SUNDIALS/AMReX_Sundials_Core.cpp index 286662058a3..4533a155ccd 100644 --- a/Src/Extern/SUNDIALS/AMReX_Sundials_Core.cpp +++ b/Src/Extern/SUNDIALS/AMReX_Sundials_Core.cpp @@ -14,7 +14,7 @@ void Initialize(int nthreads) { amrex::Print() << "Initializing SUNDIALS with " << nthreads << " threads...\n"; - // Initalize the sundials context + // Initialize the sundials context if (initialized.empty()) { initialized.resize(nthreads); std::fill(initialized.begin(), initialized.end(), 0); diff --git a/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp b/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp index 93bf211dc33..d59c2d02053 100644 --- a/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp @@ -11,6 +11,22 @@ extern "C" amrex::average_down(*S_fine, *S_crse, *fgeom, *cgeom, scomp, ncomp, rr); } + void amrex_fi_average_down_faces (MultiFab const* fmf[], MultiFab* cmf[], + Geometry const* cgeom, int scomp, int ncomp, + int rr) + { + Array fine + {AMREX_D_DECL(MultiFab(*fmf[0], amrex::make_alias, scomp, ncomp), + MultiFab(*fmf[1], amrex::make_alias, scomp, ncomp), + MultiFab(*fmf[2], amrex::make_alias, scomp, ncomp))}; + Array crse + {AMREX_D_DECL(MultiFab(*cmf[0], amrex::make_alias, scomp, ncomp), + MultiFab(*cmf[1], amrex::make_alias, scomp, ncomp), + MultiFab(*cmf[2], amrex::make_alias, scomp, ncomp))}; + amrex::average_down_faces(GetArrOfConstPtrs(fine), GetArrOfPtrs(crse), + IntVect(rr), *cgeom); + } + void amrex_fi_average_cellcenter_to_face (MultiFab* fc[], const MultiFab* cc, const Geometry* geom) { Vector fcv {AMREX_D_DECL(fc[0], fc[1], fc[2])}; diff --git a/Src/F_Interfaces/Base/AMReX_multifabutil_mod.F90 b/Src/F_Interfaces/Base/AMReX_multifabutil_mod.F90 index 267108f1f81..9575b217a73 100644 --- a/Src/F_Interfaces/Base/AMReX_multifabutil_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_multifabutil_mod.F90 @@ -8,7 +8,7 @@ module amrex_multifabutil_module implicit none private - public :: amrex_average_down, amrex_average_cellcenter_to_face + public :: amrex_average_down, amrex_average_down_faces, amrex_average_cellcenter_to_face interface subroutine amrex_fi_average_down (fmf, cmf, fgeom, cgeom, scomp, ncomp, rr) bind(c) @@ -18,6 +18,14 @@ subroutine amrex_fi_average_down (fmf, cmf, fgeom, cgeom, scomp, ncomp, rr) bind integer(c_int), value :: scomp, ncomp, rr end subroutine amrex_fi_average_down + subroutine amrex_fi_average_down_faces (fmf, cmf, cgeom, scomp, ncomp, rr) bind(c) + import + implicit none + type(c_ptr), intent(in) :: fmf(*), cmf(*) + type(c_ptr), value :: cgeom + integer(c_int), value :: scomp, ncomp, rr + end subroutine amrex_fi_average_down_faces + subroutine amrex_fi_average_cellcenter_to_face (facemf, ccmf, geom) bind(c) import implicit none @@ -38,6 +46,21 @@ subroutine amrex_average_down (fmf, cmf, fgeom, cgeom, scomp, ncomp, rr) end subroutine amrex_average_down + subroutine amrex_average_down_faces (fmf, cmf, cgeom, scomp, ncomp, rr) + type(amrex_multifab), intent(in ) :: fmf(amrex_spacedim) + type(amrex_multifab), intent(inout) :: cmf(amrex_spacedim) + type(amrex_geometry), intent(in) :: cgeom ! coarse level geometry + integer, intent(in) :: scomp, ncomp, rr + type(c_ptr) :: cp(amrex_spacedim), fp(amrex_spacedim) + integer :: idim + do idim = 1, amrex_spacedim + fp(idim) = fmf(idim)%p + cp(idim) = cmf(idim)%p + end do + call amrex_fi_average_down_faces(fp, cp, cgeom%p, scomp-1, ncomp, rr) + end subroutine amrex_average_down_faces + + subroutine amrex_average_cellcenter_to_face (facemf, ccmf, geom) type(amrex_multifab), intent(inout) :: facemf(amrex_spacedim) type(amrex_multifab), intent(in) :: ccmf diff --git a/Src/LinearSolvers/CMakeLists.txt b/Src/LinearSolvers/CMakeLists.txt index 500716be3a0..c2851d49959 100644 --- a/Src/LinearSolvers/CMakeLists.txt +++ b/Src/LinearSolvers/CMakeLists.txt @@ -49,6 +49,10 @@ foreach(D IN LISTS AMReX_SPACEDIM) MLMG/AMReX_MLEBNodeFDLaplacian.cpp MLMG/AMReX_MLEBNodeFDLap_K.H MLMG/AMReX_MLEBNodeFDLap_${D}D_K.H + MLMG/AMReX_MLNodeABecLaplacian.H + MLMG/AMReX_MLNodeABecLaplacian.cpp + MLMG/AMReX_MLNodeABecLap_K.H + MLMG/AMReX_MLNodeABecLap_${D}D_K.H ) if (D EQUAL 3) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H index 32164f54657..706fe679d7e 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H @@ -130,7 +130,7 @@ public: * * This functions must be called, and must be called before other bc * functions. This version is for single-component solve or when all the - * compoents have the same BC types. + * components have the same BC types. * * \param lobc lower boundaries * \param hibc upper boundaries @@ -262,7 +262,7 @@ public: virtual void restriction (int amrlev, int cmglev, MF& crse, MF& fine) const = 0; /** - * \brief Add interpolated coarse MG levle data to fine MG level data + * \brief Add interpolated coarse MG level data to fine MG level data * * \param amrlev AMR level * \param fmglev fine MG level @@ -355,7 +355,7 @@ public: * \param resid residual * \param x unknown in the residual-correction form * \param b RHS in the residual-correction form - * \param bc_mode Is the BC homegeneous or inhomogeneous? + * \param bc_mode Is the BC homogeneous or inhomogeneous? * \param crse_bc_data optional argument providing BC at coarse/fine boundary. */ virtual void correctionResidual (int amrlev, int mglev, MF& resid, MF& x, const MF& b, @@ -406,7 +406,7 @@ public: //! This is needed for our nodal projection solver virtual void unimposeNeumannBC (int /*amrlev*/, MF& /*rhs*/) const {} - //! Extra terms introduced when we treate inhomogeneous Nuemann BC as homogeneous. + //! Extra terms introduced when we treat inhomogeneous Nuemann BC as homogeneous. virtual void applyInhomogNeumannTerm (int /*amrlev*/, MF& /*rhs*/) const {} //! for overset solver only diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_1D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_1D_K.H new file mode 100644 index 00000000000..34a2ddda6f7 --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_1D_K.H @@ -0,0 +1,30 @@ +#ifndef AMREX_MLNODEABECLAP_1D_K_H_ +#define AMREX_MLNODEABECLAP_1D_K_H_ + +namespace amrex { + +inline void +mlndabeclap_gauss_seidel_aa (Box const& /*bx*/, Array4 const& /*sol*/, + Array4 const& /*rhs*/, + Real /*alpha*/, Real /*beta*/, + Array4 const& /*acf*/, + Array4 const& /*bcf*/, + Array4 const& /*msk*/, + GpuArray const& /*dxinv*/) noexcept +{} + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE void +mlndabeclap_jacobi_aa (int /*i*/, int /*j*/, int /*k*/, + Array4 const& /*sol*/, + Real /*lap*/, + Array4 const& /*rhs*/, + Real /*alpha*/, Real /*beta*/, + Array4 const& /*acf*/, + Array4 const& /*bcf*/, + Array4 const& /*msk*/, + GpuArray const& /*dxinv*/) noexcept +{} + +} + +#endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_2D_K.H new file mode 100644 index 00000000000..3418b19d279 --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_2D_K.H @@ -0,0 +1,67 @@ +#ifndef AMREX_MLNODEABECLAP_2D_K_H_ +#define AMREX_MLNODEABECLAP_2D_K_H_ + +namespace amrex { + +inline void +mlndabeclap_gauss_seidel_aa (Box const& bx, Array4 const& sol, + Array4 const& rhs, + Real alpha, Real beta, + Array4 const& acf, + Array4 const& bcf, + Array4 const& msk, + GpuArray const& dxinv) noexcept +{ + Real facx = Real(1.0/6.0)*dxinv[0]*dxinv[0]; + Real facy = Real(1.0/6.0)*dxinv[1]*dxinv[1]; + Real fxy = facx + facy; + Real f2xmy = Real(2.0)*facx - facy; + Real fmx2y = Real(2.0)*facy - facx; + + amrex::Loop(bx, [=] (int i, int j, int k) noexcept + { + if (msk(i,j,k)) { + sol(i,j,k) = Real(0.0); + } else { + Real s0 = (-Real(2.0))*fxy*(bcf(i-1,j-1,k)+bcf(i,j-1,k)+bcf(i-1,j,k)+bcf(i,j,k)); + Real lap = sol(i-1,j-1,k)*fxy*bcf(i-1,j-1,k) + + sol(i+1,j-1,k)*fxy*bcf(i ,j-1,k) + + sol(i-1,j+1,k)*fxy*bcf(i-1,j ,k) + + sol(i+1,j+1,k)*fxy*bcf(i ,j ,k) + + sol(i-1,j,k)*f2xmy*(bcf(i-1,j-1,k)+bcf(i-1,j,k)) + + sol(i+1,j,k)*f2xmy*(bcf(i ,j-1,k)+bcf(i ,j,k)) + + sol(i,j-1,k)*fmx2y*(bcf(i-1,j-1,k)+bcf(i,j-1,k)) + + sol(i,j+1,k)*fmx2y*(bcf(i-1,j ,k)+bcf(i,j ,k)) + + sol(i,j,k)*s0; + Real Ax = alpha*acf(i,j,k)*sol(i,j,k) - beta*lap; + + sol(i,j,k) += (rhs(i,j,k) - Ax) / (alpha*acf(i,j,k)-beta*s0); + } + }); +} + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE void +mlndabeclap_jacobi_aa (int i, int j, int k, Array4 const& sol, + Real lap, Array4 const& rhs, + Real alpha, Real beta, + Array4 const& acf, + Array4 const& bcf, + Array4 const& msk, + GpuArray const& dxinv) noexcept +{ + if (msk(i,j,k)) { + sol(i,j,k) = Real(0.0); + } else { + Real fac = -Real(2.0/6.0)*(dxinv[0]*dxinv[0] + dxinv[1]*dxinv[1]); + Real s0 = fac*(bcf(i-1,j-1,k)+bcf(i,j-1,k)+bcf(i-1,j,k)+bcf(i,j,k)); + Real Ax = alpha*acf(i,j,k)*sol(i,j,k) - beta*lap; + + sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax) + / (alpha*acf(i,j,k)-beta*s0); + } + +} + +} + +#endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_3D_K.H new file mode 100644 index 00000000000..5ddb93a958c --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_3D_K.H @@ -0,0 +1,93 @@ +#ifndef AMREX_MLNODEABECLAP_3D_K_H_ +#define AMREX_MLNODEABECLAP_3D_K_H_ + +namespace amrex { + +inline void +mlndabeclap_gauss_seidel_aa (Box const& bx, Array4 const& sol, + Array4 const& rhs, + Real alpha, Real beta, + Array4 const& acf, + Array4 const& bcf, + Array4 const& msk, + GpuArray const& dxinv) noexcept +{ + Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0]; + Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1]; + Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2]; + Real fxyz = facx + facy + facz; + Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz; + Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz; + Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz; + Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz; + Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz; + Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz; + + amrex::LoopOnCpu(bx, [=] (int i, int j, int k) noexcept + { + if (msk(i,j,k)) { + sol(i,j,k) = Real(0.0); + } else { + Real s0 = Real(-4.0)*fxyz*(bcf(i-1,j-1,k-1)+bcf(i,j-1,k-1)+bcf(i-1,j,k-1)+bcf(i,j,k-1) + +bcf(i-1,j-1,k )+bcf(i,j-1,k )+bcf(i-1,j,k )+bcf(i,j,k )); + Real lap = sol(i,j,k)*s0 + + fxyz*(sol(i-1,j-1,k-1)*bcf(i-1,j-1,k-1) + + sol(i+1,j-1,k-1)*bcf(i ,j-1,k-1) + + sol(i-1,j+1,k-1)*bcf(i-1,j ,k-1) + + sol(i+1,j+1,k-1)*bcf(i ,j ,k-1) + + sol(i-1,j-1,k+1)*bcf(i-1,j-1,k ) + + sol(i+1,j-1,k+1)*bcf(i ,j-1,k ) + + sol(i-1,j+1,k+1)*bcf(i-1,j ,k ) + + sol(i+1,j+1,k+1)*bcf(i ,j ,k )) + + fmx2y2z*(sol(i ,j-1,k-1)*(bcf(i-1,j-1,k-1)+bcf(i,j-1,k-1)) + + sol(i ,j+1,k-1)*(bcf(i-1,j ,k-1)+bcf(i,j ,k-1)) + + sol(i ,j-1,k+1)*(bcf(i-1,j-1,k )+bcf(i,j-1,k )) + + sol(i ,j+1,k+1)*(bcf(i-1,j ,k )+bcf(i,j ,k ))) + + f2xmy2z*(sol(i-1,j ,k-1)*(bcf(i-1,j-1,k-1)+bcf(i-1,j,k-1)) + + sol(i+1,j ,k-1)*(bcf(i ,j-1,k-1)+bcf(i ,j,k-1)) + + sol(i-1,j ,k+1)*(bcf(i-1,j-1,k )+bcf(i-1,j,k )) + + sol(i+1,j ,k+1)*(bcf(i ,j-1,k )+bcf(i ,j,k ))) + + f2x2ymz*(sol(i-1,j-1,k )*(bcf(i-1,j-1,k-1)+bcf(i-1,j-1,k)) + + sol(i+1,j-1,k )*(bcf(i ,j-1,k-1)+bcf(i ,j-1,k)) + + sol(i-1,j+1,k )*(bcf(i-1,j ,k-1)+bcf(i-1,j ,k)) + + sol(i+1,j+1,k )*(bcf(i ,j ,k-1)+bcf(i ,j ,k))) + + f4xm2ym2z*(sol(i-1,j,k)*(bcf(i-1,j-1,k-1)+bcf(i-1,j,k-1)+bcf(i-1,j-1,k)+bcf(i-1,j,k)) + + sol(i+1,j,k)*(bcf(i ,j-1,k-1)+bcf(i ,j,k-1)+bcf(i ,j-1,k)+bcf(i ,j,k))) + + fm2x4ym2z*(sol(i,j-1,k)*(bcf(i-1,j-1,k-1)+bcf(i,j-1,k-1)+bcf(i-1,j-1,k)+bcf(i,j-1,k)) + + sol(i,j+1,k)*(bcf(i-1,j ,k-1)+bcf(i,j ,k-1)+bcf(i-1,j ,k)+bcf(i,j ,k))) + + fm2xm2y4z*(sol(i,j,k-1)*(bcf(i-1,j-1,k-1)+bcf(i,j-1,k-1)+bcf(i-1,j,k-1)+bcf(i,j,k-1)) + + sol(i,j,k+1)*(bcf(i-1,j-1,k )+bcf(i,j-1,k )+bcf(i-1,j,k )+bcf(i,j,k ))); + Real Ax = alpha*acf(i,j,k)*sol(i,j,k) - beta*lap; + + sol(i,j,k) += (rhs(i,j,k) - Ax) / (alpha*acf(i,j,k)-beta*s0); + } + }); +} + +AMREX_GPU_DEVICE AMREX_FORCE_INLINE void +mlndabeclap_jacobi_aa (int i, int j, int k, Array4 const& sol, + Real lap, Array4 const& rhs, + Real alpha, Real beta, + Array4 const& acf, + Array4 const& bcf, + Array4 const& msk, + GpuArray const& dxinv) noexcept +{ + if (msk(i,j,k)) { + sol(i,j,k) = Real(0.0); + } else { + Real fxyz = Real(-4.0 / 36.0)*(dxinv[0]*dxinv[0] + + dxinv[1]*dxinv[1] + + dxinv[2]*dxinv[2]); + Real s0 = fxyz*(bcf(i-1,j-1,k-1)+bcf(i,j-1,k-1)+bcf(i-1,j,k-1)+bcf(i,j,k-1) + +bcf(i-1,j-1,k )+bcf(i,j-1,k )+bcf(i-1,j,k )+bcf(i,j,k)); + Real Ax = alpha*acf(i,j,k)*sol(i,j,k) - beta*lap; + + sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax) + / (alpha*acf(i,j,k)-beta*s0); + } +} + +} + +#endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_K.H new file mode 100644 index 00000000000..fd744bacd1e --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLap_K.H @@ -0,0 +1,13 @@ +#ifndef AMREX_MLNODEABECLAP_K_H_ +#define AMREX_MLNODEABECLAP_K_H_ +#include + +#if (AMREX_SPACEDIM == 1) +#include +#elif (AMREX_SPACEDIM == 2) +#include +#else +#include +#endif + +#endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.H new file mode 100644 index 00000000000..6caed940fa9 --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.H @@ -0,0 +1,88 @@ +#ifndef AMREX_MLNODEABECLAPLACIAN_H_ +#define AMREX_MLNODEABECLAPLACIAN_H_ +#include + +#include + +namespace amrex { + +// (alpha * a - beta * (del dot b grad)) phi = rhs +// a, phi and rhs are nodal. b is cell-centered. + +class MLNodeABecLaplacian + : public MLNodeLinOp +{ +public: + + MLNodeABecLaplacian () = default; + MLNodeABecLaplacian (const Vector& a_geom, + const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info = LPInfo(), + const Vector const*>& a_factory = {}); + ~MLNodeABecLaplacian () override = default; + + MLNodeABecLaplacian (const MLNodeABecLaplacian&) = delete; + MLNodeABecLaplacian (MLNodeABecLaplacian&&) = delete; + MLNodeABecLaplacian& operator= (const MLNodeABecLaplacian&) = delete; + MLNodeABecLaplacian& operator= (MLNodeABecLaplacian&&) = delete; + + void define (const Vector& a_geom, + const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info = LPInfo(), + const Vector const*>& a_factory = {}); + + std::string name () const override { return std::string("MLNodeABecLaplacian"); } + + void setScalars (Real a, Real b) { + m_a_scalar = a; + m_b_scalar = b; + } + + void setACoeffs (int amrlev, Real a_acoef); + void setACoeffs (int amrlev, const MultiFab& a_acoef); + + void setBCoeffs (int amrlev, Real a_bcoef); + void setBCoeffs (int amrlev, const MultiFab& a_bcoef); + + void Fapply (int amrlev, int mglev, MultiFab& out, const MultiFab& in) const final; + void Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs) const final; + + void fixUpResidualMask (int amrlev, iMultiFab& resmsk) final; + + bool isSingular (int /*amrlev*/) const final { return false; } + bool isBottomSingular () const final { return false; } + + void restriction (int amrlev, int cmglev, MultiFab& crse, MultiFab& fine) const final; + void interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const final; + void averageDownSolutionRHS (int camrlev, MultiFab& crse_sol, MultiFab& crse_rhs, + const MultiFab& fine_sol, const MultiFab& fine_rhs) final; + + void reflux (int crse_amrlev, + MultiFab& res, const MultiFab& crse_sol, const MultiFab& crse_rhs, + MultiFab& fine_res, MultiFab& fine_sol, const MultiFab& fine_rhs) const final; + + void prepareForSolve () final; + + [[nodiscard]] bool needsUpdate () const final { return m_needs_update; } + + void update () final; + + void averageDownCoeffs (); + void averageDownCoeffsToCoarseAmrLevel (int flev); + void averageDownCoeffsSameAmrLevel (int amrlev); + +private: + + bool m_needs_update = true; + + Real m_a_scalar = std::numeric_limits::quiet_NaN(); + Real m_b_scalar = std::numeric_limits::quiet_NaN(); + Vector > m_a_coeffs; + Vector > m_b_coeffs; +}; + +} + +#endif diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.cpp new file mode 100644 index 00000000000..07c516992f6 --- /dev/null +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeABecLaplacian.cpp @@ -0,0 +1,348 @@ +#include +#include +#include + +namespace amrex { + +MLNodeABecLaplacian::MLNodeABecLaplacian (const Vector& a_geom, + const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info, + const Vector const*>& a_factory) +{ + define(a_geom, a_grids, a_dmap, a_info, a_factory); +} + +void +MLNodeABecLaplacian::define (const Vector& a_geom, + const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info, + const Vector const*>& a_factory) +{ +#ifdef AMREX_USE_EB + amrex::Abort("MLNodeABecLaplacian does not support EB"); +#endif + + BL_PROFILE("MLNodeABecLaplacian::define()"); + + // This makes sure grids are cell-centered; + Vector cc_grids = a_grids; + for (auto& ba : cc_grids) { + ba.enclosedCells(); + } + + MLNodeLinOp::define(a_geom, cc_grids, a_dmap, a_info, a_factory); + + const int ncomp = getNComp(); + + m_a_coeffs.resize(m_num_amr_levels); + m_b_coeffs.resize(m_num_amr_levels); + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) { + m_a_coeffs[amrlev].resize(m_num_mg_levels[amrlev]); + m_b_coeffs[amrlev].resize(m_num_mg_levels[amrlev]); + for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) { + m_a_coeffs[amrlev][mglev].define + (amrex::convert(m_grids[amrlev][mglev], IntVect::TheNodeVector()), + m_dmap[amrlev][mglev], ncomp, 0); + m_b_coeffs[amrlev][mglev].define + (m_grids[amrlev][mglev], m_dmap[amrlev][mglev], ncomp, 1); + } + } +} + +void +MLNodeABecLaplacian::setACoeffs (int amrlev, Real a_acoef) +{ + m_a_coeffs[amrlev][0].setVal(a_acoef); + m_needs_update = true; +} + +void +MLNodeABecLaplacian::setACoeffs (int amrlev, const MultiFab& a_acoef) +{ + const int ncomp = getNComp(); + m_a_coeffs[amrlev][0].LocalCopy(a_acoef, 0, 0, ncomp, IntVect(0)); + m_needs_update = true; +} + +void +MLNodeABecLaplacian::setBCoeffs (int amrlev, Real a_bcoef) +{ + m_b_coeffs[amrlev][0].setVal(a_bcoef); + m_needs_update = true; +} + +void +MLNodeABecLaplacian::setBCoeffs (int amrlev, const MultiFab& a_bcoef) +{ + const int ncomp = getNComp(); + m_b_coeffs[amrlev][0].LocalCopy(a_bcoef, 0, 0, ncomp, IntVect(0)); + m_needs_update = true; +} + +void +MLNodeABecLaplacian::Fapply (int amrlev, int mglev, MultiFab& out, const MultiFab& in) const +{ + BL_PROFILE("MLNodeLaplacian::Fapply()"); + + AMREX_ALWAYS_ASSERT(getNComp() == 1); + + auto const alpha = m_a_scalar; + auto const beta = m_b_scalar; + const auto dxinvarr = m_geom[amrlev][mglev].InvCellSizeArray(); + + auto const& acoef_ma = m_a_coeffs[amrlev][mglev].const_arrays(); + auto const& bcoef_ma = m_b_coeffs[amrlev][mglev].const_arrays(); + auto const& dmskarr_ma = m_dirichlet_mask[amrlev][mglev]->const_arrays(); + + auto const& xarr_ma = in.const_arrays(); + auto const& yarr_ma = out.arrays(); + + ParallelFor(out, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + { + auto lap = mlndlap_adotx_aa(i,j,k,xarr_ma[box_no],bcoef_ma[box_no],dmskarr_ma[box_no], +#if (AMREX_SPACEDIM == 2) + false, +#endif + dxinvarr); + yarr_ma[box_no](i,j,k) = (dmskarr_ma[box_no](i,j,k)) ? Real(0.0) + : alpha*acoef_ma[box_no](i,j,k)*xarr_ma[box_no](i,j,k) - beta*lap; + }); + Gpu::streamSynchronize(); +} + +void +MLNodeABecLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs) const +{ + BL_PROFILE("MLNodeABecLaplacian::Fsmooth()"); + + auto const alpha = m_a_scalar; + auto const beta = m_b_scalar; + const auto dxinvarr = m_geom[amrlev][mglev].InvCellSizeArray(); + + auto const& acoef = m_a_coeffs[amrlev][mglev]; + auto const& bcoef = m_b_coeffs[amrlev][mglev]; + auto const& dmsk = *(m_dirichlet_mask[amrlev][mglev]); + +#ifdef AMREX_USE_GPU + + auto const& acoef_ma = acoef.const_arrays(); + auto const& bcoef_ma = bcoef.const_arrays(); + auto const& dmskarr_ma = dmsk.const_arrays(); + auto const& solarr_ma = sol.arrays(); + auto const& rhsarr_ma = rhs.const_arrays(); + + for (int ns = 0; ns < m_smooth_num_sweeps; ++ns) { + ParallelFor(sol, [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept + { + auto lap = mlndlap_adotx_aa(i,j,k,solarr_ma[box_no],bcoef_ma[box_no],dmskarr_ma[box_no], +#if (AMREX_SPACEDIM == 2) + false, +#endif + dxinvarr); + mlndabeclap_jacobi_aa(i,j,k, solarr_ma[box_no], lap, rhsarr_ma[box_no], alpha, beta, + acoef_ma[box_no], bcoef_ma[box_no], + dmskarr_ma[box_no], dxinvarr); + }); + Gpu::streamSynchronize(); + if (m_smooth_num_sweeps > 1) { nodalSync(amrlev, mglev, sol); } + } +#else + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + for (MFIter mfi(sol); mfi.isValid(); ++mfi) { + const Box& bx = mfi.validbox(); + Array4 const& aarr = acoef.array(mfi); + Array4 const& barr = bcoef.array(mfi); + Array4 const& solarr = sol.array(mfi); + Array4 const& rhsarr = rhs.const_array(mfi); + Array4 const& dmskarr = dmsk.const_array(mfi); + for (int ns = 0; ns < m_smooth_num_sweeps; ++ns) { + mlndabeclap_gauss_seidel_aa(bx, solarr, rhsarr, alpha, beta, + aarr, barr, dmskarr, dxinvarr); + } + } + nodalSync(amrlev, mglev, sol); +#endif +} + +void +MLNodeABecLaplacian::restriction (int amrlev, int cmglev, MultiFab& crse, MultiFab& fine) const +{ + BL_PROFILE("MLNodeABecLaplacian::restriction()"); + + applyBC(amrlev, cmglev-1, fine, BCMode::Homogeneous, StateMode::Solution); + + bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); + MultiFab cfine; + if (need_parallel_copy) { + const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); + cfine.define(ba, fine.DistributionMap(), 1, 0); + } + + MultiFab* pcrse = (need_parallel_copy) ? &cfine : &crse; + + auto pcrse_ma = pcrse->arrays(); + auto fine_ma = fine.const_arrays(); + auto msk_ma = m_dirichlet_mask[amrlev][cmglev-1]->const_arrays(); + + ParallelFor(*pcrse, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + { + mlndlap_restriction(i,j,k,pcrse_ma[box_no],fine_ma[box_no],msk_ma[box_no]); + }); + Gpu::streamSynchronize(); + + if (need_parallel_copy) { + crse.ParallelCopy(cfine); + } +} + +void +MLNodeABecLaplacian::interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const +{ + BL_PROFILE("MLNodeABecLaplacian::interpolation()"); + + bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); + MultiFab cfine; + const MultiFab* cmf = &crse; + if (need_parallel_copy) { + const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); + cfine.define(ba, fine.DistributionMap(), 1, 0); + cfine.ParallelCopy(crse); + cmf = &cfine; + } + + auto const& fine_ma = fine.arrays(); + auto const& crse_ma = cmf->const_arrays(); + auto const& msk_ma = m_dirichlet_mask[amrlev][fmglev]->const_arrays(); + auto const& sig_ma = m_b_coeffs[amrlev][fmglev].const_arrays(); + + ParallelFor(fine, [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept + { + mlndlap_interpadd_aa(i, j, k, fine_ma[box_no], crse_ma[box_no], + sig_ma[box_no], msk_ma[box_no]); + }); + Gpu::streamSynchronize(); +} + +void +MLNodeABecLaplacian::averageDownSolutionRHS (int camrlev, MultiFab& crse_sol, MultiFab& crse_rhs, + const MultiFab& fine_sol, const MultiFab& fine_rhs) +{ + amrex::ignore_unused(camrlev,crse_sol,crse_rhs,fine_sol,fine_rhs); + amrex::Abort("MLNodeABecLaplacian::averageDownSolutionRHS TODO"); +} + +void +MLNodeABecLaplacian::reflux (int crse_amrlev, + MultiFab& res, const MultiFab& crse_sol, const MultiFab& crse_rhs, + MultiFab& fine_res, MultiFab& fine_sol, const MultiFab& fine_rhs) const +{ + amrex::ignore_unused(crse_amrlev,res,crse_sol,crse_rhs,fine_res,fine_sol,fine_rhs); + amrex::Abort("MLNodeABecLaplacian::reflux TODO"); +} + +void +MLNodeABecLaplacian::prepareForSolve () +{ + BL_PROFILE("MLNodeABecLaplacian::prepareForSolve()"); + + MLNodeLinOp::prepareForSolve(); + + buildMasks(); + + averageDownCoeffs(); + + m_needs_update = false; +} + +void +MLNodeABecLaplacian::update () +{ + BL_PROFILE("MLNodeABecLaplacian::prepareForSolve()"); + averageDownCoeffs(); + m_needs_update = false; +} + +void +MLNodeABecLaplacian::fixUpResidualMask (int amrlev, iMultiFab& resmsk) +{ + if (!m_masks_built) { buildMasks(); } + + auto const& fmsk = m_nd_fine_mask[amrlev]->const_arrays(); + auto const& rmsk = resmsk.arrays(); + + amrex::ParallelFor(resmsk, + [=] AMREX_GPU_DEVICE (int bno, int i, int j, int k) + { + if (fmsk[bno](i,j,k) == crse_fine_node) { rmsk[bno](i,j,k) = 1; } + }); + Gpu::streamSynchronize(); +} + +void +MLNodeABecLaplacian::averageDownCoeffs () +{ + BL_PROFILE("MLNodeABecLaplacian::averageDownCoeffs()"); + + for (int amrlev = m_num_amr_levels-1; amrlev > 0; --amrlev) { + averageDownCoeffsSameAmrLevel(amrlev); + averageDownCoeffsToCoarseAmrLevel(amrlev); + } + + averageDownCoeffsSameAmrLevel(0); + + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) { + for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) { + m_b_coeffs[amrlev][mglev].FillBoundary(m_geom[amrlev][mglev].periodicity()); + + const Box& domain = m_geom[amrlev][mglev].Domain(); + const auto lobc = LoBC(); + const auto hibc = HiBC(); + + MFItInfo mfi_info; + if (Gpu::notInLaunchRegion()) { mfi_info.SetDynamic(true); } +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(m_b_coeffs[amrlev][mglev], mfi_info); mfi.isValid(); ++mfi) + { + Array4 const& sfab = m_b_coeffs[amrlev][mglev].array(mfi); + mlndlap_fillbc_cc(mfi.validbox(),sfab,domain,lobc,hibc); + } + } + } +} + +void +MLNodeABecLaplacian::averageDownCoeffsToCoarseAmrLevel (int flev) +{ + const int mglev = 0; + const int ncomp = getNComp(); + // xxxxx TODO: There is a potential issue of the coarse data not consistent + // across periodic boundaries. + amrex::average_down_nodal(m_a_coeffs[flev ][mglev], + m_a_coeffs[flev-1][mglev], + IntVect(m_amr_ref_ratio[flev-1])); + amrex::average_down(m_b_coeffs[flev ][mglev], + m_b_coeffs[flev-1][mglev], 0, ncomp, + m_amr_ref_ratio[flev-1]); +} + +void +MLNodeABecLaplacian::averageDownCoeffsSameAmrLevel (int amrlev) +{ + const int ncomp = getNComp(); + for (int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev) { + IntVect ratio(mg_coarsen_ratio); + amrex::average_down_nodal(m_a_coeffs[amrlev][mglev-1], + m_a_coeffs[amrlev][mglev ], ratio); + amrex::average_down(m_b_coeffs[amrlev][mglev-1], + m_b_coeffs[amrlev][mglev ], 0, ncomp, ratio); + } +} + +} diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H index 1de55f8a63d..d1ae9e0b7ed 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H @@ -2282,97 +2282,35 @@ void mlndlap_set_integral_eb (int i, int j, int, Array4 const& intg, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_surface_integral_eb (int i, int j, int, Array4 const& sintg, - Array4 const& flag, Array4 const& vol, - Array4 const& ax, Array4 const& ay, + Array4 const& flag, Array4 const& bcen, - Array4 const& barea) noexcept + Array4 const& barea, + Array4 const& bnorm) noexcept { if (flag(i,j,0).isCovered() || flag(i,j,0).isRegular()) { sintg(i,j,0,i_B_x ) = Real(0.); sintg(i,j,0,i_B_y ) = Real(0.); sintg(i,j,0,i_B_xy) = Real(0.); } else { - Real axm = ax(i,j,0); - Real axp = ax(i+1,j,0); - Real aym = ay(i,j,0); - Real ayp = ay(i,j+1,0); + Real bcx = bcen(i,j,0,0); + Real bcy = bcen(i,j,0,1); - Real apnorm = std::sqrt((axm-axp)*(axm-axp) + (aym-ayp)*(aym-ayp)); - if (apnorm == Real(0.)) { - amrex::Abort("amrex_mlndlap_set_surface_integral: we are in trouble"); - } - - if (vol(i,j,0) >= almostone) { - sintg(i,j,0,i_B_x ) = Real(0.); - sintg(i,j,0,i_B_y ) = Real(0.); - sintg(i,j,0,i_B_xy) = Real(0.); - if (axm < Real(1.)) { - sintg(i,j,0,i_B_x) = Real(-0.5)*barea(i,j,0); - } else if (aym < Real(1.)) { - sintg(i,j,0,i_B_y) = Real(-0.5)*barea(i,j,0); - } else if (axp < Real(1.)) { - sintg(i,j,0,i_B_x) = Real( 0.5)*barea(i,j,0); - } else if (ayp < Real(1.)) { - sintg(i,j,0,i_B_y) = Real( 0.5)*barea(i,j,0); - } else { - amrex::Abort("amrex_mlndlap_set_surface_integral: we are in trouble"); - } - } else { - Real apnorminv = Real(1.)/apnorm; - Real anrmx = (axm-axp) * apnorminv; // pointing to the wall - Real anrmy = (aym-ayp) * apnorminv; - - Real bcx = bcen(i,j,0,0); - Real bcy = bcen(i,j,0,1); - - Real c = -(bcx * anrmx + bcy * anrmy); - - GpuArray pts; //intersection points - int np = 0; - if (std::abs(anrmx) <= almostzero) { - pts[np++] = RealVect{Real(-0.5), Real(-c + Real(0.5)*anrmx)/anrmy}; - pts[np++] = RealVect{Real( 0.5), Real(-c - Real(0.5)*anrmx)/anrmy}; - } else if (std::abs(anrmy) <= almostzero) { - pts[np++] = RealVect{Real(-c + Real(0.5)*anrmy)/anrmx, Real(-0.5)}; - pts[np++] = RealVect{Real(-c - Real(0.5)*anrmy)/anrmx, Real( 0.5)}; - } else { - if ( (axm > Real(0.) && axm < Real(1.)) - || (axm > Real(0.) && aym == Real(0.)) - || (axm > Real(0.) && ayp == Real(0.))) { - pts[np++] = RealVect{Real(-0.5), Real(-c + Real(0.5)*anrmx)/anrmy}; - } - if ( (axp > Real(0.) && axp < Real(1.)) - || (axp > Real(0.) && aym == Real(0.)) - || (axp > Real(0.) && ayp == Real(0.))) { - pts[np++] = RealVect{Real( 0.5), Real(-c - Real(0.5)*anrmx)/anrmy}; - } - if ( (aym > Real(0.) && aym < Real(1.)) - || (aym > Real(0.) && axm == Real(0.)) - || (aym > Real(0.) && axp == Real(0.))) { - pts[np++] = RealVect{Real(-c + Real(0.5)*anrmy)/anrmx, Real(-0.5)}; - } - if ( (ayp > Real(0.) && ayp < Real(1.)) - || (ayp > Real(0.) && axm == Real(0.)) - || (ayp > Real(0.) && axp == Real(0.))) { - pts[np++] = RealVect{Real(-c - Real(0.5)*anrmy)/anrmx, Real( 0.5)}; - } - } + Real btanx = bnorm(i,j,0,1); + Real btany = -bnorm(i,j,0,0); - if (np != 2) { - amrex::Abort("amrex_mlndlap_set_surface_integral: we are in trouble"); - } + Real x0 = bcx - Real(0.5)*barea(i,j,0)*btanx; + Real x1 = bcx + Real(0.5)*barea(i,j,0)*btanx; - Real x0 = pts[0][0], x1 = pts[1][0]; - Real y0 = pts[0][1], y1 = pts[1][1]; + Real y0 = bcy - Real(0.5)*barea(i,j,0)*btany; + Real y1 = bcy + Real(0.5)*barea(i,j,0)*btany; - Real Bx = barea(i,j,0)*Real(0.5)*(x1 + x0); - Real By = barea(i,j,0)*Real(0.5)*(y1 + y0); - Real Bxy = barea(i,j,0)*(x0*y0 + (x0*(y1 - y0) + y0*(x1 - x0))/Real(2.) + (x1 - x0)*(y1 - y0)/Real(3.)); + Real Bx = barea(i,j,0)*Real(0.5)*(x1 + x0); + Real By = barea(i,j,0)*Real(0.5)*(y1 + y0); + Real Bxy = barea(i,j,0)*(x0*y0 + (x0*(y1 - y0) + y0*(x1 - x0))/Real(2.) + (x1 - x0)*(y1 - y0)/Real(3.)); - sintg(i,j,0,i_B_x ) = Bx; - sintg(i,j,0,i_B_y ) = By; - sintg(i,j,0,i_B_xy) = Bxy; - } + sintg(i,j,0,i_B_x ) = Bx; + sintg(i,j,0,i_B_y ) = By; + sintg(i,j,0,i_B_xy) = Bxy; } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H index 7ec9f13ce73..adbf00da231 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.H @@ -84,10 +84,6 @@ public : if (m_const_sigma == Real(0.0)) { m_coarsening_strategy = cs; } } - void setSmoothNumSweeps (int nsweeps) noexcept { - m_smooth_num_sweeps = nsweeps; - } - BottomSolver getDefaultBottomSolver () const final { return (m_coarsening_strategy == CoarseningStrategy::RAP) ? BottomSolver::bicgcg : BottomSolver::bicgstab; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp index 27e0f6c4b62..4a749a1ed08 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian.cpp @@ -230,7 +230,7 @@ MLNodeLaplacian::getSolvabilityOffset (int amrlev, int mglev, MultiFab const& rh if (m_lobc[0][idim] != LinOpBCType::Neumann && m_lobc[0][idim] != LinOpBCType::inflow) { - nddom.growLo(idim, 10); // so that the test in ParReduce will faill + nddom.growLo(idim, 10); // so that the test in ParReduce will fail } if (m_hibc[0][idim] != LinOpBCType::Neumann && m_hibc[0][idim] != LinOpBCType::inflow) @@ -334,7 +334,7 @@ MLNodeLaplacian::fixSolvabilityByOffset (int amrlev, int mglev, MultiFab& rhs, if (m_lobc[0][idim] != LinOpBCType::Neumann && m_lobc[0][idim] != LinOpBCType::inflow) { - nddom.growLo(idim, 10); // so that the test in ParReduce will faill + nddom.growLo(idim, 10); // so that the test in ParReduce will fail } if (m_hibc[0][idim] != LinOpBCType::Neumann && m_hibc[0][idim] != LinOpBCType::inflow) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_eb.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_eb.cpp index 49f80ad4c11..3088a29cb24 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_eb.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_eb.cpp @@ -99,10 +99,9 @@ MLNodeLaplacian::buildSurfaceIntegral () { const int ncomp = sintg->nComp(); const auto& flags = factory->getMultiEBCellFlagFab(); - const auto& vfrac = factory->getVolFrac(); - const auto& area = factory->getAreaFrac(); const auto& bcent = factory->getBndryCent(); const auto& barea = factory->getBndryArea(); + const auto& bnorm = factory->getBndryNormal(); MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) { mfi_info.EnableTiling().SetDynamic(true); } @@ -128,14 +127,12 @@ MLNodeLaplacian::buildSurfaceIntegral () }); } else { Array4 const& flagarr = flags.const_array(mfi); - Array4 const& vfracarr = vfrac.const_array(mfi); - Array4 const& axarr = area[0]->const_array(mfi); - Array4 const& ayarr = area[1]->const_array(mfi); Array4 const& bcarr = bcent.const_array(mfi); Array4 const& baarr = barea.const_array(mfi); + Array4 const& bnarr = bnorm.const_array(mfi); AMREX_HOST_DEVICE_FOR_3D(bx, i, j, k, { - mlndlap_set_surface_integral_eb(i,j,k,garr,flagarr,vfracarr,axarr,ayarr,bcarr,baarr); + mlndlap_set_surface_integral_eb(i,j,k,garr,flagarr,bcarr,baarr,bnarr); }); } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.H index 3c36989f79a..424d22f60c3 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.H @@ -33,6 +33,10 @@ public: const Vector const*>& a_factory = {}, int a_eb_limit_coarsening = -1); + void setSmoothNumSweeps (int nsweeps) noexcept { + m_smooth_num_sweeps = nsweeps; + } + void setLevelBC (int /*amrlev*/, const MultiFab* /*levelbcdata*/, const MultiFab* = nullptr, const MultiFab* = nullptr, const MultiFab* = nullptr) final {} diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp index 5e2389c1f93..0fb9e2ba33b 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.cpp @@ -368,7 +368,7 @@ MLNodeLinOp::buildMasks () MLNodeLinOp_set_dot_mask(m_bottom_dot_mask, omask, geom, lobc, hibc, m_coarsening_strategy); } - if (m_is_bottom_singular) + if (isBottomSingular()) { int amrlev = 0; int mglev = 0; diff --git a/Src/LinearSolvers/MLMG/Make.package b/Src/LinearSolvers/MLMG/Make.package index 22934a0d045..d66d64ec0eb 100644 --- a/Src/LinearSolvers/MLMG/Make.package +++ b/Src/LinearSolvers/MLMG/Make.package @@ -50,6 +50,9 @@ ifeq ($(USE_HYPRE),TRUE) CEXE_sources += AMReX_MLNodeLaplacian_hypre.cpp endif +CEXE_headers += AMReX_MLNodeABecLaplacian.H +CEXE_sources += AMReX_MLNodeABecLaplacian.cpp +CEXE_headers += AMReX_MLNodeABecLap_K.H AMReX_MLNodeABecLap_$(DIM)D_K.H CEXE_headers += AMReX_MLNodeTensorLaplacian.H CEXE_sources += AMReX_MLNodeTensorLaplacian.cpp diff --git a/Src/Particle/AMReX_DenseBins.H b/Src/Particle/AMReX_DenseBins.H index 67ff2c1c9c5..5ff9ecdc23f 100644 --- a/Src/Particle/AMReX_DenseBins.H +++ b/Src/Particle/AMReX_DenseBins.H @@ -111,7 +111,7 @@ public: * Finally, the set of partial sums is incremented in parallel using atomicInc, * which results in a permutation array that places the items in bin-sorted order. * - * This version uses a 3D box to specificy the index space over which the bins + * This version uses a 3D box to specify the index space over which the bins * are defined. * * This overload uses the "default" parallelization strategy. If AMReX has been @@ -166,7 +166,7 @@ public: * Finally, the set of partial sums is incremented in parallel using atomicInc, * which results in a permutation array that places the items in bin-sorted order. * - * This version uses a 3D box to specificy the index space over which the bins + * This version uses a 3D box to specify the index space over which the bins * are defined. * * This overload uses the "GPU" parallelization strategy. If AMReX has been @@ -269,11 +269,11 @@ public: * Finally, the set of partial sums is incremented in parallel using atomicInc, * which results in a permutation array that places the items in bin-sorted order. * - * This version uses a 3D box to specificy the index space over which the bins + * This version uses a 3D box to specify the index space over which the bins * are defined. * * This overload uses the "OpenMP" parallelization strategy and always runs on the - * host. If AMReX has been compiled with OpenMP support, the executation will be + * host. If AMReX has been compiled with OpenMP support, the execution will be * parallelized, otherwise it will be serial. * * \tparam N the 'size' type that can enumerate all the items @@ -315,7 +315,7 @@ public: * This version uses a 1D index space for the set of bins. * * This overload uses the "OpenMP" parallelization strategy and always runs on the - * host. If AMReX has been compiled with OpenMP support, the executation will be + * host. If AMReX has been compiled with OpenMP support, the execution will be * parallelized, otherwise it will be serial. * * \tparam N the 'size' type that can enumerate all the items @@ -408,7 +408,7 @@ public: * Finally, the set of partial sums is incremented in parallel using atomicInc, * which results in a permutation array that places the items in bin-sorted order. * - * This version uses a 3D box to specificy the index space over which the bins + * This version uses a 3D box to specify the index space over which the bins * are defined. * * This overload uses the "Serial" parallelization strategy. It always runs in diff --git a/Src/Particle/AMReX_NeighborParticles.H b/Src/Particle/AMReX_NeighborParticles.H index 5789f6f206d..781c9ddc446 100644 --- a/Src/Particle/AMReX_NeighborParticles.H +++ b/Src/Particle/AMReX_NeighborParticles.H @@ -19,7 +19,7 @@ namespace amrex { }; /// -/// This is a container for particles that undergo short-range interations. +/// This is a container for particles that undergo short-range interactions. /// In addition to the normal ParticleContainer methods, each tile contains a "neighbor /// buffer" that is filled with data corresponding to the particles within 1 neighbor cell of /// the tile boundaries. This allows the N^2 search over each pair of particles to proceed diff --git a/Src/Particle/AMReX_ParticleCommunication.H b/Src/Particle/AMReX_ParticleCommunication.H index 36ecb747e4b..ba9a4faba49 100644 --- a/Src/Particle/AMReX_ParticleCommunication.H +++ b/Src/Particle/AMReX_ParticleCommunication.H @@ -237,7 +237,7 @@ private: // // Snds - a Vector with the number of bytes that is process will send to each proc. // Rcvs - a Vector that, after calling this method, will contain the - // number of bytes this process will reveive from each proc. + // number of bytes this process will receive from each proc. // void doHandShake (const Vector& Snds, Vector& Rcvs) const; diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index 66897cc09f9..df71f2eeec1 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -421,7 +421,7 @@ public: /** * \brief * This initializes the particle container with one particle per cell, - * where the other particle data and attributes are all contant. The + * where the other particle data and attributes are all constant. The * coarsest level is used to generate the particle positions. The particle * variable values are passed in through the pdata struct. The parameters * x_off, y_off, and z_off represent offsets between 0 and 1 that show @@ -750,7 +750,7 @@ public: /** * \brief Writes a particle checkpoint to file, suitable for restarting. * This version allows the particle component names to be passed in. - * This overload exists for backwards comptability. The is_checkpoint parameter is ignored. + * This overload exists for backwards compatibility. The is_checkpoint parameter is ignored. */ void Checkpoint (const std::string& dir, const std::string& name, bool is_checkpoint, const Vector& real_comp_names = Vector(), @@ -968,13 +968,13 @@ public: void WriteAsciiFile (const std::string& file); /** - * \brief Return the underyling Vector (over AMR levels) of ParticleLevels. + * \brief Return the underlying Vector (over AMR levels) of ParticleLevels. * Const version. */ const Vector& GetParticles () const { return m_particles; } /** - * \brief Return the underyling Vector (over AMR levels) of ParticleLevels. + * \brief Return the underlying Vector (over AMR levels) of ParticleLevels. * Non-const version. */ Vector & GetParticles () { return m_particles; } diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 0b8000f6880..b845d130d5c 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1653,8 +1653,8 @@ ParticleContainer_implgetParticleTileData(); while (pindex <= last) { - auto ptd = particle_tile->getParticleTileData(); ParticleType p(ptd,pindex); if ((remove_negative == false) && (p.id() < 0)) { @@ -1669,7 +1669,7 @@ ParticleContainer_impl >& not_ours, const auto& src_tile = kv.second; auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; - auto old_size = dst_tile.GetArrayOfStructs().size(); + auto old_size = dst_tile.size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - Gpu::copyAsync(Gpu::hostToDevice, - src_tile.begin(), src_tile.end(), - dst_tile.GetArrayOfStructs().begin() + old_size); + if constexpr(! ParticleType::is_soa_particle) { + Gpu::copyAsync(Gpu::hostToDevice, + src_tile.begin(), src_tile.end(), + dst_tile.GetArrayOfStructs().begin() + old_size); + } for (int i = 0; i < NumRealComps(); ++i) { Gpu::copyAsync(Gpu::hostToDevice, diff --git a/Src/Particle/AMReX_ParticleInit.H b/Src/Particle/AMReX_ParticleInit.H index 7a8aa4d4a5d..d2568d5d70e 100644 --- a/Src/Particle/AMReX_ParticleInit.H +++ b/Src/Particle/AMReX_ParticleInit.H @@ -1143,7 +1143,7 @@ InitRandom (Long icount, const auto& src_tile = kv.second; auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; - auto old_size = dst_tile.GetArrayOfStructs().size(); + auto old_size = dst_tile.size(); auto new_size = old_size; if constexpr(!ParticleType::is_soa_particle) { @@ -1286,7 +1286,7 @@ InitRandom (Long icount, const auto& src_tile = kv.second; auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; - auto old_size = dst_tile.GetArrayOfStructs().size(); + auto old_size = dst_tile.size(); auto new_size = old_size; if constexpr(!ParticleType::is_soa_particle) { diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index 1663fece598..e35af847ec4 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -634,6 +634,12 @@ struct ConstParticleTileData } }; +struct ThisParticleTileHasNoParticleVector {}; + +struct ThisParticleTileHasNoAoS { + using ParticleVector = ThisParticleTileHasNoParticleVector; +}; + template class Allocator=DefaultAllocator> struct ParticleTile @@ -651,7 +657,10 @@ struct ParticleTile using SuperParticleType = Particle; - using AoS = ArrayOfStructs; + using AoS = typename std::conditional< + ParticleType::is_soa_particle, + ThisParticleTileHasNoAoS, + ArrayOfStructs>::type; //using ParticleVector = typename AoS::ParticleVector; using SoA = StructOfArrays; diff --git a/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile b/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile index 284f9942c50..b7242d85957 100644 --- a/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile +++ b/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile @@ -14,6 +14,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_CUDA = FALSE USE_PARTICLES = TRUE +USE_BITTREE = FALSE Bpack := ./Make.package Blocs := . diff --git a/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree b/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree new file mode 100644 index 00000000000..b136cbdc951 --- /dev/null +++ b/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree @@ -0,0 +1,81 @@ +# ***************************************************************** +# Run until nsteps == max_step or time == stop_time, +# whichever comes first +# ***************************************************************** +max_step = 1000 +stop_time = 2.0 + +# ***************************************************************** +# Are we restarting from an existing checkpoint file? +# ***************************************************************** +#amr.restart = chk00060 # restart from this checkpoint file + +# ***************************************************************** +# Problem size and geometry +# ***************************************************************** +geometry.prob_lo = 0.0 0.0 0.0 +geometry.prob_hi = 1.0 1.0 0.125 +geometry.is_periodic = 1 1 1 + +# ***************************************************************** +# VERBOSITY +# ***************************************************************** +amr.v = 1 # verbosity in Amr + +# ***************************************************************** +# Resolution and refinement +# ***************************************************************** +amr.n_cell = 64 64 8 +amr.max_level = 2 # maximum level number allowed -- + # number of levels = max_level + 1 + +amr.ref_ratio = 2 2 2 2 # refinement ratio between levels + +# ***************************************************************** +# Control of grid creation +# ***************************************************************** +# Blocking factor for grid creation in each dimension -- +# this ensures that every grid is coarsenable by a factor of 8 -- +# this is mostly relevant for multigrid performance +amr.blocking_factor_x = 8 +amr.blocking_factor_y = 8 +amr.blocking_factor_z = 8 + +amr.max_grid_size = 16 + +amr.regrid_int = 2 # how often to regrid + +amr.use_bittree = 1 + +# ***************************************************************** +# Time step control +# ***************************************************************** +adv.cfl = 0.7 # CFL constraint for explicit advection + +adv.do_subcycle = 1 # Do we subcycle in time? + +# ***************************************************************** +# Should we reflux at coarse-fine boundaries? +# ***************************************************************** +adv.do_reflux = 1 + +# ***************************************************************** +# Tagging - if phi > 1.01 at level 0, then refine +# if phi > 1.1 at level 1, then refine +# if phi > 1.5 at level 2, then refine +# ***************************************************************** +adv.phierr = 1.01 1.1 1.5 + +# ***************************************************************** +# Plotfile name and frequency +# ***************************************************************** +amr.plot_file = plt # root name of plot file +amr.plot_int = 10 # number of timesteps between plot files + # if negative then no plot files will be written + +# ***************************************************************** +# Checkpoint name and frequency +# ***************************************************************** +amr.chk_file = chk # root name of checkpoint file +amr.chk_int = -1 # number of timesteps between checkpoint files + # if negative then no checkpoint files will be written diff --git a/Tests/FortranInterface/Advection_F/Source/fillpatch_mod.F90 b/Tests/FortranInterface/Advection_F/Source/fillpatch_mod.F90 index f337de6e7ff..c79f2aa7b9a 100644 --- a/Tests/FortranInterface/Advection_F/Source/fillpatch_mod.F90 +++ b/Tests/FortranInterface/Advection_F/Source/fillpatch_mod.F90 @@ -90,7 +90,7 @@ subroutine fill_physbc (pmf, scomp, ncomp, time, pgeom) bind(c) geom%get_physical_location(plo), & ! physical location of lower left corner lo_bc, hi_bc) ! bc types for each component - ! amrex_filcc doesn't fill EXT_DIR (see amrex_bc_types_module for a list of bc types + ! amrex_filcc doesn't fill EXT_DIR/EXT_DIR_CC (see amrex_bc_types_module for a list of bc types ! In that case, the user needs to fill it. end if end do diff --git a/Tests/FortranInterface/Advection_F/Source/my_amr_mod.F90 b/Tests/FortranInterface/Advection_F/Source/my_amr_mod.F90 index 8df5117d77e..e9549d57320 100644 --- a/Tests/FortranInterface/Advection_F/Source/my_amr_mod.F90 +++ b/Tests/FortranInterface/Advection_F/Source/my_amr_mod.F90 @@ -104,7 +104,7 @@ subroutine my_amr_finalize () end subroutine my_amr_finalize ! Make a new level from scratch and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) use prob_module, only : init_prob_data integer, intent(in), value :: lev @@ -145,7 +145,7 @@ subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) end subroutine my_make_new_level_from_scratch ! Make a new level from coarse level and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_make_new_level_from_coarse (lev, time, pba, pdm) bind(c) use fillpatch_module, only : fillcoarsepatch integer, intent(in), value :: lev @@ -173,7 +173,7 @@ subroutine my_make_new_level_from_coarse (lev, time, pba, pdm) bind(c) end subroutine my_make_new_level_from_coarse ! Remake a level from current and coarse elvels and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_remake_level (lev, time, pba, pdm) bind(c) use fillpatch_module, only : fillpatch integer, intent(in), value :: lev diff --git a/Tests/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 b/Tests/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 index ea6e4513ff8..35614b825fa 100644 --- a/Tests/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 +++ b/Tests/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 @@ -84,7 +84,7 @@ subroutine my_amr_finalize () end subroutine my_amr_finalize ! Make a new level from scratch and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) use prob_module, only : init_prob_data integer, intent(in), value :: lev @@ -127,7 +127,7 @@ subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) end subroutine my_make_new_level_from_scratch ! Make a new level from coarse level and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_make_new_level_from_coarse (lev, time, pba, pdm) bind(c) use fillpatch_module, only : fillcoarsepatch integer, intent(in), value :: lev @@ -157,7 +157,7 @@ subroutine my_make_new_level_from_coarse (lev, time, pba, pdm) bind(c) end subroutine my_make_new_level_from_coarse ! Remake a level from current and coarse elvels and put the data in phi_new. - ! Note tha phi_old contains no valid data after this. + ! Note that phi_old contains no valid data after this. subroutine my_remake_level (lev, time, pba, pdm) bind(c) use fillpatch_module, only : fillpatch integer, intent(in), value :: lev diff --git a/Tests/GPU/CNS/Source/CNS_bcfill.cpp b/Tests/GPU/CNS/Source/CNS_bcfill.cpp index 1c787e29a96..a75c7ebce88 100644 --- a/Tests/GPU/CNS/Source/CNS_bcfill.cpp +++ b/Tests/GPU/CNS/Source/CNS_bcfill.cpp @@ -14,7 +14,7 @@ struct CnsFillExtDir const BCRec* /*bcr*/, const int /*bcomp*/, const int /*orig_comp*/) const { - // do something for external Dirichlet (BCType::ext_dir) + // do something for external Dirichlet (BCType::ext_dir/BCType::ext_dir_cc) } }; diff --git a/Tests/GPU/Vector/main.cpp b/Tests/GPU/Vector/main.cpp index cfa44437fb5..4a3fd1d9583 100644 --- a/Tests/GPU/Vector/main.cpp +++ b/Tests/GPU/Vector/main.cpp @@ -89,9 +89,9 @@ void async_test() // Async Vector now out of scope. Still completes correctly. #ifdef AMREX_USE_GPU - amrex::Print() << "Async Synching -- should print first." << std::endl; + amrex::Print() << "Async Syncing -- should print first." << std::endl; #else - amrex::Print() << "Async Synching -- should print second." << std::endl; + amrex::Print() << "Async Syncing -- should print second." << std::endl; #endif Gpu::Device::streamSynchronize(); diff --git a/Tests/HDF5Benchmark/sz.config b/Tests/HDF5Benchmark/sz.config index 471e9766295..f2abf5529e1 100644 --- a/Tests/HDF5Benchmark/sz.config +++ b/Tests/HDF5Benchmark/sz.config @@ -14,7 +14,7 @@ sol_name = SZ #Examples: layers = 1, 2, or 3 (layers=1 is recommended in most cases) layers = 1 -#sampleDistance determins the number of samples used to optimize the # quantization intervals +#sampleDistance determines the number of samples used to optimize the # quantization intervals #For example, sampleDistance=50 means 1/50=2% of data points are sample points. sampleDistance = 100 @@ -76,7 +76,7 @@ relBoundRatio = 1E-9 pw_relBoundRatio = 1E-2 #point-wise relative-error-bound segment size -#The larger this value, the lower overhead of maitaining error bound information for each segment/block, while the stricter absolute bound used to control the point-wise relative-error-bound. The block size will be M^K, where K is the # dimensions of the data and M is an integer such that (M-1)^K < segment_size <= M^K. For instance, for a 2D data, segment_size=32 means the whole data set will be split into multiple small blocks each being 6x6. For a 3D data, segment_size=32 means the whole data set is split into a lot of 4x4x4 blocks, each of which has an individual compression precision/error. +#The larger this value, the lower overhead of maintaining error bound information for each segment/block, while the stricter absolute bound used to control the point-wise relative-error-bound. The block size will be M^K, where K is the # dimensions of the data and M is an integer such that (M-1)^K < segment_size <= M^K. For instance, for a 2D data, segment_size=32 means the whole data set will be split into multiple small blocks each being 6x6. For a 3D data, segment_size=32 means the whole data set is split into a lot of 4x4x4 blocks, each of which has an individual compression precision/error. segment_size = 32 #point-wise relative-error-bound type diff --git a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H index afe97c9bea6..c1ed7ba4c3d 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H +++ b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H @@ -20,6 +20,7 @@ public: void initProbPoisson (); void initProbABecLaplacian (); void initProbABecLaplacianInhomNeumann (); + void initProbNodeABecLaplacian (); private: @@ -28,6 +29,7 @@ private: void solvePoisson (); void solveABecLaplacian (); void solveABecLaplacianInhomNeumann (); + void solveNodeABecLaplacian (); int max_level = 1; int ref_ratio = 2; @@ -36,7 +38,9 @@ private: bool composite_solve = true; - int prob_type = 1; // 1. Poisson, 2. ABecLaplacian + // 1. Poisson, 2. ABecLaplacian, 3. ABecLaplacian w/ inhomNeumann + // 4. NodeABecLaplacian + int prob_type = 1; // For MLMG solver int verbose = 2; diff --git a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp index 0db9f5959d8..9900ce43d7f 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp +++ b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp @@ -1,5 +1,6 @@ #include "MyTest.H" +#include #include #include #include @@ -22,6 +23,8 @@ MyTest::solve () solveABecLaplacian(); } else if (prob_type == 3) { solveABecLaplacianInhomNeumann(); + } else if (prob_type == 4) { + solveNodeABecLaplacian(); } else { amrex::Abort("Unknown prob_type"); } @@ -409,6 +412,54 @@ MyTest::solveABecLaplacianInhomNeumann () } } +void +MyTest::solveNodeABecLaplacian () +{ + LPInfo info; + info.setAgglomeration(agglomeration); + info.setConsolidation(consolidation); + info.setMaxCoarseningLevel(max_coarsening_level); + + const auto tol_rel = Real(1.e-10); + const auto tol_abs = Real(0.0); + + const auto nlevels = static_cast(geom.size()); + + if (composite_solve && nlevels > 1) + { + amrex::Abort("solveNodeABecLaplacian: TODO composite_solve"); + } + else + { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(nlevels == 1, "solveNodeABecLaplacian: nlevels > 1 TODO"); + for (int ilev = 0; ilev < nlevels; ++ilev) + { + MLNodeABecLaplacian mlndabec({geom[ilev]}, {grids[ilev]}, {dmap[ilev]}, + info); + + mlndabec.setDomainBC({AMREX_D_DECL(LinOpBCType::Dirichlet, + LinOpBCType::Neumann, + LinOpBCType::Dirichlet)}, + {AMREX_D_DECL(LinOpBCType::Neumann, + LinOpBCType::Dirichlet, + LinOpBCType::Dirichlet)}); + + mlndabec.setScalars(ascalar, bscalar); + + mlndabec.setACoeffs(0, acoef[ilev]); + mlndabec.setBCoeffs(0, bcoef[ilev]); + + MLMG mlmg(mlndabec); + mlmg.setMaxIter(max_iter); + mlmg.setMaxFmgIter(max_fmg_iter); + mlmg.setVerbose(verbose); + mlmg.setBottomVerbose(bottom_verbose); + + mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); + } + } +} + void MyTest::readParameters () { @@ -463,7 +514,7 @@ MyTest::initData () rhs.resize(nlevels); exact_solution.resize(nlevels); - if (prob_type == 2 || prob_type == 3) { + if (prob_type == 2 || prob_type == 3 || prob_type == 4) { acoef.resize(nlevels); bcoef.resize(nlevels); } @@ -491,12 +542,17 @@ MyTest::initData () for (int ilev = 0; ilev < nlevels; ++ilev) { dmap[ilev].define(grids[ilev]); - solution [ilev].define(grids[ilev], dmap[ilev], 1, 1); - rhs [ilev].define(grids[ilev], dmap[ilev], 1, 0); - exact_solution[ilev].define(grids[ilev], dmap[ilev], 1, 0); + BoxArray ba = grids[ilev]; + if (prob_type == 4) { + ba.surroundingNodes(); + } + solution [ilev].define(ba, dmap[ilev], 1, 1); + rhs [ilev].define(ba, dmap[ilev], 1, 0); + exact_solution[ilev].define(ba, dmap[ilev], 1, 0); if (!acoef.empty()) { - acoef[ilev].define(grids[ilev], dmap[ilev], 1, 0); - bcoef[ilev].define(grids[ilev], dmap[ilev], 1, 1); + acoef[ilev].define(ba , dmap[ilev], 1, 0); + const int ngb = (prob_type == 4) ? 0 : 1; + bcoef[ilev].define(grids[ilev], dmap[ilev], 1, ngb); } } @@ -506,6 +562,8 @@ MyTest::initData () initProbABecLaplacian(); } else if (prob_type == 3) { initProbABecLaplacianInhomNeumann(); + } else if (prob_type == 4) { + initProbNodeABecLaplacian(); } else { amrex::Abort("Unknown prob_type "+std::to_string(prob_type)); } diff --git a/Tests/LinearSolvers/ABecLaplacian_C/MyTestPlotfile.cpp b/Tests/LinearSolvers/ABecLaplacian_C/MyTestPlotfile.cpp index 707361a4e34..4473f978a85 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/MyTestPlotfile.cpp +++ b/Tests/LinearSolvers/ABecLaplacian_C/MyTestPlotfile.cpp @@ -8,6 +8,19 @@ using namespace amrex; void MyTest::writePlotfile () const { + if (prob_type == 4) { + for (int ilev = 0; ilev <= max_level; ++ilev) { + VisMF::Write(solution[ilev], "solution-lev"+std::to_string(ilev)); + MultiFab errmf(solution[ilev].boxArray(), + solution[ilev].DistributionMap(), 1, 1); + MultiFab::Copy(errmf, solution[ilev], 0, 0, 1, 0); + MultiFab::Subtract(errmf, exact_solution[ilev], 0, 0, 1, 0); + auto error = errmf.norminf(); + amrex::Print() << "Level " << ilev << " max-norm error: " << error << std::endl; + } + return; + } + ParmParse pp; bool gpu_regtest = false; #ifdef AMREX_USE_GPU diff --git a/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp b/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp index a7b197adbed..ec102ae0601 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp +++ b/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp @@ -172,3 +172,40 @@ MyTest::initProbABecLaplacianInhomNeumann () solution[ilev].setVal(0.0,0,1,0); // set interior to 0 } } + +void +MyTest::initProbNodeABecLaplacian () +{ + for (int ilev = 0; ilev <= max_level; ++ilev) + { + solution[ilev].setVal(0.0); + + const auto prob_lo = geom[ilev].ProbLoArray(); + const auto prob_hi = geom[ilev].ProbHiArray(); + const auto dx = geom[ilev].CellSizeArray(); + auto a = ascalar; + auto b = bscalar; + Box const& nddom = amrex::surroundingNodes(geom[ilev].Domain()); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(rhs[ilev]); mfi.isValid(); ++mfi) + { + const Box& ndbx = mfi.validbox(); + + auto rhsfab = rhs[ilev].array(mfi); + auto exactfab = exact_solution[ilev].array(mfi); + auto solfab = solution[ilev].array(mfi); + auto acoeffab = acoef[ilev].array(mfi); + auto bcoeffab = bcoef[ilev].array(mfi); + + amrex::ParallelFor(ndbx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + actual_init_nodeabeclap(i,j,k,rhsfab,exactfab,solfab, + acoeffab,bcoeffab,a,b, + nddom,prob_lo,prob_hi,dx); + }); + } + } +} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H b/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H index 4fcab046f0b..0b6fbe20f71 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H +++ b/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H @@ -310,4 +310,74 @@ void actual_init_dphi_dz_hi (int i, int j, int k, amrex::Array4 con + .25 * std::cos(fpi*x) * std::sin(fpi*y) * (-fpi) * std::sin(fpi*z); } +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void actual_init_nodeabeclap(int i, int j, int k, + amrex::Array4 const& rhs, + amrex::Array4 const& exact, + amrex::Array4 const& sol, + amrex::Array4 const& acoef, + amrex::Array4 const& bcoef, + amrex::Real a, amrex::Real b, + amrex::Box const& nddom, + amrex::GpuArray const& prob_lo, + amrex::GpuArray const& prob_hi, + amrex::GpuArray const& dx) +{ + constexpr amrex::Real w = 0.05; + constexpr amrex::Real sigma = 10.; + const amrex::Real theta = 0.5*std::log(3.) / (w + 1.e-50); + + constexpr amrex::Real pi = 3.1415926535897932; + constexpr amrex::Real tpi = 2.*pi; + constexpr amrex::Real fpi = 4.*pi; + constexpr amrex::Real fac = static_cast(AMREX_SPACEDIM*4)*pi*pi; + + // bcoef is at cell center, whereas the rest at nodes. + if (bcoef.contains(i,j,k)) { + actual_init_bcoef(i,j,k, bcoef, prob_lo, prob_hi, dx); + } + + amrex::Real xc = (prob_hi[0] + prob_lo[0])*0.5; + amrex::Real yc = (prob_hi[1] + prob_lo[1])*0.5; +#if (AMREX_SPACEDIM == 2) + amrex::Real zc = 0.0; +#else + amrex::Real zc = (prob_hi[2] + prob_lo[2])*0.5; +#endif + + amrex::Real x = prob_lo[0] + dx[0] * (i); + amrex::Real y = prob_lo[1] + dx[1] * (j); +#if (AMREX_SPACEDIM == 2) + amrex::Real z = 0.0; +#else + amrex::Real z = prob_lo[2] + dx[2] * (k); +#endif + + amrex::Real r = std::sqrt((x-xc)*(x-xc) + (y-yc)*(y-yc) + (z-zc)*(z-zc)); + amrex::Real bcnd = (sigma-1.)/2.*std::tanh(theta*(r-0.25)) + (sigma+1.)/2.; + amrex::Real tmp = std::cosh(theta*(r-0.25)); + amrex::Real dbdrfac = (r == amrex::Real(0.0)) + ? amrex::Real(0.0) : (sigma-1.)/2./(tmp*tmp) * theta/r; + dbdrfac *= b; + + acoef(i,j,k) = 1.; + + exact(i,j,k) = std::cos(tpi*x) * std::cos(tpi*y) * std::cos(tpi*z) + + .25 * std::cos(fpi*x) * std::cos(fpi*y) * std::cos(fpi*z); + + rhs(i,j,k) = bcnd*b*fac*( std::cos(tpi*x) * std::cos(tpi*y) * std::cos(tpi*z) + + std::cos(fpi*x) * std::cos(fpi*y) * std::cos(fpi*z)) + + dbdrfac*((x-xc)*(tpi*std::sin(tpi*x) * std::cos(tpi*y) * std::cos(tpi*z) + + pi*std::sin(fpi*x) * std::cos(fpi*y) * std::cos(fpi*z)) + + (y-yc)*(tpi*std::cos(tpi*x) * std::sin(tpi*y) * std::cos(tpi*z) + + pi*std::cos(fpi*x) * std::sin(fpi*y) * std::cos(fpi*z)) + + (z-zc)*(tpi*std::cos(tpi*x) * std::cos(tpi*y) * std::sin(tpi*z) + + pi*std::cos(fpi*x) * std::cos(fpi*y) * std::sin(fpi*z))) + + a * exact(i,j,k); + + if (! nddom.strictly_contains(i,j,k)) { + sol(i,j,k) = exact(i,j,k); // domain boundary + } +} + #endif diff --git a/Tests/LinearSolvers/ABecLaplacian_C/inputs-node b/Tests/LinearSolvers/ABecLaplacian_C/inputs-node new file mode 100644 index 00000000000..2d96588d241 --- /dev/null +++ b/Tests/LinearSolvers/ABecLaplacian_C/inputs-node @@ -0,0 +1,16 @@ + +max_level = 0 +ref_ratio = 2 +n_cell = 128 +max_grid_size = 64 + +composite_solve = 0 # composite solve or level by level? + +prob_type = 4 # nodal ABecLaplacian + +# For MLMG +verbose = 2 +bottom_verbose = 0 +max_iter = 100 +agglomeration = 1 # Do agglomeration on AMR Level 0? +consolidation = 1 # Do consolidation? diff --git a/Tests/LinearSolvers/Hypre/MyTest.cpp b/Tests/LinearSolvers/Hypre/MyTest.cpp index f68595ffc3f..99290987eeb 100644 --- a/Tests/LinearSolvers/Hypre/MyTest.cpp +++ b/Tests/LinearSolvers/Hypre/MyTest.cpp @@ -25,7 +25,7 @@ MyTest::MyTest () void MyTest::solve () { - // In this example, we assume the domain boundary is homegeneous Dirichlet. + // In this example, we assume the domain boundary is homogeneous Dirichlet. auto const& nddom = amrex::surroundingNodes(geom.Domain()); const auto dxi = geom.InvCellSizeArray(); @@ -51,7 +51,7 @@ MyTest::solve () // [in ] gid : gid[n] is the id for variable n at (i,j,k) // [out] ncols: # of columns in this row. // [out] cols: column indices in this row. - // [out] mat : matrix elemens in this row. + // [out] mat : matrix elements in this row. auto filler = [=] AMREX_GPU_DEVICE (int /*boxno*/, int i, int j, int k, int n, Array4 const* gid, HYPRE_Int& ncols, HYPRE_Int* cols, diff --git a/Tests/Parser2/main.cpp b/Tests/Parser2/main.cpp index dbb1526a20b..70d24bae07e 100644 --- a/Tests/Parser2/main.cpp +++ b/Tests/Parser2/main.cpp @@ -53,7 +53,7 @@ int main (int argc, char* argv[]) failed_tests.push_back(std::make_pair(icase,expr)); } } else { - amrex::Abort("How did this happend? No line after case."); + amrex::Abort("How did this happen? No line after case."); } } } diff --git a/Tests/Particles/GhostsAndVirtuals/main.cpp b/Tests/Particles/GhostsAndVirtuals/main.cpp index 02384eb43ea..1134e3f5b9a 100644 --- a/Tests/Particles/GhostsAndVirtuals/main.cpp +++ b/Tests/Particles/GhostsAndVirtuals/main.cpp @@ -232,7 +232,7 @@ void test_ghosts_and_virtuals_ascii (TestParams& parms) //Initialize particles from ascii file with extradata=4 myPC.InitFromAsciiFile("particle_file.init", 4); - //Regrid to the more compilcated BoxArray similar to NeighborParticleContainer Regrid + //Regrid to the more complicated BoxArray similar to NeighborParticleContainer Regrid for (int lev = 0; lev < nlevs; lev++) { myPC.SetParticleBoxArray(lev, ba.at(lev)); myPC.SetParticleDistributionMap(lev, dmap.at(lev)); diff --git a/Tests/Particles/RedistributeSOA/main.cpp b/Tests/Particles/RedistributeSOA/main.cpp index eba02904fb7..94715b7d6aa 100644 --- a/Tests/Particles/RedistributeSOA/main.cpp +++ b/Tests/Particles/RedistributeSOA/main.cpp @@ -132,7 +132,7 @@ class TestParticleContainer } auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + host_real[0].size(); particle_tile.resize(new_size); diff --git a/Tests/Particles/SENSEI_Insitu_SOA/main.cpp b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp index 8f9e8f7f7e6..f2dd2706df5 100644 --- a/Tests/Particles/SENSEI_Insitu_SOA/main.cpp +++ b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp @@ -115,7 +115,7 @@ class TestParticleContainer } auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + host_real[0].size(); particle_tile.resize(new_size); diff --git a/Tools/CMake/AMReXBuildInfo.cmake b/Tools/CMake/AMReXBuildInfo.cmake index 36c70057871..16795110658 100644 --- a/Tools/CMake/AMReXBuildInfo.cmake +++ b/Tools/CMake/AMReXBuildInfo.cmake @@ -189,7 +189,7 @@ function (generate_buildinfo _target _git_dir) configure_file( ${AMREX_BUILDINFO_IFILE} ${PROJECT_BINARY_DIR}/${_target}/AMReX_buildInfo.cpp @ONLY) - # add a re-usable target + # add a reusable target add_library(buildInfo${_target} STATIC) add_library(buildInfo::${_target} ALIAS buildInfo${_target}) diff --git a/Tools/CMake/AMReXClangTidy.cmake b/Tools/CMake/AMReXClangTidy.cmake index 4f002bc3820..3021d5939d7 100644 --- a/Tools/CMake/AMReXClangTidy.cmake +++ b/Tools/CMake/AMReXClangTidy.cmake @@ -9,7 +9,7 @@ macro(setup_clang_tidy) if (_tmp MATCHES "LLVM version ([0-9\.]+)") message(STATUS "Found clang-tidy ${CMAKE_MATCH_1}") if ("${CMAKE_MATCH_1}" VERSION_GREATER_EQUAL 12.0.0) - # Cofig file not supported in earlier versions + # Config file not supported in earlier versions set(AMReX_CLANG_TIDY_CONFIG_FILE_NAME ${PROJECT_SOURCE_DIR}/.clang-tidy) endif() endif() diff --git a/Tools/CMake/AMReXConfig.cmake.in b/Tools/CMake/AMReXConfig.cmake.in index b08e0fb162e..7b7f04727a2 100644 --- a/Tools/CMake/AMReXConfig.cmake.in +++ b/Tools/CMake/AMReXConfig.cmake.in @@ -236,7 +236,7 @@ endif () # CMake targets include( "${CMAKE_CURRENT_LIST_DIR}/AMReXTargets.cmake" ) -# CMake targets aliase: last dimension built will be our legacy target +# CMake targets aliases: last dimension built will be our legacy target if (NOT TARGET AMReX::amrex) # protection in case of multiple inclusions list(LENGTH AMReX_SPACEDIM list_len) math(EXPR list_last "${list_len} - 1") diff --git a/Tools/CMake/AMReXOptions.cmake b/Tools/CMake/AMReXOptions.cmake index c086bdc7489..80196639984 100644 --- a/Tools/CMake/AMReXOptions.cmake +++ b/Tools/CMake/AMReXOptions.cmake @@ -59,7 +59,7 @@ print_option( USE_XSDK_DEFAULTS ) # Defaults: # CMake: static # xSDK: shared -# Precendence of user options: +# Precedence of user options: # AMReX_BUILD_SHARED_LIBS > BUILD_SHARED_LIBS > USE_XSDK_DEFAULTS # default: unset unset OFF # diff --git a/Tools/C_scripts/mkdep b/Tools/C_scripts/mkdep index 32552d44837..b5c039b212d 100755 --- a/Tools/C_scripts/mkdep +++ b/Tools/C_scripts/mkdep @@ -7,7 +7,7 @@ # Notes: * -I defines a search path for include files # * -DBG turn on debug flag # + -fortran: parse fortran style include directives -# * -X means disgard entries with this path (NOT IMPLEMENTED) +# * -X means discard entries with this path (NOT IMPLEMENTED) # * searches current directory only if -I. is in search # path or #include directive uses double quotes rather # than angle brackets. diff --git a/Tools/C_scripts/mmclt.py b/Tools/C_scripts/mmclt.py index c8e61ec1b3a..0ddcc0f0c6e 100755 --- a/Tools/C_scripts/mmclt.py +++ b/Tools/C_scripts/mmclt.py @@ -15,9 +15,9 @@ def mmclt(argv): help="Ccache log file", default="ccache.log.txt") parser.add_argument("--identifier", - help="Unique identifier for finding compilaton line in the log file", + help="Unique identifier for finding compilation line in the log file", default="Src/Base") - # We assume Src/Base can be used as an indentifier to distinguish amrex code + # We assume Src/Base can be used as an identifier to distinguish amrex code # from cmake's temporary files like build/CMakeFiles/CMakeScratch/TryCompile-hw3x4m/test_mpi.cpp parser.add_argument("--output", help="Make file for clang-tidy", diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs index c1b5eca6bff..8ec8832d1eb 100644 --- a/Tools/GNUMake/Make.defs +++ b/Tools/GNUMake/Make.defs @@ -1,6 +1,6 @@ ifneq (,$(findstring ~,$(AMREX_HOME))) - $(warning *** AMREX_HOME string constains ~ and make will not like it. So it is replaced.) + $(warning *** AMREX_HOME string contains ~ and make will not like it. So it is replaced.) AMREX_HOME := $(shell echo $(AMREX_HOME)) endif @@ -1067,6 +1067,12 @@ ifeq ($(USE_HDF5),TRUE) include $(AMREX_HOME)/Tools/GNUMake/packages/Make.hdf5 endif +ifeq ($(USE_BITTREE),TRUE) + $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.bittree...) + include $(AMREX_HOME)/Tools/GNUMake/packages/Make.bittree +endif + + ifneq ("$(wildcard $(AMREX_HOME)/Tools/GNUMake/Make.local)","") $(info Loading $(AMREX_HOME)/Tools/GNUMake/Make.local...) include $(AMREX_HOME)/Tools/GNUMake/Make.local diff --git a/Tools/GNUMake/Make.rules b/Tools/GNUMake/Make.rules index 53ff408db54..2dba44dca6f 100644 --- a/Tools/GNUMake/Make.rules +++ b/Tools/GNUMake/Make.rules @@ -527,7 +527,7 @@ print-%: @echo ' value = $(subst ','"'"',$(value $*))' # We need to use subst on the result of $(value) because it contains single # quotes. Shell command echo does not like things like 'x'$(filiter-out)'y', -# because what it sees is 'x', $(filter-out), and 'y'. With the substition, it +# because what it sees is 'x', $(filter-out), and 'y'. With the substitution, it # will see 'x', "'", '$(filter-out)', "'", and 'y', with $(filter-out) inside a # pair of single quotes. diff --git a/Tools/GNUMake/comps/hip.mak b/Tools/GNUMake/comps/hip.mak index 424d2d03aa1..f887b206e4c 100644 --- a/Tools/GNUMake/comps/hip.mak +++ b/Tools/GNUMake/comps/hip.mak @@ -43,7 +43,7 @@ HIPCC_FLAGS += -pthread CXXFLAGS += $(HIPCC_FLAGS) -# add fopenmp targetting the gnu library +# add fopenmp targeting the gnu library ifeq ($(USE_OMP),TRUE) CXXFLAGS += -fopenmp=libgomp CFLAGS += -fopenmp=libgomp diff --git a/Tools/GNUMake/packages/Make.bittree b/Tools/GNUMake/packages/Make.bittree new file mode 100644 index 00000000000..98758a915c2 --- /dev/null +++ b/Tools/GNUMake/packages/Make.bittree @@ -0,0 +1,16 @@ + +CPPFLAGS += -DAMREX_USE_BITTREE +include $(AMREX_HOME)/Src/Extern/Bittree/Make.package + +ifndef AMREX_BITTREE_HOME +ifdef BITTREE_$(DIM)D_HOME + AMREX_BITTREE_HOME = $(BITTREE_$(DIM)D_HOME) +endif +endif + +ifdef AMREX_BITTREE_HOME + BITTREE_ABSPATH = $(abspath $(AMREX_BITTREE_HOME)) + INCLUDE_LOCATIONS += $(BITTREE_ABSPATH)/include + LIBRARY_LOCATIONS += $(BITTREE_ABSPATH)/lib + LIBRARIES += -Wl,-rpath,$(BITTREE_ABSPATH)/lib -lbittree +endif diff --git a/Tools/libamrex/configure.py b/Tools/libamrex/configure.py index 42493e4647e..1545f86dfb2 100755 --- a/Tools/libamrex/configure.py +++ b/Tools/libamrex/configure.py @@ -116,6 +116,10 @@ def configure(argv): help="Only relevant to Amr/AmrLevel based codes that need to read probin file or call amrex_probinit", choices=["yes","no"], default="yes") + parser.add_argument("--enable-bittree", + help="Enable Bittree mode [default=no]", + choices=["yes","no"], + default="no") args = parser.parse_args() if args.with_fortran == "no": @@ -154,6 +158,7 @@ def configure(argv): f.write("USE_COMPILE_PIC = {}\n".format("FALSE" if args.enable_pic == "no" else "TRUE")) f.write("CUDA_ARCH = " + args.cuda_arch.strip() + "\n") f.write("AMREX_NO_PROBINIT = {}\n".format("TRUE" if args.enable_probinit == "no" else "FALSE")) + f.write("USE_BITTREE = {}\n".format("TRUE" if args.enable_bittree == "yes" else "FALSE")) f.write("\n") fin = open("GNUmakefile.in","r")