Skip to content

Commit 5b4dfdc

Browse files
ax3lroelof-groenewald
ax3l
and
roelof-groenewald
authored
Python 3.12: Refactor out Distutils (#4392)
* CI: Unconstrain Some Upper Py Versions * Replace distutils * Update Docs * no `clean` commmand pypa/setuptools#2838 * Remove `SETUPTOOLS_USE_DISTUTILS` Old workaround for old mpi4py. * Update Doc String * Update CI * Update Tools/machines/lassen-llnl/install_v100_ml.sh Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com>
1 parent f6cb58d commit 5b4dfdc

29 files changed

+72
-54
lines changed

‎.github/workflows/cuda.yml

+8-14
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,12 @@ jobs:
1818
env:
1919
CXXFLAGS: "-Werror"
2020
CMAKE_GENERATOR: Ninja
21-
# setuptools/mp4py work-around, see
22-
# https://github.com/mpi4py/mpi4py/pull/159
23-
# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
24-
SETUPTOOLS_USE_DISTUTILS: stdlib
2521
steps:
2622
- uses: actions/checkout@v3
2723
- uses: actions/setup-python@v4
2824
name: Install Python
2925
with:
30-
python-version: '3.10'
26+
python-version: '3.x'
3127
- name: install dependencies
3228
run: |
3329
.github/workflows/dependencies/nvcc11-3.sh
@@ -75,7 +71,8 @@ jobs:
7571
-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
7672
cmake --build build_sp -j 2
7773
78-
python3 -m pip install --upgrade pip setuptools wheel
74+
python3 -m pip install --upgrade pip
75+
python3 -m pip install --upgrade build packaging setuptools wheel
7976
export WARPX_MPI=ON
8077
export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/
8178
python3 -m pip wheel .
@@ -118,13 +115,9 @@ jobs:
118115
name: NVHPC@21.11 NVCC/NVC++ Release [tests]
119116
runs-on: ubuntu-20.04
120117
if: github.event.pull_request.draft == false
121-
env:
122-
# For NVHPC, Ninja is slower than the default:
123-
#CMAKE_GENERATOR: Ninja
124-
# setuptools/mp4py work-around, see
125-
# https://github.com/mpi4py/mpi4py/pull/159
126-
# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
127-
SETUPTOOLS_USE_DISTUTILS: stdlib
118+
#env:
119+
# # For NVHPC, Ninja is slower than the default:
120+
# CMAKE_GENERATOR: Ninja
128121
steps:
129122
- uses: actions/checkout@v3
130123
- name: Dependencies
@@ -175,7 +168,8 @@ jobs:
175168
# https://github.com/mpi4py/mpi4py/issues/114
176169
#export CFLAGS="-noswitcherror"
177170
178-
#python3 -m pip install --upgrade pip setuptools wheel
171+
#python3 -m pip install --upgrade pip
172+
#python3 -m pip install --upgrade build packaging setuptools wheel
179173
#export WARPX_MPI=ON
180174
#export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/
181175
#python3 -m pip wheel .

‎.github/workflows/intel.yml

+4-3
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ jobs:
4141
export CXX=$(which icpc)
4242
export CC=$(which icc)
4343
44-
python3 -m pip install --upgrade pip setuptools wheel
44+
python3 -m pip install --upgrade pip
45+
python3 -m pip install --upgrade build packaging setuptools wheel
4546
4647
cmake -S . -B build_dp \
4748
-DCMAKE_VERBOSE_MAKEFILE=ON \
@@ -104,7 +105,7 @@ jobs:
104105
export CC=$(which icx)
105106
106107
python3 -m pip install --upgrade pip
107-
python3 -m pip install --upgrade setuptools wheel
108+
python3 -m pip install --upgrade build packaging setuptools wheel
108109
109110
cmake -S . -B build_sp \
110111
-DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \
@@ -178,6 +179,6 @@ jobs:
178179
179180
# Skip this as it will copy the binary artifacts and we are tight on disk space
180181
# python3 -m pip install --upgrade pip
181-
# python3 -m pip install --upgrade setuptools wheel
182+
# python3 -m pip install --upgrade build packaging setuptools wheel
182183
# PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel .
183184
# python3 -m pip install *.whl

‎.github/workflows/macos.yml

+1-5
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@ jobs:
1616
HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE
1717
# For macOS, Ninja is slower than the default:
1818
#CMAKE_GENERATOR: Ninja
19-
# setuptools/mp4py work-around, see
20-
# https://github.com/mpi4py/mpi4py/pull/159
21-
# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
22-
SETUPTOOLS_USE_DISTUTILS: stdlib
2319
steps:
2420
- uses: actions/checkout@v3
2521
- name: Brew Cache
@@ -65,7 +61,7 @@ jobs:
6561
- name: build WarpX
6662
run: |
6763
python3 -m pip install --upgrade pip
68-
python3 -m pip install --upgrade pip setuptools wheel
64+
python3 -m pip install --upgrade build packaging setuptools wheel
6965
7066
cmake -S . -B build_dp \
7167
-DCMAKE_VERBOSE_MAKEFILE=ON \

‎.github/workflows/ubuntu.yml

+2-5
Original file line numberDiff line numberDiff line change
@@ -167,10 +167,6 @@ jobs:
167167
CXX: clang++
168168
# On CI for this test, Ninja is slower than the default:
169169
#CMAKE_GENERATOR: Ninja
170-
# setuptools/mp4py work-around, see
171-
# https://github.com/mpi4py/mpi4py/pull/159
172-
# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
173-
SETUPTOOLS_USE_DISTUTILS: stdlib
174170
steps:
175171
- uses: actions/checkout@v3
176172
- name: install dependencies
@@ -190,7 +186,8 @@ jobs:
190186
ccache-openmp-pyfull-
191187
- name: build WarpX
192188
run: |
193-
python3 -m pip install --upgrade pip setuptools wheel
189+
python3 -m pip install --upgrade pip
190+
python3 -m pip install --upgrade build packaging setuptools wheel
194191
195192
export CXXFLAGS="-Werror -Wno-error=pass-failed"
196193

‎.github/workflows/windows.yml

+7-3
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
- uses: actions/checkout@v3
1616
- uses: actions/setup-python@v4
1717
with:
18-
python-version: '3.10'
18+
python-version: '3.x'
1919
- name: CCache Cache
2020
uses: actions/cache@v3
2121
# - once stored under a key, they become immutable (even if local cache path content changes)
@@ -41,7 +41,9 @@ jobs:
4141
cmake --build build --config Debug --parallel 2
4242
if(!$?) { Exit $LASTEXITCODE }
4343
44-
python3 -m pip install --upgrade pip setuptools wheel
44+
python3 -m pip install --upgrade pip
45+
if(!$?) { Exit $LASTEXITCODE }
46+
python3 -m pip install --upgrade build packaging setuptools wheel
4547
if(!$?) { Exit $LASTEXITCODE }
4648
cmake --build build --config Debug --target install
4749
if(!$?) { Exit $LASTEXITCODE }
@@ -100,7 +102,9 @@ jobs:
100102
cmake --build build --config Release --target install
101103
if errorlevel 1 exit 1
102104
103-
python3 -m pip install --upgrade pip setuptools wheel
105+
python3 -m pip install --upgrade pip
106+
if errorlevel 1 exit 1
107+
python3 -m pip install --upgrade build packaging setuptools wheel
104108
if errorlevel 1 exit 1
105109
python3 -m pip install --upgrade -r requirements.txt
106110
if errorlevel 1 exit 1

‎CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ option(WarpX_PYTHON_IPO
127127
)
128128

129129
set(pyWarpX_VERSION_INFO "" CACHE STRING
130-
"PEP-440 conformant version (set by distutils)")
130+
"PEP-440 conformant version (set by setup.py)")
131131

132132
# enforce consistency of dependent options
133133
if(WarpX_APP OR WarpX_PYTHON)

‎Docs/source/install/cmake.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ PICMI Python Bindings
211211

212212
.. code-block:: bash
213213
214-
python3 -m pip install -U pip setuptools wheel
214+
python3 -m pip install -U pip
215+
python3 -m pip install -U build packaging setuptools wheel
215216
python3 -m pip install -U cmake
216217
python3 -m pip install -r requirements.txt
217218

‎Docs/source/install/hpc/lawrencium.rst

+2
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,8 @@ Optionally, download and install Python packages for :ref:`PICMI <usage-picmi>`
8080
python3 -m venv $HOME/sw/v100/venvs/warpx
8181
source $HOME/sw/v100/venvs/warpx/bin/activate
8282
python3 -m pip install --upgrade pip
83+
python3 -m pip install --upgrade build
84+
python3 -m pip install --upgrade packaging
8385
python3 -m pip install --upgrade wheel
8486
python3 -m pip install --upgrade setuptools
8587
python3 -m pip install --upgrade cython

‎Docs/source/install/hpc/lxplus.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ Now, ensure Python tooling is up-to-date:
147147

148148
.. code-block:: bash
149149
150-
python3 -m pip install -U pip setuptools wheel
150+
python3 -m pip install -U pip
151+
python3 -m pip install -U build packaging setuptools wheel
151152
python3 -m pip install -U cmake
152153
153154
Then we compile WarpX as in the previous section (with or without CUDA) adding ``-DWarpX_PYTHON=ON`` and then we install it into our Python:

‎Docs/source/install/users.rst

+2-3
Original file line numberDiff line numberDiff line change
@@ -109,12 +109,11 @@ Given that you have the :ref:`WarpX dependencies <install-dependencies>` install
109109

110110
.. code-block:: bash
111111
112-
# optional: --user
113-
python3 -m pip install -U pip setuptools wheel
112+
python3 -m pip install -U pip
113+
python3 -m pip install -U build packaging setuptools wheel
114114
python3 -m pip install -U cmake
115115
116116
python3 -m pip wheel -v git+https://github.com/ECP-WarpX/WarpX.git
117-
# optional: --user
118117
python3 -m pip install *whl
119118
120119
In the future, will publish pre-compiled binary packages on `PyPI <https://pypi.org/>`__ for faster installs.

‎Docs/source/maintenance/performance_tests.rst

+1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ Then, in ``$AUTOMATED_PERF_TESTS``, create a file ``run_automated_performance_te
7575
# lines below this comment once, before submission.
7676
# The commented lines take too long for the job script.
7777
#python3 -m pip install --upgrade pip
78+
#python3 -m pip install --upgrade build packaging setuptools wheel
7879
#python3 -m pip install --upgrade cython
7980
#python3 -m pip install --upgrade numpy
8081
#python3 -m pip install --upgrade markupsafe

‎Tools/machines/adastra-cines/install_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ rm -rf ${SW_DIR}/venvs/warpx-adastra
101101
python3 -m venv ${SW_DIR}/venvs/warpx-adastra
102102
source ${SW_DIR}/venvs/warpx-adastra/bin/activate
103103
python3 -m pip install --upgrade pip
104+
python3 -m pip install --upgrade build
105+
python3 -m pip install --upgrade packaging
104106
python3 -m pip install --upgrade wheel
105107
python3 -m pip install --upgrade setuptools
106108
python3 -m pip install --upgrade cython

‎Tools/machines/crusher-olcf/install_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ rm -rf ${SW_DIR}/venvs/warpx-crusher
8686
python3 -m venv ${SW_DIR}/venvs/warpx-crusher
8787
source ${SW_DIR}/venvs/warpx-crusher/bin/activate
8888
python3 -m pip install --upgrade pip
89+
python3 -m pip install --upgrade build
90+
python3 -m pip install --upgrade packaging
8991
python3 -m pip install --upgrade wheel
9092
python3 -m pip install --upgrade setuptools
9193
python3 -m pip install --upgrade cython

‎Tools/machines/frontier-olcf/install_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ rm -rf ${SW_DIR}/venvs/warpx-frontier
8686
python3 -m venv ${SW_DIR}/venvs/warpx-frontier
8787
source ${SW_DIR}/venvs/warpx-frontier/bin/activate
8888
python3 -m pip install --upgrade pip
89+
python3 -m pip install --upgrade build
90+
python3 -m pip install --upgrade packaging
8991
python3 -m pip install --upgrade wheel
9092
python3 -m pip install --upgrade setuptools
9193
python3 -m pip install --upgrade cython

‎Tools/machines/hpc3-uci/install_gpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx-gpu
116116
python3 -m venv ${SW_DIR}/venvs/warpx-gpu
117117
source ${SW_DIR}/venvs/warpx-gpu/bin/activate
118118
python3 -m pip install --upgrade pip
119+
python3 -m pip install --upgrade build
120+
python3 -m pip install --upgrade packaging
119121
python3 -m pip install --upgrade wheel
120122
python3 -m pip install --upgrade setuptools
121123
python3 -m pip install --upgrade cython

‎Tools/machines/karolina-it4i/install_cpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ rm -rf ${SW_DIR}/venvs/warpx-cpu
120120
python3 -m venv ${SW_DIR}/venvs/warpx-cpu
121121
source ${SW_DIR}/venvs/warpx-cpu/bin/activate
122122
python3 -m pip install --upgrade pip
123+
python3 -m pip install --upgrade build
124+
python3 -m pip install --upgrade packaging
123125
python3 -m pip install --upgrade wheel
124126
python3 -m pip install --upgrade setuptools
125127
python3 -m pip install --upgrade cython

‎Tools/machines/karolina-it4i/install_gpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ rm -rf ${SW_DIR}/venvs/warpx-gpu
120120
python3 -m venv ${SW_DIR}/venvs/warpx-gpu
121121
source ${SW_DIR}/venvs/warpx-gpu/bin/activate
122122
python3 -m pip install --upgrade pip
123+
python3 -m pip install --upgrade build
124+
python3 -m pip install --upgrade packaging
123125
python3 -m pip install --upgrade wheel
124126
python3 -m pip install --upgrade setuptools
125127
python3 -m pip install --upgrade cython

‎Tools/machines/lassen-llnl/install_v100_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-lassen
117117
source ${SW_DIR}/venvs/warpx-lassen/bin/activate
118118
python3 -m pip install --upgrade pip
119119
python3 -m pip cache purge
120+
python3 -m pip install --upgrade build
121+
python3 -m pip install --upgrade packaging
120122
python3 -m pip install --upgrade wheel
121123
python3 -m pip install --upgrade setuptools
122124
# Older version for h4py

‎Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh

+2
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-lassen-toss3
117117
source ${SW_DIR}/venvs/warpx-lassen-toss3/bin/activate
118118
python3 -m pip install --upgrade pip
119119
python3 -m pip cache purge
120+
python3 -m pip install --upgrade build
121+
python3 -m pip install --upgrade packaging
120122
python3 -m pip install --upgrade wheel
121123
python3 -m pip install --upgrade setuptools
122124
# Older version for h4py

‎Tools/machines/lassen-llnl/install_v100_ml.sh

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ rm -rf build
6464
cd -
6565

6666
# optional: optimas dependencies (based on libEnsemble & ax->botorch->gpytorch->pytorch)
67+
# TODO: scikit-learn needs a BLAS hint
6768
# commented because scikit-learn et al. compile > 2 hrs
6869
# please run manually on a login node if needed
6970
#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt

‎Tools/machines/leonardo-cineca/install_gpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ source ${SW_DIR}/venvs/warpx/bin/activate
8585
python3 -m ensurepip --upgrade
8686
python3 -m pip cache purge
8787
python3 -m pip install --upgrade pip
88+
python3 -m pip install --upgrade build
89+
python3 -m pip install --upgrade packaging
8890
python3 -m pip install --upgrade wheel
8991
python3 -m pip install --upgrade setuptools
9092
python3 -m pip install --upgrade cython

‎Tools/machines/lumi-csc/install_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ rm -rf ${SW_DIR}/venvs/warpx-lumi
101101
python3 -m venv ${SW_DIR}/venvs/warpx-lumi
102102
source ${SW_DIR}/venvs/warpx-lumi/bin/activate
103103
python3 -m pip install --upgrade pip
104+
python3 -m pip install --upgrade build
105+
python3 -m pip install --upgrade packaging
104106
python3 -m pip install --upgrade wheel
105107
python3 -m pip install --upgrade setuptools
106108
python3 -m pip install --upgrade cython

‎Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx
116116
python3 -m venv ${SW_DIR}/venvs/warpx
117117
source ${SW_DIR}/venvs/warpx/bin/activate
118118
python3 -m pip install --upgrade pip
119+
python3 -m pip install --upgrade build
120+
python3 -m pip install --upgrade packaging
119121
python3 -m pip install --upgrade wheel
120122
python3 -m pip install --upgrade setuptools
121123
python3 -m pip install --upgrade cython

‎Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ rm -rf ${SW_DIR}/venvs/warpx
116116
python3 -m venv ${SW_DIR}/venvs/warpx
117117
source ${SW_DIR}/venvs/warpx/bin/activate
118118
python3 -m pip install --upgrade pip
119+
python3 -m pip install --upgrade build
120+
python3 -m pip install --upgrade packaging
119121
python3 -m pip install --upgrade wheel
120122
python3 -m pip install --upgrade setuptools
121123
python3 -m pip install --upgrade cython

‎Tools/machines/quartz-llnl/install_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ python3 -m venv ${SW_DIR}/venvs/warpx-quartz
9999
source ${SW_DIR}/venvs/warpx-quartz/bin/activate
100100
python3 -m pip install --upgrade pip
101101
python3 -m pip cache purge
102+
python3 -m pip install --upgrade build
103+
python3 -m pip install --upgrade packaging
102104
python3 -m pip install --upgrade wheel
103105
python3 -m pip install --upgrade setuptools
104106
python3 -m pip install --upgrade cython

‎Tools/machines/summit-olcf/install_gpu_dependencies.sh

+2
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ rm -rf ${SW_DIR}/venvs/warpx-summit
9999
python3 -m venv ${SW_DIR}/venvs/warpx-summit
100100
source ${SW_DIR}/venvs/warpx-summit/bin/activate
101101
python3 -m pip install --upgrade pip
102+
python3 -m pip install --upgrade build
103+
python3 -m pip install --upgrade packaging
102104
python3 -m pip install --upgrade wheel
103105
python3 -m pip install --upgrade setuptools
104106
python3 -m pip install --upgrade cython

‎pyproject.toml

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
requires = [
33
"setuptools>=42",
44
"wheel",
5-
"cmake>=3.20.0,<4.0.0"
5+
"cmake>=3.20.0,<4.0.0",
6+
"packaging>=23",
67
]
78
build-backend = "setuptools.build_meta"

‎run_test.sh

+2-5
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,9 @@ echo "cd $PWD"
6161
rm -rf py-venv
6262
python3 -m venv py-venv
6363
source py-venv/bin/activate
64-
python3 -m pip install --upgrade pip setuptools wheel
64+
python3 -m pip install --upgrade pip
65+
python3 -m pip install --upgrade build packaging setuptools wheel
6566
python3 -m pip install --upgrade cmake
66-
# setuptools/mp4py work-around, see
67-
# https://github.com/mpi4py/mpi4py/pull/159
68-
# https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
69-
export SETUPTOOLS_USE_DISTUTILS="stdlib"
7067
python3 -m pip install --upgrade -r warpx/Regression/requirements.txt
7168

7269
# Clone AMReX and warpx-data

0 commit comments

Comments
 (0)