Remove Python 3.6, upgrade NumPy to 1.21.5, upgrade to manylinux2014 (onnx#4001)

jcwchen · web-flow · commit 661eda4703bf · 2022-02-18T15:16:24.000-08:00
* Remove Python 3.6 and upgrade NumPy to 1.21.5

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* bump visual studio to Visual Studio 17 2022

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* use setup-python@v2

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* upgrade to manylinux2014_x86_64

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* pip install cmake instead

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* let azp pipelines cover 3.7 to 3.9

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;

* update CIPipelines

Signed-off-by: Chun-Wei Chen &lt;jacky82226@gmail.com&gt;
diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml
@@ -7,13 +7,18 @@ jobs:
     vmImage: 'Ubuntu-18.04'
   strategy:
     matrix:
-      py36:
-        python.version: '3.6'
+      py39-ml-debug:
+        python.version: '3.9'
+        onnx_ml: 1
+        onnx_debug: 1
+        onnx_lite: 0
+      py38:
+        python.version: '3.8'
         onnx_ml: 0
         onnx_debug: 0
         onnx_lite: 1
-      py36-ml:
-        python.version: '3.6'
+      py38-ml:
+        python.version: '3.8'
         onnx_ml: 1
         onnx_debug: 0
         onnx_lite: 1
@@ -27,11 +32,6 @@ jobs:
         onnx_ml: 1
         onnx_debug: 0
         onnx_lite: 1
-      py37-ml-debug:
-        python.version: '3.7'
-        onnx_ml: 1
-        onnx_debug: 1
-        onnx_lite: 0
     maxParallel: 6
 
   steps:
diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -7,18 +7,28 @@ jobs:
     vmImage: 'macOS-10.15'
   strategy:
     matrix:
-      py36:
-        python.version: '3.6'
+      py39:
+        python.version: '3.9'
+        onnx_ml: 0
+        onnx_debug: 0
+        onnx_lite: 0
+      py38:
+        python.version: '3.8'
         onnx_ml: 0
         onnx_debug: 0
         onnx_lite: 1
-      py36-ml:
-        python.version: '3.6'
+      py38-ml:
+        python.version: '3.8'
         onnx_ml: 1
         onnx_debug: 0
         onnx_lite: 1
-      py36-ml-debug:
-        python.version: '3.6'
+      py38-ml-debug:
+        python.version: '3.8'
+        onnx_ml: 1
+        onnx_debug: 1
+        onnx_lite: 0
+      py37-ml-debug:
+        python.version: '3.7'
         onnx_ml: 1
         onnx_debug: 1
         onnx_lite: 0
diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml
@@ -12,6 +12,10 @@ jobs:
         python.version: '3.9'
         onnx_ml: 0
         onnx_verify_proto: 0
+      py39_verify_proto:
+        python.version: '3.9'
+        onnx_ml: 0
+        onnx_verify_proto: 1
       py38:
         python.version: '3.8'
         onnx_ml: 0
@@ -20,18 +24,10 @@ jobs:
         python.version: '3.7'
         onnx_ml: 0
         onnx_verify_proto: 0
-      py36:
-        python.version: '3.6'
-        onnx_ml: 0
-        onnx_verify_proto: 0
       py37_onnx_ml:
         python.version: '3.7'
         onnx_ml: 1
         onnx_verify_proto: 0
-      py36_verify_proto:
-        python.version: '3.6'
-        onnx_ml: 0
-        onnx_verify_proto: 1
     maxParallel: 4
 
   steps:
diff --git a/.github/workflows/manylinux/entrypoint.sh b/.github/workflows/manylinux/entrypoint.sh
@@ -6,19 +6,26 @@ set -e -x
 PY_VERSION=$1
 PLAT=$2
 GITHUB_EVENT_NAME=$3
-SYSTEM_PACKAGES='cmake3'
-if [ `uname -m` == 'aarch64' ]; then
- SYSTEM_PACKAGES='cmake'
-else
- SYSTEM_PACKAGES='cmake3'
-fi
 
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
 
-if [ ! -z "$SYSTEM_PACKAGES" ]; then
-    yum install -y ${SYSTEM_PACKAGES}  || { echo "Installing yum package(s) failed."; exit 1; }
+# Compile wheels
+# Need to be updated if there is a new Python Version
+if [ `uname -m` == 'aarch64' ]; then
+ PIP_COMMAND="$PY_VERSION -m pip install --no-cache-dir -q"
+ PYTHON_COMMAND="$PY_VERSION"
+else
+ declare -A python_map=(["3.7"]="cp37-cp37m" ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39")
+ declare -A python_include=(["3.7"]="3.7m" ["3.8"]="3.8" ["3.9"]="3.9")
+ PY_VER=${python_map[$PY_VERSION]}
+ PIP_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q"
+ PYTHON_COMMAND="/opt/python/"${PY_VER}"/bin/python"
 fi
 
+# Update pip
+$PIP_COMMAND --upgrade pip
+$PIP_COMMAND cmake
+
 # Build protobuf
 ONNX_PATH=$(pwd)
 cd ..
@@ -33,30 +40,14 @@ make -j$(nproc)
 make install
 cd $ONNX_PATH
 
-# Compile wheels
-# Need to be updated if there is a new Python Version
-if [ `uname -m` == 'aarch64' ]; then
- PIP_COMMAND="$PY_VERSION -m pip install --no-cache-dir -q"
- PYTHON_COMMAND="$PY_VERSION"
-else
- declare -A python_map=( ["3.6"]="cp36-cp36m" ["3.7"]="cp37-cp37m" ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39")
- declare -A python_include=( ["3.6"]="3.6m" ["3.7"]="3.7m" ["3.8"]="3.8" ["3.9"]="3.9")
- PY_VER=${python_map[$PY_VERSION]}
- PIP_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q"
- PYTHON_COMMAND="/opt/python/"${PY_VER}"/bin/python"
-fi
-
 # set ONNX build environments
 export ONNX_ML=1
 export CMAKE_ARGS="-DPYTHON_INCLUDE_DIR=/opt/python/${PY_VER}/include/python${python_include[$PY_VERSION]}"
 
-# Update pip
-$PIP_COMMAND --upgrade pip
-
 # Install Python dependency
-if [ "$PLAT" == "manylinux2010_i686" ]; then
+if [ "$PLAT" == "manylinux2014_i686" ]; then
     # pip install -r requirements-release will bump into issue in i686 due to pip install cryptography failure
-    $PIP_COMMAND numpy==1.16.6 protobuf==3.16.0 || { echo "Installing Python requirements failed."; exit 1; }
+    $PIP_COMMAND numpy==1.21.5 protobuf==3.16.0 || { echo "Installing Python requirements failed."; exit 1; }
 else
     $PIP_COMMAND -r requirements-release.txt || { echo "Installing Python requirements failed."; exit 1; }
 fi
@@ -68,7 +59,6 @@ else
     $PYTHON_COMMAND setup.py bdist_wheel || { echo "Building wheels failed."; exit 1; }
 fi
 
-
 # Bundle external shared libraries into the wheels
 # find -exec does not preserve failed exit codes, so use an output file for failures
 failed_wheels=$PWD/failed-wheels
diff --git a/.github/workflows/manylinux/test_package_i686.sh b/.github/workflows/manylinux/test_package_i686.sh
@@ -6,10 +6,10 @@ set -e -x
 PY_VERSION=$1
 
 # Need to be updated if there is a new Python Version
-declare -A python_map=( ["3.6"]="cp36-cp36m" ["3.7"]="cp37-cp37m" ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39")
+declare -A python_map=(["3.7"]="cp37-cp37m" ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39")
 PY_VER=${python_map[$PY_VERSION]}
 
-yum install -y protobuf-devel cmake3
+yum install -y protobuf-devel
 
 PYTHON_BIN="/opt/python/${PY_VER}/bin/"
 PIP_INSTALL_COMMAND="${PYTHON_BIN}pip install --no-cache-dir -q"
@@ -18,6 +18,7 @@ PYTHON_COMAND="${PYTHON_BIN}python"
 PYTEST_COMMAND="${PYTHON_BIN}pytest"
 
 $PIP_INSTALL_COMMAND --upgrade pip
+$PIP_INSTALL_COMMAND cmake
 
 # pip install -r requirements-release will bump into issue in i686 due to pip install cryptography failure
 # Verify ONNX with the latest numpy
@@ -26,7 +27,7 @@ if [ "$PY_VERSION" == "3.8" ] || [ "$PY_VERSION" == "3.9" ]; then
 else
     $PIP_INSTALL_COMMAND numpy protobuf==3.16.0 pytest==5.4.3 nbval ipython==7.16.1 || { echo "Installing Python requirements failed."; exit 1; }
 fi
-$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_INSTALL_COMMAND dist/*manylinux2014_i686.whl
 
 # pytest with the built wheel
 $PYTEST_COMMAND
@@ -44,17 +45,17 @@ if [ "$PY_VERSION" == "3.8" ] || [ "$PY_VERSION" == "3.9" ]; then
 else
     $PIP_UNINSTALL_COMMAND numpy onnx && $PIP_INSTALL_COMMAND numpy
 fi
-$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_INSTALL_COMMAND dist/*manylinux2014_i686.whl
 $PYTEST_COMMAND
 
 # Verify ONNX with the latest protobuf
 $PIP_UNINSTALL_COMMAND protobuf onnx && $PIP_INSTALL_COMMAND protobuf
-$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_INSTALL_COMMAND dist/*manylinux2014_i686.whl
 $PYTEST_COMMAND
 
 # Verify ONNX the minimum supported protobuf (from requirements.txt)
 $PIP_UNINSTALL_COMMAND protobuf onnx && $PIP_INSTALL_COMMAND protobuf==3.12.2
-$PIP_INSTALL_COMMAND dist/*manylinux2010_i686.whl
+$PIP_INSTALL_COMMAND dist/*manylinux2014_i686.whl
 $PYTEST_COMMAND
 
 echo "Succesfully test the wheel"
diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml
@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         # the different python versions for building wheels
-        python-version: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39]
+        python-version: [cp37-cp37m, cp38-cp38, cp39-cp39]
     env:
       # setting up python and docker image
       py: /opt/python/${{ matrix.python-version }}/bin/python
diff --git a/.github/workflows/release_linux_i686.yml b/.github/workflows/release_linux_i686.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
         architecture: ['x64']
 
     steps:
@@ -34,14 +34,14 @@ jobs:
         python-version: ${{ matrix.python-version }}
         architecture: ${{ matrix.architecture }}
 
-    - name: Build manylinux2010_i686
-      uses: docker://quay.io/pypa/manylinux2010_i686
+    - name: Build manylinux2014_i686
+      uses: docker://quay.io/pypa/manylinux2014_i686
       with:
         entrypoint: bash
-        args: .github/workflows/manylinux/entrypoint.sh ${{ matrix.python-version }} manylinux2010_i686 ${{ github.event_name }}
+        args: .github/workflows/manylinux/entrypoint.sh ${{ matrix.python-version }} manylinux2014_i686 ${{ github.event_name }}
 
-    - name: Test manylinux2010_i686 package in a clean docker environment
-      uses: docker://quay.io/pypa/manylinux2010_i686
+    - name: Test manylinux2014_i686 package in a clean docker environment
+      uses: docker://quay.io/pypa/manylinux2014_i686
       with:
         entrypoint: bash
         args: .github/workflows/manylinux/test_package_i686.sh ${{ matrix.python-version }}
diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
         architecture: ['x64']
 
     steps:
@@ -39,11 +39,11 @@ jobs:
         python -m pip install -q --upgrade pip
         python -m pip install -q -r requirements-release.txt
 
-    - name: Build manylinux2010_x86_64
-      uses: docker://quay.io/pypa/manylinux2010_x86_64
+    - name: Build manylinux2014_x86_64
+      uses: docker://quay.io/pypa/manylinux2014_x86_64
       with:
         entrypoint: bash
-        args: .github/workflows/manylinux/entrypoint.sh ${{ matrix.python-version }} manylinux2010_x86_64 ${{ github.event_name }}
+        args: .github/workflows/manylinux/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 ${{ github.event_name }}
 
     - name: Install protobuf in the GitHub Action environment for testing the wheel
       run: |
@@ -60,7 +60,7 @@ jobs:
 
     - name: Test wheel with Python ${{ matrix.python-version }}
       run: |
-        python -m pip install dist/*manylinux2010_x86_64.whl
+        python -m pip install dist/*manylinux2014_x86_64.whl
         pytest
 
     - name: Test backend test data
@@ -86,21 +86,21 @@ jobs:
       if: ${{ always() }}
       run: |
         python -m pip uninstall -y numpy onnx && python -m pip install numpy
-        python -m pip install dist/*manylinux2010_x86_64.whl
+        python -m pip install dist/*manylinux2014_x86_64.whl
         pytest
 
     - name: Verify ONNX with the latest protobuf
       if: ${{ always() }}
       run: |
         python -m pip uninstall -y protobuf onnx && python -m pip install protobuf
-        python -m pip install dist/*manylinux2010_x86_64.whl
+        python -m pip install dist/*manylinux2014_x86_64.whl
         pytest        
 
     - name: Verify ONNX with the minimum supported protobuf (from requirements.txt)
       if: ${{ always() }}
       run: |
         python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.12.2
-        python -m pip install dist/*manylinux2010_x86_64.whl
+        python -m pip install dist/*manylinux2014_x86_64.whl
         pytest
 
     - name: Verify ONNX with ort-nightly
diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -20,7 +20,7 @@ jobs:
     runs-on: macos-10.15
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
         architecture: ['x64']
 
     steps:
diff --git a/.github/workflows/release_win.yml b/.github/workflows/release_win.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
         architecture: ['x64', 'x86']
     steps:         
     - name: Checkout Protobuf 
diff --git a/.github/workflows/weekly_mac_ci.yml b/.github/workflows/weekly_mac_ci.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.9]
         architecture: ['x64']
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/win_no_exception_ci.yml b/.github/workflows/win_no_exception_ci.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
         architecture: ['x64']
     steps:
     - name: Checkout Protobuf 
diff --git a/docs/CIPipelines.md b/docs/CIPipelines.md
@@ -12,8 +12,8 @@
   [Windows_No_Exception CI](/.github/workflows/win_no_exception_ci.yml) | Every PR  | <ul><li>vs2019-winlatest</li><li>ONNX_DISABLE_EXCEPTIONS=ON</li><li>ONNX_USE_LITE_PROTO=ON</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li><li>ONNX_USE_MSVC_STATIC_RUNTIME=ON</li><li>ONNX_DISABLE_STATIC_REGISTRATION=ON or OFF</li></ul>| <ul><li>Only ONNX C++ tests</li><li>Test selective schema loading</li></ul> |
   [WindowsRelease](/.github/workflows/release_win.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly(1)</li></ul> | <ul><li>Latest Windows</li><li>x86 and x64</li><li>ONNX_USE_LITE_PROTO=ON</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li><li>ONNX_USE_MSVC_STATIC_RUNTIME=OFF</li></ul>| <ul><li> Release Windows wheel</li><li>Release onnx-weekly package</li><li>Verify backend node data</li><li>Verify node test generation</li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version(2)</li><li>Verify ONNX with the latest [ort-nightly](https://test.pypi.org/project/ort-nightly/)(3).</li></ul> |
   [LinuxRelease_aarch64](/.github/workflows/release_linux_aarch64.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul>  | <ul><li>Latest manylinux2014_aarch64</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul>| <ul><li> Release Linux aarch64 wheel</li><li>Release onnx-weekly package</li><li>Verify backend node data</li><li>Verify node test generation</li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li><li>Verify ONNX with the latest ort-nightly.</li></ul> |
-  [LinuxRelease_i686](/.github/workflows/release_linux_i686.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul>  | <ul><li>Latest manylinux2010_x86_64</li><li> Release Linux i686 wheel</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul> | <ul><li>Release onnx-weekly package</li><li>Verify backend node data</li><li>Verify node test generation</li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li></ul>
-  [LinuxRelease_x86_64](/.github/workflows/release_linux_x86_64.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul> | <ul><li>Latest manylinux2014_aarch64</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul>| <ul><li> Release Linux x86_64 wheel</li><li>Release onnx-weekly package</li><li>Test TEST_HUB=1(4)</li><li>Verify backend node data</li><li>Verify node test generation</li></li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li><li>Verify ONNX with the latest ort-nightly.</li></ul> |
+  [LinuxRelease_i686](/.github/workflows/release_linux_i686.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul>  | <ul><li>Latest manylinux2014_x86_64</li><li> Release Linux i686 wheel</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul> | <ul><li>Release onnx-weekly package</li><li>Verify backend node data</li><li>Verify node test generation</li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li></ul>
+  [LinuxRelease_x86_64](/.github/workflows/release_linux_x86_64.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul> | <ul><li>Latest LinuxRelease_x86_64</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul>| <ul><li> Release Linux x86_64 wheel</li><li>Release onnx-weekly package</li><li>Test TEST_HUB=1(4)</li><li>Verify backend node data</li><li>Verify node test generation</li></li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li><li>Verify ONNX with the latest ort-nightly.</li></ul> |
   [MacRelease](/.github/workflows/release_win.yml) | <ul><li>Main branch</li><li>Release branch</li><li>Weekly</li></ul> | <ul><li>macos-10.15</li><li> MACOSX_DEPLOYMENT_TARGET=10.12(5) </li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul>| <ul><li>Release Mac wheel</li><li>Release onnx-weekly package</li><li>Verify backend node data</li><li>Verify node test generation</li><li>Verify with different dependency versions - latest numpy version, latest and min supported protobuf version</li><li>Verify ONNX with the latest ort-nightly.</li><li>Test source distribution generation</li><li>Test build with source distribution</li><li>Release onnx-weekly source distribution</li></ul>  
   [Weekly CI with latest onnx.checker](/.github/workflows/weekly_mac_ci.yml) | weekly(6) |<ul><li>macos-latest</li><li>MACOSX_DEPLOYMENT_TARGET=10.15</li><li>ONNX_USE_PROTOBUF_SHARED_LIBS=OFF</li><li>ONNX_ML=1</li></ul>| <ul><li>Test latest ONNX checker</li><li>Test latest ONNX shape inference</li><li>With all models from [onnx/models](https://github.com/onnx/models)(7)</li></ul> |  
   
diff --git a/requirements-release.txt b/requirements-release.txt
@@ -1,4 +1,4 @@
-numpy == 1.16.6
+numpy == 1.21.5
 protobuf == 3.16.0
 pytest
 nbval
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
-numpy >= 1.16.6
+numpy >= 1.21.5
 protobuf >= 3.12.2
 typing-extensions >= 3.6.2.1

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-numpy == 1.16.6`
	`1`	`+numpy == 1.21.5`
`2`	`2`	`protobuf == 3.16.0`
`3`	`3`	`pytest`
`4`	`4`	`nbval`