Skip to content

Commit 5680236

Browse files
author
Tyler Titsworth
authored
Merge branch 'develop' into ttitswor/composite-external-source
2 parents 101416a + 6636c37 commit 5680236

26 files changed

+137
-63
lines changed

‎.github/dependabot.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ version: 2
2121
updates:
2222
- directory: /test-runner
2323
groups:
24-
python-requirements:
24+
test-runner:
2525
patterns:
2626
- "*"
2727
package-ecosystem: pip
@@ -33,39 +33,39 @@ updates:
3333
interval: weekly
3434
- directory: /pytorch
3535
groups:
36-
python-requirements:
36+
pytorch:
3737
patterns:
3838
- "*"
3939
package-ecosystem: pip
4040
schedule:
4141
interval: weekly
4242
- directory: /classical-ml
4343
groups:
44-
python-requirements:
44+
classical-ml:
4545
patterns:
4646
- "*"
4747
package-ecosystem: pip
4848
schedule:
4949
interval: weekly
5050
- directory: /tensorflow
5151
groups:
52-
python-requirements:
52+
tensorflow:
5353
patterns:
5454
- "*"
5555
package-ecosystem: pip
5656
schedule:
5757
interval: weekly
5858
- directory: /python
5959
groups:
60-
python-requirements:
60+
python:
6161
patterns:
6262
- "*"
6363
package-ecosystem: pip
6464
schedule:
6565
interval: weekly
6666
- directory: /docs
6767
groups:
68-
python-requirements:
68+
docs:
6969
patterns:
7070
- "*"
7171
package-ecosystem: pip

‎.github/workflows/integration-test.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ jobs:
3636
# Get diff array filtered by specific filetypes
3737
DIFF=$(git diff --diff-filter=d \
3838
--name-only ${{ github.event.merge_group.base_sha || github.event.pull_request.base.sha }}...${{ github.event.merge_group.head_sha || github.event.pull_request.head.sha }} \
39-
-- '*/Dockerfile' '*.py' '*.yaml' '*.yml' '*.sh' '*/requirements.txt' '*.json' | \
39+
-- '*/*Dockerfile' '*.py' '*.yaml' '*.yml' '*.sh' '*/*requirements.txt' '*.json' | \
4040
jq -R '.' | jq -sc '.' \
4141
)
4242
# Search for compose files in each file to determine the container groups

‎.github/workflows/lint.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
- uses: actions/checkout@v4
3434
with:
3535
fetch-depth: 0
36-
- uses: super-linter/super-linter/slim@v6.4.1
36+
- uses: super-linter/super-linter/slim@v6.5.0
3737
env:
3838
# To report GitHub Actions status checks
3939
GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }}

‎classical-ml/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ services:
2727
BASE_IMAGE_TAG: ${BASE_IMAGE_TAG:-22.04}
2828
GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-0}
2929
IDP_VERSION: core
30-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
30+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
3131
PYTHON_VERSION: ${PYTHON_VERSION:-3.10}
3232
PACKAGE_OPTION: ${PACKAGE_OPTION:-pip}
3333
REGISTRY: ${REGISTRY}

‎classical-ml/jupyter-requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
jupyterlab==4.2.0
22
jupyterhub==4.1.5
3-
notebook==7.2.0rc1
3+
notebook==7.2.0
44
jupyter-server-proxy>=4.1.2

‎classical-ml/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
daal4py==2024.4.0
2-
matplotlib==3.8.4
2+
matplotlib==3.9.0
33
numpy==1.26.4
44
scikit-learn-intelex==2024.4.0
55
threadpoolctl==3.5.0

‎docs/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
mkdocs-callouts>=1.13.2
22
mkdocs-git-authors-plugin>=0.8.0
33
mkdocs-git-revision-date-localized-plugin>=1.2.5
4-
mkdocs-material==9.5.22
4+
mkdocs-material==9.5.24
55
mkdocs-table-reader-plugin>=2.1.0
66
mkdocs==1.6.0
77
pandas>=2.0.3

‎preset/classical-ml/Dockerfile

+6-6
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,16 @@ USER dev
5151
WORKDIR /home/dev
5252

5353
ENV CONDA_ROOT=/home/dev/conda
54-
ARG MINICONDA_VERSION
54+
ARG MINIFORGE_VERSION
5555
ARG PYTHON_VERSION
5656
ARG IDP_VERSION
5757
ARG INTEL_CHANNEL
5858

59-
RUN wget --progress=dot:giga --no-check-certificate https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}.sh -O miniconda.sh && \
60-
chmod +x miniconda.sh && \
61-
./miniconda.sh -b -p "${CONDA_ROOT}" && \
62-
rm ./miniconda.sh && \
63-
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniconda3" && \
59+
RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh -O miniforge.sh && \
60+
chmod +x miniforge.sh && \
61+
./miniforge.sh -b -p "${CONDA_ROOT}" && \
62+
rm ./miniforge.sh && \
63+
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge3" && \
6464
export PATH="${CONDA_ROOT}/bin/:${PATH}" && \
6565
conda update -y conda && \
6666
conda config --add channels conda-forge && \

‎preset/classical-ml/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ services:
2525
DPNP_VERSION: ${NUMBA_DPEX_VERSION:-0.14.0}
2626
IDP_VERSION: ${IDP_VERSION:-2024.1.0}
2727
INTEL_CHANNEL: ${INTEL_CHANNEL:-intel}
28-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
28+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
2929
MODIN_VERSION: ${MODIN_VERSION:-0.26.1}
3030
MPI_VERSION: ${MPI_VERSION:-2021.12.0}
3131
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}

‎preset/classical-ml/notebooks/xgboost/IntelPython_XGBoost_Performance.ipynb

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
"id": "028a3491",
4141
"metadata": {},
4242
"source": [
43-
"This example is executed in two Anaconda environments. \n",
43+
"This example is executed in two Conda environments. \n",
4444
"\n",
4545
"The first environment is the latest [Intel® oneAPI AI Analytics Toolkit](https://software.intel.com/content/www/us/en/develop/tools/oneapi/ai-analytics-toolkit.html) base environment, which includes data analytics and machine learning workflows and Intel optimizations for XGboost. See [here](https://software.intel.com/content/www/us/en/develop/articles/installing-ai-kit-with-conda.html) for more installation information."
4646
]

‎preset/data-analytics/Dockerfile

+6-6
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,16 @@ WORKDIR /home/dev
5252

5353
ENV CONDA_ROOT=/home/dev/conda
5454

55-
ARG MINICONDA_VERSION
55+
ARG MINIFORGE_VERSION
5656
ARG PYTHON_VERSION
5757
ARG IDP_VERSION
5858
ARG INTEL_CHANNEL
5959

60-
RUN wget --progress=dot:giga --no-check-certificate "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}.sh" -O miniconda.sh && \
61-
chmod +x miniconda.sh && \
62-
./miniconda.sh -b -p "${CONDA_ROOT}" && \
63-
rm ./miniconda.sh && \
64-
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniconda3" && \
60+
RUN wget --progress=dot:giga --no-check-certificate "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh" -O miniforge.sh && \
61+
chmod +x miniforge.sh && \
62+
./miniforge.sh -b -p "${CONDA_ROOT}" && \
63+
rm ./miniforge.sh && \
64+
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge3" && \
6565
export PATH="${CONDA_ROOT}/bin/:${PATH}" && \
6666
conda update -y conda && \
6767
conda config --add channels conda-forge && \

‎preset/data-analytics/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ services:
2525
DPNP_VERSION: ${NUMBA_DPEX_VERSION:-0.14.0}
2626
IDP_VERSION: ${IDP_VERSION:-2024.1.0}
2727
INTEL_CHANNEL: ${INTEL_CHANNEL:-intel}
28-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
28+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
2929
MODIN_VERSION: ${MODIN_VERSION:-0.26.1}
3030
MPI_VERSION: ${MPI_VERSION:-2021.12.0}
3131
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}

‎preset/deep-learning/Dockerfile

+7-7
Original file line numberDiff line numberDiff line change
@@ -101,17 +101,17 @@ WORKDIR /home/dev
101101

102102
ENV CONDA_ROOT=/home/dev/conda
103103

104-
# Miniconda Python Installation
105-
ARG MINICONDA_VERSION
104+
# Miniforge Python Installation
105+
ARG MINIFORGE_VERSION
106106
ARG PYTHON_VERSION
107107
ARG IDP_VERSION
108108
ARG INTEL_CHANNEL
109109

110-
RUN wget --progress=dot:giga --no-check-certificate "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}.sh" -O miniconda.sh && \
111-
chmod +x miniconda.sh && \
112-
./miniconda.sh -b -p "${CONDA_ROOT}" && \
113-
rm ./miniconda.sh && \
114-
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniconda3" && \
110+
RUN wget --progress=dot:giga --no-check-certificate "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh" -O miniforge.sh && \
111+
chmod +x miniforge.sh && \
112+
./miniforge.sh -b -p "${CONDA_ROOT}" && \
113+
rm ./miniforge.sh && \
114+
ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge3" && \
115115
export PATH="${CONDA_ROOT}/bin/:${PATH}" && \
116116
conda update -y conda && \
117117
conda config --add channels conda-forge && \

‎preset/deep-learning/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ services:
3636
LEVEL_ZERO_DEV_VER: 1.14.0-744~22.04
3737
LEVEL_ZERO_GPU_VER: 1.3.27642.40-803~22.04
3838
LEVEL_ZERO_VER: 1.14.0-744~22.04
39-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
39+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
4040
MPI_VERSION: ${MPI_VERSION:-2021.12.0}
4141
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}
4242
NUMPY_VERSION: ${NUMPY_VERSION:-1.26.4}

‎preset/inference-optimization/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ services:
3535
LEVEL_ZERO_DEV_VER: 1.14.0-744~22.04
3636
LEVEL_ZERO_GPU_VER: 1.3.27642.40-803~22.04
3737
LEVEL_ZERO_VER: 1.14.0-744~22.04
38-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
38+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
3939
MPI_VERSION: ${MPI_VERSION:-2021.12.0}
4040
NEURAL_COMPRESSOR_VERSION: ${NEURAL_COMPRESSOR_VERSION:-2.4.1}
4141
NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}

‎python/Dockerfile

+6-6
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ RUN apt-get update -y && \
4545
FROM base AS idp
4646

4747
ARG PYTHON_VERSION
48-
ARG MINICONDA_VERSION
48+
ARG MINIFORGE_VERSION
4949
ARG IDP_VERSION=core
5050

51-
RUN wget --progress=dot:giga --no-check-certificate "https://repo.anaconda.com/miniconda/Miniconda3-${MINICONDA_VERSION}.sh" -O miniconda.sh && \
52-
chmod +x miniconda.sh && \
53-
./miniconda.sh -b -p /opt/conda && \
54-
rm ./miniconda.sh && \
55-
ln -s /opt/conda ~/miniconda3 && \
51+
RUN wget --progress=dot:giga --no-check-certificate "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh" -O miniforge.sh && \
52+
chmod +x miniforge.sh && \
53+
./miniforge.sh -b -p /opt/conda && \
54+
rm ./miniforge.sh && \
55+
ln -s /opt/conda ~/miniforge3 && \
5656
export PATH="/opt/conda/bin/:${PATH}" && \
5757
conda update -y conda && \
5858
conda config --add channels intel && \

‎python/docker-compose.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ services:
1616
idp:
1717
build:
1818
args:
19-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
19+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
2020
context: .
2121
labels:
2222
dependency.apt.wget: true

‎pytorch/Dockerfile

+3-8
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,15 @@ COPY multinode-requirements.txt .
9191

9292
RUN python -m pip install --no-cache-dir -r multinode-requirements.txt
9393

94+
ARG PYTHON_VERSION
95+
RUN echo "source /usr/local/lib/python${PYTHON_VERSION}/dist-packages/oneccl_bindings_for_pytorch/env/setvars.sh" >> ~/.bashrc
96+
9497
RUN mkdir -p /licensing
9598

9699
RUN wget -q --no-check-certificate https://raw.githubusercontent.com/oneapi-src/oneCCL/b7d66de16e17f88caffd7c6df4cd5e12b266af84/third-party-programs.txt -O /licensing/oneccl_third_party_programs.txt && \
97100
wget -q --no-check-certificate https://raw.githubusercontent.com/intel/neural-compressor/master/docker/third-party-programs-pytorch.txt -O /licensing/third-party-programs-pytorch.txt && \
98101
wget -q --no-check-certificate https://raw.githubusercontent.com/intel/neural-compressor/master/LICENSE -O /licensing/LICENSE
99102

100-
FROM multinode AS multinode-pip
101-
102-
RUN apt-get install -y --no-install-recommends --fix-missing virtualenv
103-
104-
RUN virtualenv --system-site-packages inc && echo "source /inc/bin/activate" > ~/.bashrc
105-
106-
FROM multinode AS multinode-idp
107-
108103
FROM ${PYTHON_BASE} AS ipex-xpu-base
109104

110105
RUN apt-get update && \

‎pytorch/docker-compose.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ services:
2626
BASE_IMAGE_TAG: ${BASE_IMAGE_TAG:-22.04}
2727
GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-0}
2828
IPEX_VERSION: ${IPEX_VERSION:-2.3.0}
29-
MINICONDA_VERSION: ${MINICONDA_VERSION:-latest-Linux-x86_64}
29+
MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
3030
NO_PROXY: ''
3131
PACKAGE_OPTION: ${PACKAGE_OPTION:-pip}
3232
PYTHON_VERSION: ${PYTHON_VERSION:-3.10}
@@ -81,7 +81,7 @@ services:
8181
org.opencontainers.base.name: "intel/intel-optimized-pytorch:${IPEX_VERSION:-2.2.0}-${PACKAGE_OPTION:-pip}-base"
8282
org.opencontainers.image.title: "Intel® Extension for PyTorch MultiNode Image"
8383
org.opencontainers.image.version: ${IPEX_VERSION:-2.2.0}-${PACKAGE_OPTION:-pip}-multinode
84-
target: multinode-${PACKAGE_OPTION:-pip}
84+
target: multinode
8585
command: >
8686
sh -c "python -c 'import neural_compressor;import
8787
oneccl_bindings_for_pytorch as oneccl; print(\"Neural Compressor

‎pytorch/requirements.txt

-2
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,3 @@ torchaudio==2.3.0
44
-f https://download.pytorch.org/whl/cpu/torch_stable.html
55
intel_extension_for_pytorch==2.3.0+cpu
66
--extra-index-url https://developer.intel.com/ipex-whl-stable-cpu
7-
setuptools==69.5.1
8-
psutil==5.9.8

‎pytorch/tests/inc-quant.py

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) 2024 Intel Corporation
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from neural_compressor.config import PostTrainingQuantConfig
16+
from neural_compressor.data import DataLoader, Datasets
17+
from neural_compressor.quantization import fit
18+
19+
# pylint: skip-file
20+
from torchvision import models
21+
22+
float_model = models.resnet18()
23+
dataset = Datasets("pytorch")["dummy"](shape=(1, 3, 224, 224))
24+
calib_dataloader = DataLoader(framework="pytorch", dataset=dataset)
25+
static_quant_conf = PostTrainingQuantConfig()
26+
quantized_model = fit(
27+
model=float_model, conf=static_quant_conf, calib_dataloader=calib_dataloader
28+
)

‎pytorch/tests/ipex-resnet50.py

+23
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,44 @@
1414

1515
# pylint: skip-file
1616
import argparse
17+
import os
1718

1819
import intel_extension_for_pytorch as ipex
1920
import torch
21+
import torch.distributed as dist
2022
import torchvision.models as models
23+
from torch.nn.parallel import DistributedDataParallel as DDP
24+
25+
os.environ["RANK"] = str(os.environ.get("PMI_RANK", 0))
26+
os.environ["WORLD_SIZE"] = str(os.environ.get("PMI_SIZE", 1))
27+
init_method = "tcp://127.0.0.1:29500"
2128

2229
parser = argparse.ArgumentParser()
2330
parser.add_argument("--device", default="cpu", choices=["cpu", "xpu"])
2431
parser.add_argument("--ipex", action="store_true")
32+
parser.add_argument("--backend", default="gloo", choices=["gloo", "ccl"])
2533
args = parser.parse_args()
2634

35+
try:
36+
import oneccl_bindings_for_pytorch
37+
except:
38+
pass
39+
40+
dist.init_process_group(
41+
backend=args.backend,
42+
init_method=init_method,
43+
world_size=int(os.environ.get("WORLD_SIZE")),
44+
rank=int(os.environ.get("RANK")),
45+
)
46+
2747
model = models.resnet50(pretrained=False)
48+
2849
model.eval()
2950
data = torch.rand(1, 3, 224, 224)
3051

3152
model = model.to(args.device)
53+
if dist.get_world_size() > 1:
54+
model = DDP(model, device_ids=[args.device] if (args.device != "cpu") else None)
3255
data = data.to(args.device)
3356
print("Choosing device: {}".format(args.device))
3457

0 commit comments

Comments
 (0)