Merge branch 'main' into sramakr1/apptainer_python

Author: sramakintel

.github/workflows/container-ci.yaml

@@ -119,7 +119,7 @@ jobs:
       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
       with:
         egress-policy: audit
-    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
       with:
         path: matrix
     - name: Set Matrix

.github/workflows/lint.yaml

@@ -37,7 +37,7 @@ jobs:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
-    - uses: super-linter/super-linter/slim@88ea3923a7e1f89dd485d079f6eb5f5e8f937589 # v6.6.0
+    - uses: super-linter/super-linter/slim@3fe03abab2eafb293ace16d4a3b07aeabcb3f1a0 # v6.7.0
       env:
         # To report GitHub Actions status checks
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/scorecard.yaml

@@ -48,7 +48,7 @@ jobs:
         results_format: sarif
         repo_token: ${{ secrets.GITHUB_TOKEN }}
         publish_results: true
-    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+    - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
       with:
         name: SARIF file
         path: results.sarif

.github/workflows/security-report.yaml

@@ -35,7 +35,7 @@ jobs:
         sarifReportDir: ${{ github.workspace }}
         template: report
         token: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+    - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
       with:
         name: Security Report Summary
         path: ./*.pdf

.github/workflows/test-runner-ci.yaml

@@ -37,7 +37,7 @@ jobs:
       with:
         egress-policy: audit
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+    - uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0
       with:
         driver: docker
     - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
@@ -80,7 +80,7 @@ jobs:
       with:
         egress-policy: audit
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-    - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+    - uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0
       with:
         driver: docker
     - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0

docs/requirements.txt

@@ -1,7 +1,7 @@
 mkdocs-callouts>=1.13.2
 mkdocs-git-authors-plugin>=0.8.0
 mkdocs-git-revision-date-localized-plugin>=1.2.5
-mkdocs-material==9.5.27
+mkdocs-material==9.5.28
 mkdocs-table-reader-plugin>=2.1.0
 mkdocs==1.6.0
 pandas>=2.0.3

preset/deep-learning/Dockerfile

@@ -421,7 +421,8 @@ RUN apt-get install -y --no-install-recommends --fix-missing \
         /etc/ssh/ssh_host_*_key.pub && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /var/run/sshd
+RUN mkdir -p /var/run/sshd && \
+    echo 'LoginGraceTime 0' >> /etc/ssh/sshd_config
 
 # https://github.com/openucx/ucx/issues/4742#issuecomment-584059909
 ENV UCX_TLS=ud,sm,self

pytorch/Dockerfile

@@ -85,9 +85,10 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin
 ENV SIGOPT_PROJECT=.
 
 WORKDIR /
-COPY multinode-requirements.txt .
+COPY multinode/requirements.txt requirements.txt
 
-RUN python -m pip install --no-cache-dir -r multinode-requirements.txt
+RUN python -m pip install --no-cache-dir -r requirements.txt && \
+    rm -rf requirements.txt
 
 ENV LD_LIBRARY_PATH="/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}:/usr/local/lib/python${PYTHON_VERSION}/dist-packages/oneccl_bindings_for_pytorch/opt/mpi/libfabric/lib:/usr/local/lib/python${PYTHON_VERSION}/dist-packages/oneccl_bindings_for_pytorch/lib"
 
@@ -99,16 +100,11 @@ RUN apt-get install -y --no-install-recommends --fix-missing \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Allow OpenSSH to talk to containers without asking for confirmation
-# hadolint global ignore=SC2002
-RUN mkdir -p /var/run/sshd && \
-    cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
-    echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
-    mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
+RUN mkdir -p /var/run/sshd
 
 ARG PYTHON_VERSION
 
-COPY generate_ssh_keys.sh .
+COPY multinode/generate_ssh_keys.sh /generate_ssh_keys.sh
 
 # modify generate_ssh_keys to be a helper script
 # print how to use helper script on bash startup
@@ -117,24 +113,9 @@ RUN echo "source /usr/local/lib/python${PYTHON_VERSION}/dist-packages/oneccl_bin
     cat '/generate_ssh_keys.sh' >> ~/.startup && \
     rm -rf /generate_ssh_keys.sh
 
-# hadolint global ignore=SC3037
-RUN echo -e "#!/bin/bash \n\
-set -e \n\
-set -a \n\
-source ~/.startup \n\
-set +a \n\
-eval \"\$@\"" >> /usr/local/bin/dockerd-entrypoint.sh && \
-    chmod +x /usr/local/bin/dockerd-entrypoint.sh
-
-RUN echo 'HostKey /etc/ssh/ssh_host_dsa_key' > /var/run/sshd_config && \
-    echo 'HostKey /etc/ssh/ssh_host_rsa_key' > /var/run/sshd_config && \
-    echo 'HostKey /etc/ssh/ssh_host_ecdsa_key' > /var/run/sshd_config && \
-    echo 'HostKey /etc/ssh/ssh_host_ed25519_key' > /var/run/sshd_config && \
-    echo 'AuthorizedKeysFile /etc/ssh/authorized_keys' > /var/run/sshd_config && \
-    echo '## Enable DEBUG log. You can ignore this but this may help you debug any issue while enabling SSHD for the first time' > /var/run/sshd_config && \
-    echo 'LogLevel DEBUG3' > /var/run/sshd_config && \
-    echo 'UsePAM yes' > /var/run/sshd_config && \
-    echo 'Subsystem       sftp    /usr/lib/openssh/sftp-server' > /var/run/sshd_config
+COPY multinode/dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh
+COPY multinode/sshd_config /etc/ssh/sshd_config
+COPY multinode/ssh_config /etc/ssh/ssh_config
 
 RUN mkdir -p /licensing
 

pytorch/README.md

@@ -45,9 +45,8 @@ The images below additionally include [Jupyter Notebook](https://jupyter.org/) s
 
 | Tag(s)                | Pytorch  | IPEX          | Driver | Jupyter Port | Dockerfile      |
 | --------------------- | -------- | ------------- | ------ | ------------ | --------------- |
-| `xpu-jupyter`         | [v2.1.0] | [v2.1.30+xpu] | [803]  | `8888`       | [v0.4.0-Beta]   |
-| `2.1.20-xpu-pip-base` | [v2.1.0] | [v2.1.20+xpu] | [803]  | `8888`       | [v0.3.4]        |
-| `2.1.10-xpu-pip-base` | [v2.1.0] | [v2.1.10+xpu] | [736]  | `8888`       | [v0.3.4]        |
+| `2.1.20-xpu-pip-jupyter` | [v2.1.0] | [v2.1.20+xpu] | [803]  | `8888`       | [v0.3.4]     |
+| `2.1.10-xpu-pip-jupyter` | [v2.1.0] | [v2.1.10+xpu] | [736]  | `8888`       | [v0.2.3]     |
 
 ### Run the XPU Jupyter Container
 
@@ -58,7 +57,7 @@ docker run -it --rm \
     --device /dev/dri \
     -v /dev/dri/by-path:/dev/dri/by-path \
     --ipc=host \
-    intel/intel-extension-for-pytorch:xpu-jupyter
+    intel/intel-extension-for-pytorch:2.1.20-xpu-pip-jupyter
 ```
 
 After running the command above, copy the URL (something like `http://127.0.0.1:$PORT/?token=***`) into your browser to access the notebook server.
@@ -114,12 +113,8 @@ The images below additionally include [Intel® oneAPI Collective Communications
 | `2.0.0-pip-multinode` | [v2.0.0] | [v2.0.0+cpu] | [v2.0.0][ccl-v2.0.0] | [v2.1.1]  | [v0.1.0]        |
 
 > **Note:** Passwordless SSH connection is also enabled in the image.
-> The container does not contain the SSH ID keys. The user needs to mount those keys at `/root/.ssh/id_rsa` and `/root/.ssh/id_rsa.pub`.
-> User also need to append content of id_rsa.pub in `/etc/ssh/authorized_keys` in the SSH server container.
-> Since the SSH key is not owned by default user account in docker, please also do "chmod 644 id_rsa.pub; chmod 644 id_rsa" to grant read access for default user account.
-> Users could also use "/usr/bin/ssh-keygen -t rsa -b 4096 -N '' -f ~/mnt/ssh_key/id_rsa" to generate a new SSH Key inside the container.
-> Users need to mount a config file to list all hostnames at location `/root/.ssh/config` on the SSH client container.
-> Once all files are added
+> The container does not contain the SSH ID keys. The user needs to mount those keys at `/root/.ssh/id_rsa` and `/etc/ssh/authorized_keys`.
+> Since the SSH key is not owned by default user account in docker, please also do "chmod 600 authorized_keys; chmod 600 id_rsa" to grant read access for default user account.
 
 #### Setup and Run IPEX Multi-Node Container
 
@@ -131,8 +126,7 @@ SSH Server (Worker)
 
 SSH Client (Launcher)
 
-1. *Config File with Host IPs* : `/root/.ssh/config`
-2. *Private User Key* : `/root/.ssh/id_rsa`
+1. *Private User Key* : `/root/.ssh/id_rsa`
 
 To add these files correctly please follow the steps described below.
 
@@ -146,47 +140,33 @@ To add these files correctly please follow the steps described below.
     cat id_rsa.pub >> authorized_keys
     ```
 
-2. Add hosts to config
-
-    The launcher container needs to have the a config file with all hostnames and ports specified. An example of a hostfile is provided below.
+2. Configure the permissions and ownership for all of the files you have created so far.
 
     ```bash
-    touch config
+    chmod 600 id_rsa config authorized_keys
+    chown root:root id_rsa.pub id_rsa config authorized_keys
     ```
 
+3. Setup hostfile. The hostfile is needed for running torch distributed using `ipexrun` utility. If you're not using `ipexrun` you can skip this step.
+
     ```txt
-    Host host1
-        HostName <Hostname of host1>
-        IdentitiesOnly yes
-        Port <SSH Port>
-    Host host2
-        HostName <Hostname of host2>
-        IdentitiesOnly yes
-        Port <SSH Port>
+    <Host 1 IP/Hostname>
+    <Host 2 IP/Hostname>
     ...
     ```
 
-3. Configure the permissions and ownership for all of the files you have created so far.
-
-    ```bash
-    chmod 600 id_rsa.pub id_rsa config authorized_keys
-    chown root:root id_rsa.pub id_rsa config authorized_keys
-    ```
-
 4. Now start the workers and execute DDP on the launcher.
 
     1. Worker run command:
 
         ```bash
-        export SSH_PORT=<SSH Port>
         docker run -it --rm \
             --net=host \
-            -v $PWD/authorized_keys:/root/.ssh/authorized_keys \
+            -v $PWD/authorized_keys:/etc/ssh/authorized_keys \
             -v $PWD/tests:/workspace/tests \
             -w /workspace \
-            -e SSH_PORT=${SSH_PORT} \
             intel/intel-extension-for-pytorch:2.3.0-pip-multinode \
-            bash -c '/usr/sbin/sshd -D -p ${SSH_PORT} -f /var/run/sshd_config'
+            bash -c '/usr/sbin/sshd -D'
         ```
 
     2. Launcher run command:
@@ -195,12 +175,65 @@ To add these files correctly please follow the steps described below.
         docker run -it --rm \
             --net=host \
             -v $PWD/id_rsa:/root/.ssh/id_rsa \
-            -v $PWD/config:/root/.ssh/config \
             -v $PWD/tests:/workspace/tests \
+            -v $PWD/hostfile:/workspace/hostfile \
             -w /workspace \
+            intel/intel-extension-for-pytorch:2.3.0-pip-multinode \
+            bash -c 'ipexrun cpu  --nnodes 2 --nprocs-per-node 1 --master-addr 127.0.0.1 --master-port 3022 /workspace/tests/ipex-resnet50.py --ipex --device cpu --backend ccl'
+        ```
+
+5. Start SSH server with a custom port.
+    If the user wants to define their own port to start the SSH server, it can be done so using the commands described below.
+
+    1. Worker command:
+
+        ```bash
+        export SSH_PORT=<User SSH Port>
+        docker run -it --rm \
+            --net=host \
+            -v $PWD/authorized_keys:/etc/ssh/authorized_keys \
+            -v $PWD/tests:/workspace/tests \
             -e SSH_PORT=${SSH_PORT} \
+            -w /workspace \
             intel/intel-extension-for-pytorch:2.3.0-pip-multinode \
-            bash -c 'ipexrun cpu /workspace/tests/ipex-resnet50.py --ipex --device cpu --backend ccl'
+            bash -c '/usr/sbin/sshd -D -p ${SSH_PORT}'
+        ```
+
+    2. Add hosts to config. (**Note:** This is an optional step)
+
+        User can optionally mount their own custom client config file to define a list of hosts and ports where the SSH server is running inside the container. An example of a hostfile is provided below. This file is supposed to be mounted in the launcher container at `/etc/ssh/ssh_config`.
+
+        ```bash
+        touch config
+        ```
+
+       ```txt
+        Host host1
+            HostName <Hostname of host1>
+            IdentitiesOnly yes
+            IdentityFile ~/.root/id_rsa
+            Port <SSH Port>
+        Host host2
+            HostName <Hostname of host2>
+            IdentitiesOnly yes
+            IdentityFile ~/.root/id_rsa
+            Port <SSH Port>
+        ...
+        ```
+
+    3. Launcher run command:
+
+        ```bash
+        docker run -it --rm \
+            --net=host \
+            -v $PWD/id_rsa:/root/.ssh/id_rsa \
+            -v $PWD/config:/etc/ssh/ssh_config \
+            -v $PWD/hostfile:/workspace/hostfile \
+            -v $PWD/tests:/workspace/tests \
+            -e SSH_PORT=${SSH_PORT} \
+            -w /workspace \
+            intel/intel-extension-for-pytorch:2.3.0-pip-multinode \
+            bash -c 'ipexrun cpu --nnodes 2 --nprocs-per-node 1 --master-addr 127.0.0.1 --master-port ${SSH_PORT} /workspace/tests/ipex-resnet50.py --ipex --device cpu --backend ccl'
         ```
 
 > [!NOTE]
@@ -246,6 +279,21 @@ The images below additionally include [Intel® oneAPI Collective Communications
 | `2.1.0-idp-mulitnode` | [v2.1.0] | [v2.1.0+cpu] | [v2.1.0][ccl-v2.1.0] | [v2.3.1]  | [v0.2.3]        |
 | `2.0.0-idp-multinode` | [v2.0.0] | [v2.0.0+cpu] | [v2.0.0][ccl-v2.0.0] | [v2.1.1]  | [v0.1.0]        |
 
+## XPU images with Intel® Distribution for Python*
+
+The images below are built only with CPU and GPU optimizations and include [Intel® Distribution for Python*]:
+
+| Tag(s)           | Pytorch  | IPEX         | Driver | Dockerfile      |
+| ---------------- | -------- | ------------ | -------- | ------ |
+| `2.1.10-xpu-idp-base` | [v2.1.0] | [v2.1.10+xpu]  | [736]  | [v0.2.3] |
+
+The images below additionally include [Jupyter Notebook](https://jupyter.org/) server:
+
+| Tag(s)                | Pytorch  | IPEX          | Driver | Jupyter Port | Dockerfile      |
+| --------------------- | -------- | ------------- | ------ | ------------ | --------------- |
+| `2.1.20-xpu-idp-jupyter` | [v2.1.0] | [v2.1.20+xpu] | [803]  | `8888`       | [v0.3.4]     |
+| `2.1.10-xpu-idp-jupyter` | [v2.1.0] | [v2.1.10+xpu] | [736]  | `8888`       | [v0.2.3]     |
+
 ## Build from Source
 
 To build the images from source, clone the [Intel® AI Containers](https://github.com/intel/ai-containers) repository, follow the main `README.md` file to setup your environment, and run the following command:

pytorch/docker-compose.yaml

@@ -77,7 +77,7 @@ services:
         dependency.apt.libglib2: true
         dependency.apt.python3-dev: true
         dependency.pip.apt.virtualenv: true
-        dependency.python.pip: multinode-requirements.txt
+        dependency.python.pip: multinode/requirements.txt
         org.opencontainers.base.name: "intel/intel-optimized-pytorch:${IPEX_VERSION:-2.2.0}-${PACKAGE_OPTION:-pip}-base"
         org.opencontainers.image.title: "Intel® Extension for PyTorch MultiNode Image"
         org.opencontainers.image.version: ${IPEX_VERSION:-2.2.0}-${PACKAGE_OPTION:-pip}-multinode

pytorch/multinode/dockerd-entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -a
+# shellcheck disable=SC1091
+source "$HOME/.startup"
+set +a
+"$@"

pytorch/generate_ssh_keys.sh pytorch/multinode/generate_ssh_keys.sh

@@ -0,0 +1,4 @@
+Host *
+    Port 3022
+    IdentityFile ~/.ssh/id_rsa
+    StrictHostKeyChecking no

pytorch/multinode-requirements.txt pytorch/multinode/requirements.txt

@@ -0,0 +1,12 @@
+HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+AuthorizedKeysFile /etc/ssh/authorized_keys
+## Enable DEBUG log. You can ignore this but this may help you debug any issue while enabling SSHD for the first time
+LogLevel DEBUG3
+Port 3022
+UsePAM yes
+Subsystem       sftp    /usr/lib/openssh/sftp-server
+# https://ubuntu.com/security/CVE-2024-6387
+LoginGraceTime 0

pytorch/multinode/ssh_config

@@ -106,7 +106,8 @@ ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
 ENV OMPI_MCA_tl_tcp_if_exclude="lo,docker0"
 
 # Install OpenSSH for MPI to communicate between containers
-RUN mkdir -p /var/run/sshd
+RUN mkdir -p /var/run/sshd && \
+    echo 'LoginGraceTime 0' >> /etc/ssh/sshd_config
 
 # Install Horovod
 ARG HOROVOD_WITH_TENSORFLOW=1

pytorch/multinode/sshd_config

@@ -3,7 +3,7 @@ coverage>=7.5.0
 coveralls>=4.0.1
 expandvars>=0.12.0
 hypothesis>=6.100.1
-pydantic==2.7.4
+pydantic==2.8.2
 pylint>=3.1.0
 pytest>=8.1.1
 python_on_whales>=0.70.1