Merge branch 'master' into onednn_kv_cache_compression

Author: p-durandin

.github/actions/handle_docker/get_images_to_build.py

@@ -64,7 +64,10 @@ def main():
             expected_tag = f'pr-{args.pr}'
 
             if head_tag != expected_tag:
-                logger.error(f"Please update docker tag in {args.head_tag_file} to {expected_tag}")
+                logger.error(f"Some of your changes affected Docker environment for CI. "
+                             f"Please update docker tag in {args.head_tag_file} to {expected_tag}. "
+                             f"For more details please see "
+                             f"https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/ci/github_actions/docker_images.md")
                 sys.exit(1)
 
     elif merge_queue_target_branch:

.github/actions/setup_python/action.yml

@@ -22,7 +22,6 @@ inputs:
 runs:
   using: 'composite'
   steps:
-
     - name: Check if Python is already installed (Linux)
       if: ${{ runner.os == 'Linux' }}
       shell: bash
@@ -54,13 +53,11 @@ runs:
       with:
         python-version: ${{ inputs.version }}
 
-    - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }}
+    - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false') }}
       name: Setup Python ${{ inputs.version }}
       uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
       with:
         python-version: ${{ inputs.version }}
-      env:
-        PIP_CACHE_DIR: ${{ inputs.self-hosted-runner == 'true' && inputs.pip-cache-path || '' }}
 
     - if: ${{ inputs.should-setup-pip-paths == 'true' && runner.os != 'Windows' }}
       name: Setup pip variables (cache and install path)
@@ -82,9 +79,4 @@ runs:
     - if: ${{ inputs.show-cache-info == 'true' }}
       name: Get pip cache info
       shell: bash
-      run: |
-        echo "Cache size: "
-        du -h -d2 ${{ env.PIP_CACHE_DIR }}
-        echo "Cache info: "
-        python3 -m pip cache info
-      continue-on-error: true
+      run: python3 -m pip cache info

.github/components.yml

@@ -110,7 +110,6 @@ IR_FE:
 
 ONNX_FE:
   revalidate:
-    - MO
     - OVC
     - ONNX_RT
   build:
@@ -119,15 +118,13 @@ ONNX_FE:
 
 PDPD_FE:
   revalidate:
-    - MO
     - OVC
   build:
     - CPU
     - Python_API
 
 TF_FE:
   revalidate:
-    - MO
     - OVC
   build:
     - CPU
@@ -136,23 +133,21 @@ TF_FE:
 
 TFL_FE:
   revalidate:
-    - MO
     - OVC
   build:
     - CPU
     - Python_API
 
 PyTorch_FE:
   revalidate:
-    - MO
     - OVC
   build:
     - CPU
     - Python_API
+    - TOKENIZERS # PyTorch_FE tests depend on tokenizers build
 
 JAX_FE:
   revalidate:
-    - MO
     - OVC
   build:
     - CPU
@@ -170,10 +165,10 @@ C_API:
 Python_API:
   revalidate:
     - samples
-    - MO
     - OVC
     - tools
     - TF_FE
+    - docs_snippets
   build:
     - CPU
     - HETERO
@@ -226,14 +221,6 @@ OVC:
     - Python_API
     - TOKENIZERS # TF_FE tests depends on tokenizers build
 
-MO:
-  revalidate:
-    - PyTorch_FE
-    - TF_FE
-  build:
-    - Python_API
-    - TOKENIZERS # TF_FE tests depends on tokenizers build
-
 tools:
   build:
     - CPU
@@ -242,6 +229,12 @@ tools:
 docs:
   revalidate: []
   build: []
+  
+docs_snippets:
+  revalidate:
+    - docs_snippets
+  build:
+    - Python_API
 
 licensing:
   revalidate: []

.github/dependabot.yml

@@ -141,3 +141,18 @@ updates:
       - "mryzhov"
       - "ilya-lavrenov"
     open-pull-requests-limit: 3
+
+  # Docker images
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: "daily"
+      time: "09:00"
+      timezone: "Asia/Dubai"
+    allow:
+      - dependency-type: "direct"
+    assignees:
+      - "akashchi"
+      - "mryzhov"
+      - "ilya-lavrenov"
+    open-pull-requests-limit: 3

.github/dockerfiles/docker_tag

@@ -1 +1 @@
-pr-27430
+pr-28380

.github/dockerfiles/ov_build/fedora_29/Dockerfile

@@ -3,7 +3,11 @@ FROM ${REGISTRY}/library/fedora:29
 
 USER root
 
-RUN yum update -y && yum install -y \
+# dnf configuration
+RUN echo "timeout=60" >> /etc/dnf/dnf.conf && \
+    echo "retries=10" >> /etc/dnf/dnf.conf
+
+RUN dnf update -y && dnf install -y \
     git \
     curl \
     python3 \

.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile

@@ -62,10 +62,13 @@ RUN echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricte
 
 RUN dpkg --add-architecture riscv64 && \
     apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list && \
-    apt-get install -y --no-install-recommends libpython3-dev:riscv64
+    apt-get install -y --no-install-recommends libpython3-dev:riscv64 && \
+    apt-get install libgomp1:riscv64 && \
+    apt-get install libatomic1:riscv64
 
 # Setup pip
 ENV PIP_VERSION="24.0"
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
     python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
     rm -f get-pip.py
+

.github/dockerfiles/ov_build/ubuntu_22_04_riscv_xuantie/Dockerfile

@@ -0,0 +1,102 @@
+ARG REGISTRY="docker.io"
+FROM ${REGISTRY}/library/ubuntu:22.04
+
+USER root
+
+# APT configuration
+RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
+    echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    TZ="Europe/London"
+
+RUN apt-get update && \
+    apt-get install software-properties-common && \
+    add-apt-repository --yes --no-update ppa:git-core/ppa && \
+    add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
+    apt-get update && \
+    # install compilers to build OpenVINO for RISC-V 64
+    apt-get install gcc-riscv64-linux-gnu g++-riscv64-linux-gnu && \
+    apt-get install \
+        curl \
+        git \
+        cmake \
+        ccache \
+        ninja-build \
+        fdupes \
+        patchelf \
+        ca-certificates \
+        gpg-agent \
+        tzdata \
+        # parallel gzip
+        pigz \
+        # Python \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+        python3-distutils \
+        # Compilers
+        gcc \
+        g++ \
+        # xuantie-gnu-toolchain build dependencies
+        autoconf \
+        automake \
+        autotools-dev \
+        libmpc-dev \
+        libmpfr-dev\ 
+        libgmp-dev \
+        gawk \
+        build-essential \
+        bison \
+        flex \
+        texinfo \
+        gperf \
+        libtool \
+        patchutils \
+        bc \
+        zlib1g-dev \
+        libexpat-dev \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install RISC-V native debian packages
+RUN echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted > riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy universe >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates universe >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy multiverse >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security universe >> riscv64-sources.list && \
+    echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security multiverse >> riscv64-sources.list && \
+    echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy main >> riscv64-sources.list && \
+    echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy universe >> riscv64-sources.list && \
+    echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main >> riscv64-sources.list && \
+    echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main >> riscv64-sources.list && \
+    mv riscv64-sources.list /etc/apt/sources.list.d/
+
+RUN dpkg --add-architecture riscv64 && \
+    apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list && \
+    apt-get install -y --no-install-recommends libpython3-dev:riscv64
+
+# build xuintie toolchain
+ARG XUANTIE_VERSION="V2.8.1"
+ARG XUANTIE_REPO="https://github.com/XUANTIE-RV/xuantie-gnu-toolchain"
+ARG XUINTIE_PATH="/opt/riscv"
+ARG XUINTIE_TMP_PATH="/tmp/xuantie"
+ARG XUINTIE_SRC="/tmp/xuantie/src"
+    
+RUN mkdir -p ${XUINTIE_TMP_PATH} && cd ${XUINTIE_TMP_PATH} && \
+    git clone --branch ${XUANTIE_VERSION} --depth 1 ${XUANTIE_REPO} ${XUINTIE_SRC} && cd ${XUINTIE_SRC} && \
+    ./configure --prefix=${XUINTIE_PATH} --disable-gdb && \
+    make linux -j$(nproc) && make install && \
+    rm -rf ${XUINTIE_TMP_PATH}
+
+# Setup pip
+ENV PIP_VERSION="24.0"
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
+    rm -f get-pip.py

.github/dockerfiles/ov_test/debian_10_py310/Dockerfile

@@ -0,0 +1,76 @@
+ARG REGISTRY="docker.io"
+FROM ${REGISTRY}/library/debian:10.13
+
+USER root
+
+# APT configuration
+RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
+    echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
+    echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    TZ="Europe/London"
+
+RUN apt-get update && \
+    apt-get install  \
+        git \
+        libc6-dev \
+        # parallel gzip
+        pigz \
+        # Python
+        python3 \
+        python3-pip \
+        python3-dev \
+        python3-venv \
+        python3-distutils \
+        # To build Python 3.10 from source
+        build-essential \
+        libffi-dev \
+        libgdbm-dev \
+        libc6-dev \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        xz-utils \
+        tk-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        liblzma-dev \
+        wget \
+        curl \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install openvino dependencies
+ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh
+RUN chmod +x /install_openvino_dependencies.sh && \
+    /install_openvino_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/*
+
+# Setup Python 3.10
+RUN wget https://www.python.org/ftp/python/3.10.9/Python-3.10.9.tar.xz
+
+RUN tar -xf Python-3.10.9.tar.xz && \
+    cd Python-3.10.9 && \
+    ./configure --enable-optimizations && \
+    make -j 8 && \
+    make altinstall
+
+# Setup pip
+ENV PIP_VERSION="24.0"
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3.10 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
+    rm -f get-pip.py
+
+# Use Python 3.10 as default instead of Python 3.7
+# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
+RUN python3.10 -m venv venv
+ENV PATH="/venv/bin:$PATH"
+
+ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
+ENV PIP_INSTALL_PATH=/venv/lib/python3.10/site-packages

.github/dockerfiles/ov_test/fedora_33/Dockerfile

@@ -3,10 +3,20 @@ FROM ${REGISTRY}/library/fedora:33
 
 USER root
 
-RUN yum update -y && yum install -y \
+# dnf configuration
+RUN echo "timeout=60" >> /etc/dnf/dnf.conf && \
+    echo "retries=10" >> /etc/dnf/dnf.conf
+
+RUN dnf update -y && dnf install -y \
     git \
     curl \
-    python3
+    python3 \
+    findutils \
+    ocl-icd \
+    ocl-icd-devel \
+    # parallel gzip
+    pigz \
+    xz
 
 # Install Node
 ENV NODE_VERSION=21.7.3