Merge branch 'master' into fix/20210

Author: p-wysocki

.github/actions/cache/package-lock.json

@@ -16,12 +16,14 @@ def add_common_args(parser: argparse.ArgumentParser):
                         default=os.getenv('GITHUB_EVENT_NAME'))
     parser.add_argument('--storage_root', help='Root path of the artifacts storage', required=False,
                         default=os.getenv('ARTIFACTS_SHARE'))
+    parser.add_argument('-n', '--product_name', required=False, default='dldt',
+                        help='Product name for which artifacts are generated')
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument('-d', '--storage_dir', help='Subdirectory name for artifacts, same as product type',
-                       choices=[platform_key.value for platform_key in ProductType])
-    group.add_argument('-p', '--platform', type=str,
+                       choices=[product_type.value for product_type in ProductType], type=str.lower)
+    group.add_argument('-p', '--platform', type=str.lower,
                        help='Platform for which to restore artifacts. Used if storage_dir is not set',
-                       choices=[product_type.value for product_type in PlatformKey])
+                       choices=[platform_key.value for platform_key in PlatformKey])
 
 
 def get_event_type(event_name: str = os.getenv('GITHUB_EVENT_NAME')) -> str:

.github/actions/common/artifact_utils.py

@@ -14,11 +14,12 @@ class EventType(Enum):
     'public_linux_ubuntu_22_04_x86_64_release',
     'public_linux_ubuntu_22_04_dpcpp_x86_64_release',
     'public_linux_ubuntu_24_04_x86_64_release',
-    'public_windows_vs2019_Release',
-    'public_windows_vs2019_Debug',
-    'public_windows_vs2022_Release',
-    'public_windows_vs2022_Debug',
+    'public_windows_vs2019_release',
+    'public_windows_vs2019_debug',
+    'public_windows_vs2022_release',
+    'public_windows_vs2022_debug',
     'public_manylinux2014_x86_64_release',
+    'public_macos_x86_64_release',
 )
 ProductType = Enum('ProductType', {t.upper(): t for t in productTypes})
 
@@ -44,4 +45,5 @@ class EventType(Enum):
     PlatformKey.UBUNTU22_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_22_04_X86_64_RELEASE,
     PlatformKey.UBUNTU24_X86_64: ProductType.PUBLIC_LINUX_UBUNTU_24_04_X86_64_RELEASE,
     PlatformKey.WINDOWS_X86_64: ProductType.PUBLIC_WINDOWS_VS2022_RELEASE,
+    PlatformKey.MACOS_12_6_X86_64: ProductType.PUBLIC_MACOS_X86_64_RELEASE,
 }

.github/actions/common/constants.py

@@ -82,7 +82,7 @@ def parse_ov_version(header_file: str | Path) -> str:
 
 def generate_manifest(repos: list, product_type: str, event_type: str, build_type: str, target_arch: str) -> Manifest:
     manifest = Manifest()
-    component_name = 'dldt'  # historical, keep for internal compatibility
+    component_name = None
     repositories = []
     ov_version = None
     trigger_repo = None
@@ -95,6 +95,7 @@ def generate_manifest(repos: list, product_type: str, event_type: str, build_typ
             ov_version = parse_ov_version(version_file)
         if repo.trigger:
             trigger_repo = repo
+            component_name = repo.name
 
     custom_branch_name = f'-{trigger_repo.branch}' if trigger_repo.branch != 'master' else ''
     run_number_postfix = f'-{os.environ.get("GITHUB_RUN_NUMBER")}' if os.environ.get("GITHUB_RUN_NUMBER") else ''

.github/actions/create_manifest/create_manifest.py

@@ -141,7 +141,7 @@ runs:
       run: |
         artifacts_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
         cd $artifacts_path
-        version=$(yq eval '.components.dldt.custom_params.wheel_product_version' manifest.yml)
+        version=$(yq eval '.components.openvino.custom_params.wheel_product_version // .components.dldt.custom_params.wheel_product_version' manifest.yml)
         wheel_path=${{ inputs.install_dir && '$artifacts_path/wheels' || './wheels' }}
         default_find_links_cmd="--find-links=$wheel_path"
         find_links_cmd=$([[ -n "$PIP_FIND_LINKS" ]] && echo "" || echo "$default_find_links_cmd")

.github/actions/openvino_provider/action.yml

@@ -16,6 +16,9 @@ inputs:
   branch_name:
     description: "Target branch name. If not set - taken from github context"
     required: false
+  product_name:
+    description: "Product name for which artifacts are generated"
+    required: false
   trigger_repo_sha:
     description: "Commit hash of a trigger repo. If not set - taken from github context"
     required: false
@@ -50,6 +53,7 @@ runs:
         ${{ inputs.unpack_archives == 'true' && '--unpack_archives' || '' }}
         ${{ inputs.branch_name && env.BRANCH_PARAM || '' }}
         ${{ inputs.event_name && env.EVENT_PARAM || '' }}
+        ${{ inputs.product_name && env.PRODUCT_NAME_PARAM || '' }}
         ${{ inputs.to_restore && env.TO_RESTORE || '' }}
       env:
         ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}
@@ -58,6 +62,7 @@ runs:
         PLATFORM_PARAM: "--platform ${{ inputs.platform }}"
         BRANCH_PARAM: "-b ${{ inputs.branch_name }}"
         EVENT_PARAM: "-e ${{ inputs.event_name }}"
+        PRODUCT_NAME_PARAM: "-n ${{ inputs.product_name }}"
         TO_RESTORE: "-r ${{ inputs.to_restore }}"
         DEFAULT_TARGET_DIR: "${{ env.GITHUB_WORKSPACE }}/${{ inputs.storage_dir || inputs.platform }}"
         MERGE_QUEUE_BASE_REF: "${{ github.event.merge_group.base_ref }}"

.github/actions/restore_artifacts/action.yml

@@ -50,13 +50,14 @@ def main():
 
     if args.commit_sha == 'latest_available_commit':
         latest_artifacts_link = artifact_utils.get_latest_artifacts_link(storage_dir, args.storage_root,
-                                                                         args.branch_name, args.event_name)
+                                                                         args.branch_name, args.event_name,
+                                                                         args.product_name)
         latest_artifacts_path = PureWindowsPath(latest_artifacts_link.read_text())
         normalized_path = latest_artifacts_path.as_posix() if os.name == 'posix' else latest_artifacts_path
         storage = Path(args.storage_root) / normalized_path
     else:
         storage = artifact_utils.get_storage_dir(storage_dir, args.commit_sha, args.storage_root, args.branch_name,
-                                                 args.event_name)
+                                                 args.event_name, args.product_name)
 
     action_utils.set_github_output("artifacts_storage_path", str(storage))
     logger.info(f"Artifacts are taken from here: {storage}")

.github/actions/restore_artifacts/restore_artifacts.py

@@ -19,6 +19,9 @@ inputs:
   branch_name:
     description: "Target branch name. If not set - taken from github context"
     required: false
+  product_name:
+    description: "Product name for which artifacts are generated"
+    required: false
   trigger_repo_sha:
     description: "Commit hash of a trigger repo. If not set - taken from github context"
     required: false
@@ -54,10 +57,12 @@ runs:
         ${{ inputs.storage_dir && env.STORAGE_PARAM || env.PLATFORM_PARAM }}
         ${{ inputs.branch_name && env.BRANCH_PARAM || '' }}
         ${{ inputs.event_name && env.EVENT_PARAM || '' }}
+        ${{ inputs.product_name && env.PRODUCT_NAME_PARAM || '' }}
       env:
         ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}
         BRANCH_PARAM: "-b ${{ inputs.branch_name }}"
         EVENT_PARAM: "-e ${{ inputs.event_name }}"
         STORAGE_PARAM: "--storage_dir ${{ inputs.storage_dir }}"
         PLATFORM_PARAM: "--platform ${{ inputs.platform }}"
+        PRODUCT_NAME_PARAM: "-n ${{ inputs.product_name }}"
         MERGE_QUEUE_BASE_REF: ""

.github/actions/store_artifacts/action.yml

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import logging
 import os
 import sys
@@ -62,6 +63,33 @@ def rotate_dir(directory: Path) -> bool:
     return True
 
 
+def generate_sha256sum(file_path: str | Path) -> str:
+    """
+    Generates the SHA-256 checksum for the given file.
+
+    :param file_path: Path to the file
+    :return: SHA-256 checksum as a hexadecimal string
+    """
+    sha256 = hashlib.sha256()
+    with open(file_path, 'rb') as file:
+        for chunk in iter(lambda: file.read(4096), b''):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
+def store_checksums(artifact_path: Path) -> None:
+    """
+    Generates SHA-256 checksums for a given artifact and stores them in separate files.
+
+    :param artifact_path: Path to either a single artifact file or the directory with files to generate checksums for
+    """
+    files = [path for path in artifact_path.rglob('*') if path.is_file] if artifact_path.is_dir() else [artifact_path]
+    for file in files:
+        hashsum_filepath = file.with_suffix('.sha256')
+        with open(hashsum_filepath, 'w') as hashsum_file:
+            hashsum_file.write(generate_sha256sum(file))
+
+
 def main():
     action_utils.init_logger()
     logger = logging.getLogger(__name__)
@@ -70,7 +98,7 @@ def main():
     storage_root = args.storage_root or os.getenv('ARTIFACTS_SHARE')
     storage_dir = args.storage_dir or PlatformMapping[PlatformKey[args.platform.upper()]].value
     storage = artifact_utils.get_storage_dir(storage_dir, args.commit_sha, args.storage_root, args.branch_name,
-                                             args.event_name)
+                                             args.event_name, args.product_name)
     action_utils.set_github_output("artifacts_storage_path", str(storage))
 
     logger.info(f"Storing artifacts to {storage}")
@@ -79,6 +107,14 @@ def main():
     error_found = False
     for artifact in args.artifacts.split():
         artifact_path = Path(artifact)
+
+        logger.debug(f"Calculating checksum for {artifact_path}")
+        try:
+            store_checksums(artifact_path)
+        except Exception as e:
+            logger.error(f'Failed to calculate checksum for {artifact}: {e}')
+            error_found = True
+
         logger.debug(f"Copying {artifact_path} to {storage / artifact_path.name}")
         try:
             with preserve_stats_context():
@@ -87,6 +123,9 @@ def main():
                 else:
                     storage.mkdir(parents=True, exist_ok=True)
                     shutil.copy2(artifact_path, storage / artifact_path.name)
+                    if artifact_path.with_suffix('.sha256').exists():
+                        shutil.copy2(artifact_path.with_suffix('.sha256'),
+                                     storage / artifact_path.with_suffix('.sha256').name)
         except Exception as e:
             logger.error(f'Failed to copy {artifact}: {e}')
             error_found = True
@@ -95,12 +134,16 @@ def main():
     if github_server:  # If running from GHA context
         # TODO: write an exact job link, but it's not trivial to get
         workflow_link = f"{github_server}/{os.getenv('GITHUB_REPOSITORY')}/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
-        with open(storage / 'workflow_link.txt', 'w') as file:
+        workflow_link_file = storage / 'workflow_link.txt'
+        with open(workflow_link_file, 'w') as file:
             file.write(workflow_link)
+        store_checksums(workflow_link_file)
 
-    if not error_found:
+    # TODO: remove 'openvinotoolkit/openvino' condition when Tokenizers artifacts are stored in a separate structure
+    if not error_found and os.getenv('GITHUB_REPOSITORY') == 'openvinotoolkit/openvino':
         latest_artifacts_for_branch = artifact_utils.get_latest_artifacts_link(storage_dir, args.storage_root,
-                                                                               args.branch_name, args.event_name)
+                                                                               args.branch_name, args.event_name,
+                                                                               args.product_name)
         # Overwrite path to "latest" built artifacts only if a given commit is the head of a given branch
         if args.event_name != 'pull_request' and args.commit_sha == os.getenv('GITHUB_SHA'):
             # TODO: lock to avoid corruption in case of a parallel build (unlikely event for now, but still)

.github/actions/store_artifacts/store_artifacts.py

@@ -15,14 +15,11 @@ updates:
       timezone: "Poland"
     open-pull-requests-limit: 3
     assignees:
-      - "jiwaszki"
       - "p-wysocki"
       - "akuporos"
       - "rkazants"
       - "ceciliapeng2011"
       - "meiyang-intel"
-      - "mbencer"
-      - "tomdol"
       - "jane-intel"
     versioning-strategy: increase-if-necessary
 

.github/dependabot.yml

@@ -1 +1 @@
-pr-28725
+pr-28972

.github/dockerfiles/docker_tag

@@ -32,6 +32,13 @@ RUN apt-get update && \
         libhdf5-dev \
         # For TF Models tests
         wget \
+        # libGL for PyTorch Models tests
+        ffmpeg  \
+        libsm6  \
+        libxext6 \
+        libgl1 \
+        libgl1-mesa-glx \
+        libglib2.0-0 \
         && \
     rm -rf /var/lib/apt/lists/*
 

.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile

@@ -30,7 +30,7 @@ jobs:
           packages: graphviz texlive liblua5.2-0 libclang1-9 libclang-cpp9
           version: 3.0
 
-      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         id: cp310
         with:
           python-version: '3.10'
@@ -64,7 +64,7 @@ jobs:
 
       - name: Cache documentation
         id: cache_sphinx_docs
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache

.github/workflows/build_doc.yml

@@ -10,7 +10,7 @@ permissions: read-all
 jobs:
   Cleanup_PIP:
     name: Cleanup PIP cache
-    runs-on: aks-linux-2-cores-8gb
+    runs-on: aks-linux-small
     if: ${{ github.repository_owner == 'openvinotoolkit' }}
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
@@ -35,9 +35,36 @@ jobs:
           echo "Cache info: "
           du -h -d2 ${PIP_CACHE_PATH}
 
+  Cleanup_HF_Cache:
+    name: Cleanup HuggingFace cache
+    runs-on: aks-linux-4-cores-16gb
+    if: ${{ github.repository_owner == 'openvinotoolkit' }}
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      volumes:
+        - /mount:/mount
+    env:
+      HF_CACHE_PATH: /mount/caches/huggingface
+
+    steps:
+      - name: Checkout cache action
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        timeout-minutes: 15
+        with:
+          sparse-checkout: .github/actions/cache
+
+      - name: Cleanup HF cache
+        uses: ./.github/actions/cache/cleanup
+        with:
+          cache-size: 1700
+          max-cache-size: 2000
+          cache-path: ${{ env.HF_CACHE_PATH }}
+          recursive: true
+          key: '.'
+
   Cleanup_ccache_lin:
     name: Cleanup Linux ccache
-    runs-on: aks-linux-2-cores-8gb
+    runs-on: aks-linux-small
     if: ${{ github.repository_owner == 'openvinotoolkit' }}
     container:
       image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
@@ -47,7 +74,7 @@ jobs:
       CCACHE_PATH: /mount/caches/ccache
 
     steps:
-      - name: Checkout cach action
+      - name: Checkout cache action
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         timeout-minutes: 15
         with:
@@ -70,7 +97,7 @@ jobs:
       CCACHE_PATH: C:\\mount\\caches\\ccache
 
     steps:
-      - name: Checkout cach action
+      - name: Checkout cache action
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         timeout-minutes: 15
         with:

.github/workflows/cleanup_caches.yml

@@ -20,14 +20,14 @@ jobs:
 
     steps:
       - name: Setup python
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.10.10'
           architecture: 'x64'
 
 
       - name: Setup ccache
-        uses: hendrikmuhs/ccache-action@53911442209d5c18de8a31615e0923161e435875 # v1.2.16
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
         with:
           max-size: 50G
 
@@ -63,7 +63,6 @@ jobs:
             -DCMAKE_VERBOSE_MAKEFILE=ON
             -DENABLE_PYTHON=ON
             -DENABLE_ONEDNN_FOR_GPU=ON
-            -DBUILD_SHARED_LIBS=ON
             -DENABLE_TESTS=ON
             -DENABLE_OV_ONNX_FRONTEND=ON
             -DENABLE_FASTER_BUILD=ON
@@ -130,6 +129,6 @@ jobs:
           lcov --capture --directory ${{ github.workspace }}/. --output-file coverage.info
           genhtml coverage.info --output-directory coverage-report
       - name: Collect coverage
-        uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1
+        uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0
         with:
           verbose: true

.github/workflows/coverage.yml

@@ -122,6 +122,7 @@ jobs:
       runner: 'aks-linux-16-cores-arm'
       image: ${{ fromJSON(needs.docker.outputs.images).ov_test.debian_10_arm }}
       python-version: '3.7'
+      scope: ${{ github.event_name == 'workflow_dispatch' && 'nightly' || 'smoke' }}
 
   Overall_Status:
     name: ci/gha_overall_status_debian_10_arm