[docker-sonic-mgmt] Add dash-pipeline-utils

dockers/docker-sonic-mgmt/Dockerfile.j2

@@ -209,6 +209,8 @@ RUN mkdir -p /etc/apt/keyrings \
     && echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ `lsb_release -cs` main" | tee /etc/apt/sources.list.d/azure-cli.list \
     && apt-get update && apt-get install -y azure-cli
 
+# Remove old python protobuf package if existing
+RUN python3 -m pip show protobuf && python3 -m pip uninstall -y protobuf
 # Install protobuf 3.21.12 which is from https://deb.debian.org/debian/pool/main/p/protobuf/protobuf_3.21.12-3.dsc
 RUN mkdir -p /tmp/protobuf \
     && cd /tmp/protobuf \
@@ -227,6 +229,16 @@ COPY debs/sonic-device-data_*.deb debs/libdashapi_*.deb debs/
 
 RUN dpkg -i debs/sonic-device-data_*.deb debs/libdashapi_*.deb
 
+# Install dash-pipeline-utils
+COPY ["files/github_get.py", "/usr/bin"]
+RUN cd /tmp \
+    && python3 /usr/bin/github_get.py https://api.github.com/repos/sonic-net/DASH/contents/dash-pipeline/utils \
+    && cd utils \
+    && python3 setup.py bdist_wheel \
+    && python3 -m pip install dist/dash_pipeline_utils*.whl \
+    && cd / \
+    && rm -rf /tmp/utils
+
 RUN mkdir /var/run/sshd
 EXPOSE 22
 

rules/docker-sonic-mgmt.mk

@@ -2,5 +2,6 @@
 DOCKER_SONIC_MGMT = docker-sonic-mgmt.gz
 $(DOCKER_SONIC_MGMT)_PATH = $(DOCKERS_PATH)/docker-sonic-mgmt
 $(DOCKER_SONIC_MGMT)_DEPENDS += $(SONIC_DEVICE_DATA) $(PTF) $(LIB_SONIC_DASH_API)
+$(DOCKER_SONIC_MGMT)_FILES += $(GITHUB_GET)
 SONIC_DOCKER_IMAGES += $(DOCKER_SONIC_MGMT)
 SONIC_BULLSEYE_DOCKERS += $(DOCKER_SONIC_MGMT)

rules/scripts.mk

@@ -35,6 +35,9 @@ $(COPP_CONFIG_TEMPLATE)_PATH = files/image_config/copp
 RSYSLOG_PLUGIN_CONF_J2 = rsyslog_plugin.conf.j2
 $(RSYSLOG_PLUGIN_CONF_J2)_PATH = files/build_templates
 
+GITHUB_GET = github_get.py
+$(GITHUB_GET)_PATH = scripts
+
 SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
                     $(ARP_UPDATE_SCRIPT) \
                     $(ARP_UPDATE_VARS_TEMPLATE) \
@@ -46,4 +49,5 @@ SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
                     $(UPDATE_CHASSISDB_CONFIG_SCRIPT) \
                     $(SWSS_VARS_TEMPLATE) \
                     $(RSYSLOG_PLUGIN_CONF_J2) \
+                    $(GITHUB_GET) \
                     $(COPP_CONFIG_TEMPLATE)

scripts/github_get.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import sys
+import re
+import json
+import requests
+
+def get(url, file_name=None, dir_name=None):
+    '''
+    Get the files in the given URL
+    :param url: The URL is a github API URL that returns a JSON object
+    :param file_name: if specified, url will be github download_url
+    :return: None
+    :raises: Exception if the download fails
+    '''
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Request failed: {e}")
+
+    if file_name:
+        with open(file_name, 'wb') as file:
+            file.write(response.content)
+        return
+
+    def get_file(file):
+        if file['download_url']:
+            get(file['download_url'], file_name=file['name'])
+        else:
+            if file['type'] == 'dir':
+                get(file['url'], dir_name=file['name'])
+            else:
+                get(file['git_url'].replace('/git/trees/', '/contents?ref='))
+
+    try:
+        data = response.json()
+    except json.JSONDecodeError as e:
+        raise Exception(f"JSON decode error: {e}")
+
+    if isinstance(data, list):
+        if not dir_name:
+            pattern = r"https://api\.github\.com/repos/[^/]+/([^/]+)/contents(.*)"
+            repo_name, path = re.search(pattern, url).groups()
+            path = path.split('/')[-1].split('?')[0]
+            dir_name = path if path else repo_name
+
+        cur = os.getcwd()
+        os.mkdir(dir_name)
+        os.chdir(dir_name)
+        for item in data:
+            get_file(item)
+        os.chdir(cur)
+    else:
+        get_file(data)
+
+def help():
+    print(f"Usage: {sys.argv[0]} <github REST API url>")
+    print("        url format: https://api.github.com/repos/{owner}/{repo}/contents/{directory}?ref={branch}")
+    print(f"       url example: https://api.github.com/repos/sonic-net/DASH/contents/dash-pipeline/utils?ref=main")
+    sys.exit(1)
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2 or sys.argv[1] == '-h':
+        help()
+
+    try:
+        get(sys.argv[1])
+    except Exception as e:
+        print(e)
+        sys.exit(1)