From 17783b3225f1c5d1ab39399de9e4dbf79fd5b343 Mon Sep 17 00:00:00 2001
From: Yongqin Liu <yongqin.liu@linaro.org>
Date: Mon, 26 Feb 2024 15:58:23 +0800
Subject: [PATCH 1/2] upload-to-squad.sh: add support to upload archives to
SQUAD
to replace the uploading to archive.vlo
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>
---
automated/utils/upload-to-squad.sh | 74 ++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
create mode 100755 automated/utils/upload-to-squad.sh
diff --git a/automated/utils/upload-to-squad.sh b/automated/utils/upload-to-squad.sh
new file mode 100755
index 000000000..df9d2fb06
--- /dev/null
+++ b/automated/utils/upload-to-squad.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+
+ATTACHMENT=""
+ARTIFACTORIAL_URL=""
+CURL_VERBOSE_FLAG=""
+FAILURE_RETURN_VALUE=0
+
+usage() {
+ echo "Usage: $0 [-a <attachment>] [-u <artifactorial_url>] [-v] [-r]" 1>&2
+ echo " -a attachment Path to the file to upload" 1>&2
+ echo " -u squad_url SQUAD_URL where the attachment will be uploaded to" 1>&2
+ echo " This script will try to fetch the SQUAD_ARCHIVE_SUBMIT_TOKEN" 1>&2
+ echo " token from (lava_test_dir)/secrets or environments for the upload." 1>&2
+ echo " -v Pass -v (verbose) flag to curl for debugging." 1>&2
+ echo " -r Report failure. If the upload fails and this flag is set, the script will exit" 1>&2
+ echo " with return value 1. If the upload is skipped (no URL or no token found)," 1>&2
+ echo " this script will still return 0." 1>&2
+ exit 1
+}
+
+while getopts ":a:u:vr" opt; do
+ case "${opt}" in
+ a) ATTACHMENT="${OPTARG}" ;;
+ u) ARTIFACTORIAL_URL="${OPTARG}" ;;
+ v) CURL_VERBOSE_FLAG="-v" ;;
+ r) FAILURE_RETURN_VALUE=1 ;;
+ *) usage ;;
+ esac
+done
+
+if [ -z "${ARTIFACTORIAL_URL}" ]; then
+ echo "test-attachment skip"
+ command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "skip"
+ exit 0
+fi
+
+if command -v lava-test-reference > /dev/null 2>&1; then
+ # The 'SQUAD_ARCHIVE_SUBMIT_TOKEN' needs to be defined in 'secrects' dictionary in job
+ # definition file, or defined in the environment, it will be used.
+ # One issue here Milosz pointed out:
+ # If there is lava_test_results_dir set in the job context, "/lava-*" might not be correct.
+ lava_test_dir="$(find /lava-* -maxdepth 0 -type d | grep -E '^/lava-[0-9]+' 2>/dev/null | sort | tail -1)"
+ if test -f "${lava_test_dir}/secrets"; then
+ # shellcheck disable=SC1090
+ . "${lava_test_dir}/secrets"
+ fi
+
+ if [ -z "${SQUAD_ARCHIVE_SUBMIT_TOKEN}" ]; then
+ echo "WARNING: SQUAD_ARCHIVE_SUBMIT_TOKEN is empty! File uploading skipped."
+ echo "test-attachment skip"
+ command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "skip"
+ exit 0
+ else
+ # return is the squad testrun id
+ return=$(curl ${CURL_VERBOSE_FLAG} --header "Auth-Token: ${SQUAD_ARCHIVE_SUBMIT_TOKEN}" --form "attachment=@${ATTACHMENT}" "${ARTIFACTORIAL_URL}")
+ fi
+
+ attachmentBasename="$(basename "${ATTACHMENT}")"
+ if echo "${return}" | grep -E "^[0-9]+$"; then
+ # ARTIFACTORIAL_URL will be in the format like this:
+ # https://qa-reports.linaro.org/api/submit/squad_group/squad_project/squad_build/environment
+ url_squad=$(echo "${ARTIFACTORIAL_URL}"|sed 's|/api/submit/.*||')
+ url_uploaded="${url_squad}/api/testruns/${return}/attachments/?filename=${attachmentBasename}"
+ lava-test-reference "test-attachment" --result "pass" --reference "${url_uploaded}"
+ else
+ echo "test-attachment fail"
+ echo "Expected one SQUAD testrun id returend, but curl returned \"${return}\"."
+ command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "fail"
+ exit "${FAILURE_RETURN_VALUE}"
+ fi
+else
+ echo "test-attachment skip"
+ command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "skip"
+fi
From df99980c1d9d5112b910e2045d984750eabb03f2 Mon Sep 17 00:00:00 2001
From: Yongqin Liu <yongqin.liu@linaro.org>
Date: Mon, 26 Feb 2024 16:01:56 +0800
Subject: [PATCH 2/2] noninteractive-tradefed: add support for uploading
archives to SQUAD
via specifying the SQUAD_UPLOAD_URL variables.
To be noted, to avoid leak of tokens, the SQUAD_ARCHIVE_SUBMIT_TOKEN
used to upload is not supported to be defined in the job definition
in the plain text format, it's must be defined as one profile managed
token by the submitter
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>
---
automated/android/noninteractive-tradefed/tradefed.yaml | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/automated/android/noninteractive-tradefed/tradefed.yaml b/automated/android/noninteractive-tradefed/tradefed.yaml
index a03900976..33487a34c 100644
--- a/automated/android/noninteractive-tradefed/tradefed.yaml
+++ b/automated/android/noninteractive-tradefed/tradefed.yaml
@@ -25,6 +25,11 @@ params:
TEST_PATH: "android-cts"
# Specify result format: aggregated or atomic
RESULTS_FORMAT: "aggregated"
+ # The SQUAD url to be used to upload the result and log files.
+ # see https://squad.readthedocs.io/en/latest/intro.html#submitting-results.
+ # SQUAD_ARCHIVE_SUBMIT_TOKEN is used for uploading authentication,
+ # and must be defined by the submitter as one profile managed token
+ SQUAD_UPLOAD_URL: ""
# Specify url and token for file uploading.
URL: "https://archive.validation.linaro.org/artifacts/team/qa/"
TOKEN: ""
@@ -65,7 +70,8 @@ run:
- sudo dmesg > ./output/dmesg-host.txt || true
- if ! tar caf tradefed-output-$(date +%Y%m%d%H%M%S).tar.xz ./output; then error_fatal "tradefed - failed to collect results and log files [$ANDROID_SERIAL]"; fi
- ATTACHMENT=$(ls tradefed-output-*.tar.xz)
- - ../../utils/upload-to-artifactorial.sh -a "${ATTACHMENT}" -u "${URL}" -t "${TOKEN}"
+ - if [ -n "${SQUAD_UPLOAD_URL}" ]; then ../../utils/upload-to-squad.sh -a "${ATTACHMENT}" -u "${SQUAD_UPLOAD_URL}"; fi
+ - if [ -z "${SQUAD_UPLOAD_URL}" ]; then ../../utils/upload-to-artifactorial.sh -a "${ATTACHMENT}" -u "${URL}" -t "${TOKEN}"; fi
# Send test result to LAVA.
- ../../utils/send-to-lava.sh ./output/result.txt
- userdel testuser -f -r || true