upload-to-squad.sh: add retry to help success rate for uploading (#543)

when the uploading fails.

Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

Author: liuyq

automated/utils/upload-to-squad.sh

@@ -4,24 +4,30 @@ ATTACHMENT=""
 ARTIFACTORIAL_URL=""
 CURL_VERBOSE_FLAG=""
 FAILURE_RETURN_VALUE=0
+RETRY_COUNT=5
+RETRY_INTERVAL=30
 
 usage() {
-    echo "Usage: $0 [-a <attachment>] [-u <artifactorial_url>] [-v] [-r]" 1>&2
+    echo "Usage: $0 [-a <attachment>] [-u <artifactorial_url>] [-c <retry_count>] [-i <retry_interval>] [-v] [-r]" 1>&2
     echo "  -a attachment           Path to the file to upload" 1>&2
     echo "  -u squad_url            SQUAD_URL where the attachment will be uploaded to" 1>&2
     echo "                          This script will try to fetch the SQUAD_ARCHIVE_SUBMIT_TOKEN" 1>&2
     echo "                          token from (lava_test_dir)/secrets or environments for the upload." 1>&2
+    echo "  -c retry_count          How many times to try when the uploading failed" 1>&2
+    echo "  -i retry_interval       The interval seconds between the re-tries." 1>&2
     echo "  -v      Pass -v (verbose) flag to curl for debugging." 1>&2
     echo "  -r      Report failure. If the upload fails and this flag is set, the script will exit" 1>&2
     echo "          with return value 1. If the upload is skipped (no URL or no token found)," 1>&2
     echo "          this script will still return 0." 1>&2
     exit 1
 }
 
-while getopts ":a:u:vr" opt; do
+while getopts ":a:u:c:i:vr" opt; do
     case "${opt}" in
         a) ATTACHMENT="${OPTARG}" ;;
         u) ARTIFACTORIAL_URL="${OPTARG}" ;;
+        c) RETRY_COUNT="${OPTARG}" ;;
+        i) RETRY_INTERVAL="${OPTARG}" ;;
         v) CURL_VERBOSE_FLAG="-v" ;;
         r) FAILURE_RETURN_VALUE=1 ;;
         *) usage ;;
@@ -50,24 +56,39 @@ if command -v lava-test-reference > /dev/null 2>&1; then
         echo "test-attachment skip"
         command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "skip"
         exit 0
-    else
-        # return is the squad testrun id
-        return=$(curl ${CURL_VERBOSE_FLAG} --header "Auth-Token: ${SQUAD_ARCHIVE_SUBMIT_TOKEN}" --form "attachment=@${ATTACHMENT}" "${ARTIFACTORIAL_URL}")
     fi
 
     attachmentBasename="$(basename "${ATTACHMENT}")"
-    if echo "${return}" | grep -E "^[0-9]+$"; then
-        # ARTIFACTORIAL_URL will be in the format like this:
-        #    https://qa-reports.linaro.org/api/submit/squad_group/squad_project/squad_build/environment
-        url_squad=$(echo "${ARTIFACTORIAL_URL}"|sed 's|/api/submit/.*||')
-        url_uploaded="${url_squad}/api/testruns/${return}/attachments/?filename=${attachmentBasename}"
-        lava-test-reference "test-attachment" --result "pass" --reference "${url_uploaded}"
-    else
-        echo "test-attachment fail"
-        echo "Expected one SQUAD testrun id returend, but curl returned \"${return}\"."
-        command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "fail"
-        exit "${FAILURE_RETURN_VALUE}"
-    fi
+    # Re-run the upload for ${RETRY_COUNT} times with the interval of ${RETRY_INTERVAL} seconds when it fails
+    i=1
+    while [ $i -le "${RETRY_COUNT}" ]; do
+        # response is the squad testrun id when succeed
+        response=$(curl ${CURL_VERBOSE_FLAG} --header "Auth-Token: ${SQUAD_ARCHIVE_SUBMIT_TOKEN}" --form "attachment=@${ATTACHMENT}" "${ARTIFACTORIAL_URL}")
+
+        # generate the SQUAD url for download and report pass when uploading succeed
+        if echo "${response}" | grep -E "^[0-9]+$"; then
+            # ARTIFACTORIAL_URL will be in the format like this:
+            #    https://qa-reports.linaro.org/api/submit/squad_group/squad_project/squad_build/environment
+            url_squad=$(echo "${ARTIFACTORIAL_URL}"|sed 's|/api/submit/.*||')
+            url_uploaded="${url_squad}/api/testruns/${response}/attachments/?filename=${attachmentBasename}"
+            lava-test-reference "test-attachment" --result "pass" --reference "${url_uploaded}"
+            break
+        fi
+
+        # still print the output every time for investigation purpose
+        echo "Expected one SQUAD testrun id returend, but curl returned \"${response}\"."
+
+        # report fail if the uploading failed for ${RETRY_COUNT} times
+        if [ $i -eq "${RETRY_COUNT}" ]; then
+            echo "test-attachment fail"
+            command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "fail"
+            exit "${FAILURE_RETURN_VALUE}"
+        fi
+
+        # try again in ${RETRY_INTERVAL} seconds
+        sleep "${RETRY_INTERVAL}"
+        i=$((i + 1))
+    done
 else
     echo "test-attachment skip"
     command -v lava-test-case > /dev/null 2>&1 && lava-test-case "test-attachment" --result "skip"