Skip to content

Commit 52f6dd6

Browse files
authored
Improve remote fetch (sonic-net#12795)
### Why I did it To fix those errors: One: ``` Connecting to urm.nvidia.com (urm.nvidia.com)|*.*.*.*|:443... connected. GnuTLS: Error in the pull function. Unable to establish SSL connection. Error 4 make[1]: Leaving directory '/sonic/src/smartmontools' [ target/debs/bullseye/smartmontools_6.6-1_amd64.deb ] ``` Second: ``` Get:90 https://debian-mirror-url buster/main amd64 librrd-dev amd64 1.7.1-2 [284 kB] Get:91 https://debian-mirror-url buster/main amd64 psmisc amd64 23.2-1+deb10u1 [126 kB] Get:92 https://debian-mirror-url buster/main amd64 python-smbus amd64 4.1-1 [12.2 kB] Get:93 https://debian-mirror-url buster/main amd64 python3.7-dev amd64 3.7.3-2+deb10u3 [510 kB] Get:94 https://debian-mirror-url buster/main amd64 python3-dev amd64 3.7.3-1 [1264 B] Get:95 https://debian-mirror-url buster/main amd64 python3-smbus amd64 4.1-1 [12.5 kB] Get:96 https://debian-mirror-url buster/main amd64 rrdtool amd64 1.7.1-2 [485 kB] Fetched 122 MB in 12s (9976 kB/s) �[91mE: Failed to fetch https://debian-mirror-url/pool/main/p/python-defaults/python2-minimal_2.7.16-1_amd64.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Failed to fetch https://debian-mirror-url/pool/main/f/fontconfig/fontconfig-config_2.13.1-2_all.deb 500 Internal Server Error [IP: *.*.*.* 443] E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing? �[0mThe command '/bin/sh -c apt-get update && apt-get install -y build-essential python3-dev ipmitool librrd8 librrd-dev rrdtool python-smbus python3-smbus dmidecode i2c-tools psmisc libpci3' returned a non-zero code: 100 [ target/docker-platform-monitor.gz ] Error 1 ``` #### How I did it Add retry mechanism to apt, wget, and curl hooks
1 parent 5aea3a9 commit 52f6dd6

File tree

15 files changed

+57
-9
lines changed

15 files changed

+57
-9
lines changed

.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,6 @@ htmlcov/
105105
# Debian mirror Sources
106106
sources.list.*
107107
!sources.list*.j2
108+
109+
# Generated mirror configs
110+
apt-retries-count

build_debian.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
113113
## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
114114
scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
115115
sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
116-
sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until,apt-multiple-retries} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
116+
sudo cp files/apt/apt-retries-count $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
117+
sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/
117118

118119
## Note: set lang to prevent locale warnings in your chroot
119120
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y update

dockers/docker-base-buster/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
3030
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3131
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3232
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
33+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
3334

3435
# Update apt cache and
3536
# pre-install fundamental packages

dockers/docker-base-stretch/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
3030
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
3131
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
3232
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
33+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
3334

3435
# Update apt cache and
3536
# pre-install fundamental packages

dockers/docker-base/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
3838
COPY ["sources.list", "/etc/apt/sources.list"]
3939
{% endif %}
4040
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
41+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
4142
RUN apt-get update
4243

4344
# Pre-install fundamental packages

files/apt/apt-retries-count.j2

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Instruct apt to retry downloads on failures
2+
Acquire::Retries "{{ APT_RETRIES_COUNT }}";

scripts/build_mirror_config.sh

+10
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ CONFIG_PATH=$1
55
export ARCHITECTURE=$2
66
export DISTRIBUTION=$3
77

8+
# Handling default
9+
[[ -z $APT_RETRIES_COUNT ]] && APT_RETRIES_COUNT=20
10+
export APT_RETRIES_COUNT
11+
812
DEFAULT_MIRROR_URL_PREFIX=http://packages.trafficmanager.net
913
MIRROR_VERSION_FILE=
1014
[[ "$SONIC_VERSION_CONTROL_COMPONENTS" == *deb* || $SONIC_VERSION_CONTROL_COMPONENTS == *all* ]] && MIRROR_VERSION_FILE=files/build/versions/default/versions-mirror
@@ -40,6 +44,7 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then
4044
fi
4145
fi
4246

47+
# Handle sources list
4348
[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
4449
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS
4550

@@ -53,3 +58,8 @@ if [ "$MIRROR_SNAPSHOT" == y ]; then
5358
# Set the snapshot mirror, and add the SET_REPR_MIRRORS flag
5459
sed -i -e "/^#*deb.*packages.trafficmanager.net/! s/^#*deb/#&/" -e "\$a#SET_REPR_MIRRORS" $CONFIG_PATH/sources.list.$ARCHITECTURE
5560
fi
61+
62+
# Handle apt retry count config
63+
APT_RETRIES_COUNT_FILENAME=apt-retries-count
64+
TEMPLATE=files/apt/$APT_RETRIES_COUNT_FILENAME.j2
65+
j2 $TEMPLATE > $CONFIG_PATH/$APT_RETRIES_COUNT_FILENAME

sonic-slave-bullseye/Dockerfile.j2

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ FROM {{ prefix }}debian:bullseye
2222
MAINTAINER gulv@microsoft.com
2323

2424
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
25-
COPY ["apt-multiple-retries", "/etc/apt/apt.conf.d/"]
25+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
2626

2727
{%- if CROSS_BUILD_ENVIRON != "y" %}
2828
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]

sonic-slave-bullseye/apt-multiple-retries

-4
This file was deleted.

sonic-slave-buster/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ FROM {{ prefix }}debian:buster
2121
MAINTAINER gulv@microsoft.com
2222

2323
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
24+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
2425

2526
{%- if CROSS_BUILD_ENVIRON != "y" %}
2627
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]

sonic-slave-jessie/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ FROM {{ prefix }}debian:jessie
1010
MAINTAINER johnar@microsoft.com
1111

1212
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
13+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
1314

1415
## Remove retired jessie-updates repo
1516
RUN sed -i '/http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list

sonic-slave-stretch/Dockerfile.j2

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ FROM {{ prefix }}debian:stretch
1010
MAINTAINER gulv@microsoft.com
1111

1212
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]
13+
COPY ["apt-retries-count", "/etc/apt/apt.conf.d"]
1314
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
1415

1516
## Make apt-get non-interactive

src/sonic-build-hooks/hooks/curl

+13
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,19 @@
33
. /usr/local/share/buildinfo/scripts/buildinfo_base.sh
44
[ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/curl
55

6+
# Retry if something super-weird has happened. Use --retry-connrefused and
7+
# --retry options for curl.
8+
# --retry-connrefused - Consider "connection refused" a transient error and try
9+
# again. Normally wget/curl gives up on a URL when it is unable to connect to
10+
# the site because failure to connect is taken as a sign that the server is not
11+
# running at all and that retries would not help. This option is for mirroring
12+
# unreliable sites whose servers tend to disappear for short periods of time.
13+
# --retry - If a transient error is returned when curl tries to perform a
14+
# transfer, it will retry this number of times before giving up. Transient error
15+
# means either: a timeout, an FTP 4xx response code or an HTTP 408, 429, 500,
16+
# 502, 503 or 504 response code.
17+
REAL_COMMAND="$REAL_COMMAND --retry-connrefused --retry 5"
18+
619
if [ "$SKIP_BUILD_HOOK" == y ]; then
720
$REAL_COMMAND "$@"
821
exit $?

src/sonic-build-hooks/hooks/wget

+9
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@ if [ -z "$REAL_COMMAND" ]; then
77
exit 1
88
fi
99

10+
# Retry if something super-weird has happened. Use --retry-connrefused option
11+
# for wget.
12+
# --retry-connrefused - Consider "connection refused" a transient error and try
13+
# again. Normally wget/curl gives up on a URL when it is unable to connect to
14+
# the site because failure to connect is taken as a sign that the server is not
15+
# running at all and that retries would not help. This option is for mirroring
16+
# unreliable sites whose servers tend to disappear for short periods of time.
17+
REAL_COMMAND="$REAL_COMMAND --retry-connrefused"
18+
1019
if [ "$SKIP_BUILD_HOOK" == y ]; then
1120
$REAL_COMMAND "$@"
1221
exit $?

src/sonic-build-hooks/scripts/buildinfo_base.sh

+11-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ WEB_VERSION_FILE=$VERSION_PATH/versions-web
1313
BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web
1414
REPR_MIRROR_URL_PATTERN='http:\/\/packages.trafficmanager.net\/'
1515
DPKG_INSTALLTION_LOCK_FILE=/tmp/.dpkg_installation.lock
16+
GET_RETRY_COUNT=5
1617

1718
. $BUILDINFO_PATH/config/buildinfo.config
1819

@@ -242,10 +243,17 @@ download_packages()
242243
return $result
243244
fi
244245

245-
$REAL_COMMAND "${parameters[@]}"
246-
result=$?
246+
# Retry if something super-weird has happened
247+
for ((i = 1; i <= GET_RETRY_COUNT; i++)); do
248+
$REAL_COMMAND "${parameters[@]}"
249+
result=$?
250+
if [ $result -eq 0 ]; then
251+
break
252+
fi
253+
log_err "Try $i: $REAL_COMMAND failed to get: ${parameters[@]}. Retry.."
254+
done
247255

248-
#Return if there is any error
256+
# Return if there is any error
249257
if [ ${result} -ne 0 ]; then
250258
exit ${result}
251259
fi

0 commit comments

Comments
 (0)