Skip to content

Commit d31e84a

Browse files
authored
[build] enable pigz by default (sonic-net#16959)
Why I did it pigz, which stands for parallel implementation of gzip, is a fully functional replacement for gzip that exploits multiple processors and multiple cores to the hilt when compressing data. We can use it for compression instead of gzip to speed up builds. It reduces build time (especially at final stage where we compress whole file system). But solution with special option GZ_COMPRESS_PROGRAM is not good, because it's hard to maintain (developers need to pass this flag to every script). I think it's better to remove this option at all and use pigz directly where it's possible. How I did it Remove config flag GZ_COMPRESS_PROGRAM Use pigz directly instead of gzip where it's possible. How to verify it I have tested builds with pigz almost 1 year and no issues were detected. So I think it's better to enable it by default for all builds. If we found some issues we can always replace pigz with gzip.
1 parent 3b3fa20 commit d31e84a

File tree

9 files changed

+14
-27
lines changed

9 files changed

+14
-27
lines changed

Makefile.work

-9
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,6 @@
5454
# * ENABLE_BOOTCHART: Enable SONiC bootchart
5555
# * Default: n
5656
# * Values: y,n
57-
# * GZ_COMPRESS_PROGRAM: Select pigz to reduce build time
58-
# * Default: gzip
59-
# * Values: pigz,gzip
6057
# * UNATTENDED: Don't wait for interactive input from terminal, setting this
6158
# * value to anything will enable it
6259
# * Default: unset
@@ -157,10 +154,6 @@ ifeq ($(LEGACY_SONIC_MGMT_DOCKER),)
157154
override LEGACY_SONIC_MGMT_DOCKER = y
158155
endif
159156

160-
ifneq ($(GZ_COMPRESS_PROGRAM), pigz)
161-
override GZ_COMPRESS_PROGRAM = gzip
162-
endif
163-
164157
ifeq ($(CONFIGURED_ARCH),amd64)
165158
SLAVE_BASE_IMAGE = $(SLAVE_DIR)
166159
MULTIARCH_QEMU_ENVIRON = n
@@ -229,7 +222,6 @@ $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
229222
INCLUDE_FIPS=$(INCLUDE_FIPS) \
230223
DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) \
231224
DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
232-
GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
233225
j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
234226

235227
$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) \
@@ -587,7 +579,6 @@ SONIC_BUILD_INSTRUCTION := $(MAKE) \
587579
SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \
588580
MIRROR_URLS=$(MIRROR_URLS) \
589581
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
590-
GZ_COMPRESS_PROGRAM=$(GZ_COMPRESS_PROGRAM) \
591582
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
592583
SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
593584
ONIE_IMAGE_PART_SIZE=$(ONIE_IMAGE_PART_SIZE) \

build_debian.sh

+3-5
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,7 @@ else
150150
fi
151151

152152
## docker and mkinitramfs on target system will use pigz/unpigz automatically
153-
if [[ $GZ_COMPRESS_PROGRAM == pigz ]]; then
154-
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install pigz
155-
fi
153+
sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install pigz
156154

157155
## Install initramfs-tools and linux kernel
158156
## Note: initramfs-tools recommends depending on busybox, and we really want busybox for
@@ -879,8 +877,8 @@ if [[ $MULTIARCH_QEMU_ENVIRON == y || $CROSS_BUILD_ENVIRON == y ]]; then
879877
fi
880878

881879
## Compress docker files
882-
pushd $FILESYSTEM_ROOT && sudo tar -I $GZ_COMPRESS_PROGRAM -cf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd
880+
pushd $FILESYSTEM_ROOT && sudo tar -I pigz -cf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd
883881

884882
## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file
885-
pushd $FILESYSTEM_ROOT && sudo tar -I $GZ_COMPRESS_PROGRAM -cf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd
883+
pushd $FILESYSTEM_ROOT && sudo tar -I pigz -cf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd
886884
sudo zip -g -n .squashfs:.gz $INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS

build_image.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ generate_kvm_image()
6060
exit 1
6161
}
6262

63-
$GZ_COMPRESS_PROGRAM $KVM_IMAGE_DISK
63+
pigz $KVM_IMAGE_DISK
6464

6565
[ -r $KVM_IMAGE_DISK.gz ] || {
66-
echo "Error : $GZ_COMPRESS_PROGRAM $KVM_IMAGE_DISK failed!"
66+
echo "Error : pigz $KVM_IMAGE_DISK failed!"
6767
exit 1
6868
}
6969

rules/config

-4
Original file line numberDiff line numberDiff line change
@@ -307,10 +307,6 @@ ENABLE_FIPS ?= n
307307
# SONIC_SLAVE_DOCKER_DRIVER - set the sonic slave docker storage driver
308308
SONIC_SLAVE_DOCKER_DRIVER ?= vfs
309309

310-
# GZ_COMPRESS_PROGRAM - select pigz (a parallel implementation of gzip) to reduce a build time
311-
# and speed up a decompression of docker images on target system
312-
GZ_COMPRESS_PROGRAM ?= gzip
313-
314310
# SONIC_OS_VERSION - sonic os version
315311
SONIC_OS_VERSION ?= 12
316312

slave.mk

+1-3
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ export DOCKER_BASE_ARCH
9292
export CROSS_BUILD_ENVIRON
9393
export BLDENV
9494
export BUILD_WORKDIR
95-
export GZ_COMPRESS_PROGRAM
9695
export MIRROR_SNAPSHOT
9796
export SONIC_OS_VERSION
9897
export FILES_PATH
@@ -461,7 +460,6 @@ ifeq ($(CONFIGURED_PLATFORM),vs)
461460
$(info "BUILD_MULTIASIC_KVM" : "$(BUILD_MULTIASIC_KVM)")
462461
endif
463462
$(info "CROSS_BUILD_ENVIRON" : "$(CROSS_BUILD_ENVIRON)")
464-
$(info "GZ_COMPRESS_PROGRAM" : "$(GZ_COMPRESS_PROGRAM)")
465463
$(info "LEGACY_SONIC_MGMT_DOCKER" : "$(LEGACY_SONIC_MGMT_DOCKER)")
466464
$(info "INCLUDE_EXTERNAL_PATCHES" : "$(INCLUDE_EXTERNAL_PATCHES)")
467465
$(info "PTF_ENV_PY_VER" : "$(PTF_ENV_PY_VER)")
@@ -553,7 +551,7 @@ define docker-image-save
553551
@echo "Tagging docker image $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) as $(1):$(call docker-get-tag,$(1))" $(LOG)
554552
docker tag $(1)-$(DOCKER_USERNAME):$(DOCKER_USERTAG) $(1):$(call docker-get-tag,$(1)) $(LOG)
555553
@echo "Saving docker image $(1):$(call docker-get-tag,$(1))" $(LOG)
556-
docker save $(1):$(call docker-get-tag,$(1)) | $(GZ_COMPRESS_PROGRAM) -c > $(2)
554+
docker save $(1):$(call docker-get-tag,$(1)) | pigz -c > $(2)
557555
if [ x$(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD) == x"y" ]; then
558556
@echo "Removing docker image $(1):$(call docker-get-tag,$(1))" $(LOG)
559557
docker rmi -f $(1):$(call docker-get-tag,$(1)) $(LOG)

sonic-slave-bookworm/Dockerfile.j2

+2-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ RUN apt-get update && apt-get install -y \
8888
curl \
8989
wget \
9090
unzip \
91-
{{ GZ_COMPRESS_PROGRAM }} \
91+
gzip \
92+
pigz \
9293
git \
9394
build-essential \
9495
remake \

sonic-slave-bullseye/Dockerfile.j2

+2-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ RUN apt-get update && apt-get install -y eatmydata && eatmydata apt-get install
9494
curl \
9595
wget \
9696
unzip \
97-
{{ GZ_COMPRESS_PROGRAM }} \
97+
gzip \
98+
pigz \
9899
git \
99100
build-essential \
100101
remake \

sonic-slave-buster/Dockerfile.j2

+2-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ RUN apt-get update && apt-get install -y eatmydata && eatmydata apt-get install
9090
curl \
9191
wget \
9292
unzip \
93-
{{ GZ_COMPRESS_PROGRAM }} \
93+
gzip \
94+
pigz \
9495
git \
9596
build-essential \
9697
remake \

sonic-slave-stretch/Dockerfile.j2

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ RUN apt-get update && apt-get install -y \
2323
curl \
2424
wget \
2525
unzip \
26-
{{ GZ_COMPRESS_PROGRAM }} \
26+
gzip \
27+
pigz \
2728
git \
2829
build-essential \
2930
libtool \

0 commit comments

Comments
 (0)