PMM-12899 extract server rpm script

ademidoff · ademidoff · commit 6d0b02bdcab1 · 2024-04-01T01:19:42.000Z
diff --git a/build/local/build.sh b/build/local/build.sh
@@ -1,7 +1,38 @@
 #!/bin/bash -e
 
-# Update submodules and PR branches
-/bin/bash $(dirname $0)/update.sh
+# Define global variables
+NO_UPDATE=0
+NO_CLIENT=0
+NO_CLIENT_DOCKER=0
+NO_SERVER_RPM=0
+
+while test "$#" -gt 0; do
+  case "$1" in
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [ "$NO_UPDATE" -eq 0 ]; then
+  # Update submodules and PR branches
+  /bin/bash $(dirname $0)/update.sh
+fi
 
 get_branch_name() {
   local module="${1:-}"
@@ -33,7 +64,7 @@ echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
 
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
 # Important: the docker container's user need to be able to write to these directories.
 # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
 # that the directories we create on the host are owned by a user with same uid and gid.
@@ -51,22 +82,26 @@ export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-# Build client source: 4m39s from scratch, 0m27s using cache
-"$PATH_TO_SCRIPTS/build-client-source"
+if [ "$NO_CLIENT" -eq 0 ]; then
+  # Build client source: 4m39s from scratch, 0m27s using cache
+  "$PATH_TO_SCRIPTS/build-client-source"
 
-# Build client binary: ??? from scratch, 0m20s using cache
-"$PATH_TO_SCRIPTS/build-client-binary"
+  # Build client binary: ??? from scratch, 0m20s using cache
+  "$PATH_TO_SCRIPTS/build-client-binary"
 
-# Building client source rpm takes 13s (caching does not apply)
-"$PATH_TO_SCRIPTS/build-client-srpm"
+  # Building client source rpm takes 13s (caching does not apply)
+  "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client rpm takes 1m40s
-"$PATH_TO_SCRIPTS/build-client-rpm"
+  # Building client rpm takes 1m40s
+  "$PATH_TO_SCRIPTS/build-client-rpm"
+fi
 
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-"$PATH_TO_SCRIPTS/build-client-docker"
+if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-client-docker"
+fi
 
 # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
 # total time: 6m26s - build from scratch, no initial cache
@@ -79,20 +114,23 @@ export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
 
 export RPM_EPOCH=1
 export RPMBUILD_DIST="el9"
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-# 3rd-party
-"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+  # 3rd-party
+  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
 export RPMBUILD_DIST=el9
-export DOCKERFILE=Dockerfile.el9.local
+export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
 # Clean up temporary files
diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
@@ -32,15 +32,15 @@ if [ -z "${DOCKER_TAG}" ]; then
     DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
 fi
 
-IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2`
-
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${docker_root}/${docker_file} \
-             -t ${DOCKER_TAG} \
-             ${docker_root}/
+IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+docker buildx build \
+    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg VERSION="$IMAGE_VERSION" \
+    --progress plain \
+    -f ${docker_root}/${docker_file} \
+    -t ${DOCKER_TAG} \
+    ${docker_root}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_tag_file})
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
@@ -24,7 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -36,13 +36,13 @@ is_build_needed() {
 
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
-        cache_dir=PR-BUILDS
+        s3_cache_dir=PR-BUILDS
     fi
     if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -102,7 +102,7 @@ build() {
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
@@ -170,12 +170,12 @@ build() {
     fi
 
     if [ "$RPM_EPOCH" = 1 ]; then
-      cache_dir=PR-BUILDS
+      s3_cache_dir=PR-BUILDS
     fi
     aws s3 sync \
         --region us-east-2 \
         ${rpms_dir}/${spec_name}-${rpm_version} \
-        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
 
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -o errexit
+set -o xtrace
+
+. $(dirname $0)/vars
+
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+
+# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+if [ ${RPM_EPOCH} = 1 ]; then
+    sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+fi
+
+rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+
+printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+    | sudo tee /etc/yum.repos.d/local.repo
+
+until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+    echo waiting
+    sleep 1
+done
+
+# Only these two specs have build dependencies
+if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+    sleep 5s
+    sudo yum-builddep -y SOURCES/${spec_name}.spec
+fi
+
+spectool -C SOURCES -g SOURCES/${spec_name}.spec
+rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+          --define 'dist .${rpmbuild_dist}' \
+          --define 'debug_package %{nil}' \
+          -ba SOURCES/${spec_name}.spec
+
+rm -f SOURCES/${spec_name}.spec*
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+
+# vim: expandtab shiftwidth=4 tabstop=4
