Skip to content

Commit 9310b83

Browse files
ademidoffademidoff
authored
PMM-12641 clean up build scripts (#2728)
* PMM-12641 clean up build scripts * PMM-12641 update build Makefile * PMM-12641 update build Makefile again * PMM-12641 provide comments * PMM-12641 update goreleaser version * PMM-12641 bump up goreleaser version again * PMM-12641 format json * PMM-12641 remove pmm.el9.json * PMM-12641 install percona-release * PMM-12641 print all installed gpg keys * PMM-12641 try to install ppg with yum * PMM-12641 final cleanup * PMM-12641 disable percona-release * PMM-12641 bump up software versions
1 parent 86b90a9 commit 9310b83

File tree

14 files changed

+50
-434
lines changed

14 files changed

+50
-434
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ compose.yml
3535
*.sw[o,p]
3636

3737
.env
38+
.netrc
3839

3940
cli-tests/node_modules/
4041
cli-tests/playwright-report/

‎build/Makefile

+9-35
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,33 @@
11
export PACKER_CACHE_DIR := .cache
2-
export PACKER_VERSION := 1.8.2
3-
export CENTOS_ISO := 2004.01
2+
export PACKER_VERSION := 1.9.4
43

54
## ----------------- PACKER ------------------
65
fetch:
7-
mkdir -p ${PACKER_CACHE_DIR}/${CENTOS_ISO} || :
8-
test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
9-
|| curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
10-
-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
11-
chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
12-
test -f ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova \
13-
|| wget --progress=dot:giga https://app.vagrantup.com/centos/boxes/7/versions/${CENTOS_ISO}/providers/virtualbox.box \
14-
-O ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova
15-
test -f ${PACKER_CACHE_DIR}/${CENTOS_ISO}/box.ovf \
16-
|| tar -C ${PACKER_CACHE_DIR}/${CENTOS_ISO} -xvf ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova
17-
18-
fetch-el9:
196
mkdir -p ${PACKER_CACHE_DIR}/box || :
207
test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
218
|| curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
229
-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
2310
chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
2411
test -f ${PACKER_CACHE_DIR}/box/oracle9.ova \
25-
|| wget --progress=dot:giga https://vagrantcloud.com/bento/boxes/oracle-9.0/versions/202207.20.0/providers/virtualbox.box -O ${PACKER_CACHE_DIR}/box/oracle9.ova
12+
|| curl -fL https://vagrantcloud.com/bento/boxes/oracle-9.0/versions/202207.20.0/providers/virtualbox.box -o ${PACKER_CACHE_DIR}/box/oracle9.ova
2613

2714
# NOTE: image from vagrant registry is twice as large
2815
test -f ${PACKER_CACHE_DIR}/box/box.ovf \
2916
|| tar -C ${PACKER_CACHE_DIR}/box -xvf ${PACKER_CACHE_DIR}/box/oracle9.ova
3017

3118
deps:
3219
mkdir -p ${PACKER_CACHE_DIR} ~/bin || :
33-
curl https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip -o ${PACKER_CACHE_DIR}/packer.zip
20+
curl -fL https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip -o ${PACKER_CACHE_DIR}/packer.zip
3421
unzip -o ${PACKER_CACHE_DIR}/packer.zip -d ~/bin
3522

3623
pmm-ovf: fetch
37-
packer build -only virtualbox-ovf packer/pmm.json
38-
39-
pmm-ovf-el9-rc: fetch-el9
40-
sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
4124
/usr/bin/packer build \
42-
-var 'pmm_client_repos=original testing' \
43-
-var 'pmm_client_repo_name=percona-testing-x86_64' \
44-
-var 'pmm_server_repo=testing' \
45-
-only virtualbox-ovf -color=false packer/pmm.el9.json \
46-
| tee build.log
25+
-only virtualbox-ovf -color=false packer/pmm.json | tee build.log
4726

48-
pmm-ovf-el9-3-dev-latest: fetch-el9
49-
sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
27+
# NOTE: no difference between rc and dev-latest (i.e. pmm-ovf) ATM
28+
pmm-ovf-rc: fetch
5029
/usr/bin/packer build \
51-
-var 'pmm_client_repos=original experimental' \
52-
-var 'pmm_client_repo_name=percona-experimental-x86_64' \
53-
-var 'pmm_server_repo=experimental' \
54-
-only virtualbox-ovf -color=false packer/pmm.el9.json \
55-
| tee build.log
30+
-only virtualbox-ovf -color=false packer/pmm.json | tee build.log
5631

5732
pmm-digitalocean:
5833
packer build -only digitalocean -var 'single_disk=true' packer/pmm.json
@@ -62,11 +37,10 @@ pmm-azure:
6237

6338
pmm-ami:
6439
docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build \hashicorp/packer:${PACKER_VERSION} \
65-
build -only amazon-ebs -color=false packer/pmm.json | tee build.log
66-
40+
build -only amazon-ebs -color=false packer/pmm.json | tee build.log
6741
## ----------------- PACKER ------------------
6842

69-
check: ## Run required checkers and linters
43+
check: ## Run required checks and linters
7044
ansible-playbook --syntax-check ansible/pmm-docker/update.yml
7145
ansible-playbook --check ansible/pmm-docker/update.yml
7246
ansible-lint ansible/pmm-docker/update.yml

‎build/ansible/pmm-docker/update.yml

-24
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@
111111
become: true
112112
become_user: pmm
113113
become_method: su
114-
# changed_when: true
115114
with_nested:
116115
- - nginx
117116
- grafana
@@ -125,29 +124,6 @@
125124
- name: Check grafana logs
126125
shell: cat /srv/logs/grafana.log
127126

128-
- name: Fix grafana fields type
129-
postgresql_query:
130-
db: grafana
131-
query: "{{ item }}"
132-
loop:
133-
- ALTER TABLE tag ALTER COLUMN key TYPE text;
134-
- ALTER TABLE tag ALTER COLUMN value TYPE text;
135-
- ALTER TABLE api_key ALTER COLUMN key TYPE text;
136-
- ALTER TABLE api_key ALTER COLUMN name TYPE text;
137-
when: not ansible_check_mode
138-
139-
- name: Change default admin id
140-
postgresql_query:
141-
db: grafana
142-
query: UPDATE "user" SET id='1' WHERE login='admin';
143-
when: not ansible_check_mode
144-
145-
# - name: Remove redundant packages
146-
# yum:
147-
# state: absent
148-
# name:
149-
# - logrotate # https://jira.percona.com/browse/PMM-7627
150-
151127
# Regenerating pmm.ini and enabling pmm-update-perform-init
152128
- name: Generate new supervisor config
153129
command: pmm-managed-init

‎build/ansible/pmm/post-build-actions.yml

-8
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
become: yes
66
gather_facts: yes
77
vars:
8-
pmm_client_repos: "original testing"
9-
pmm_client_repos_final: "original release"
108
pmm_server_distribution: "docker"
119

1210
tasks:
@@ -20,12 +18,6 @@
2018
owner: pmm
2119
group: pmm
2220

23-
- name: Disable testing repo for pmm-client
24-
command: percona-release disable {{ pmm_client_repos }}
25-
26-
- name: Enable release repo for pmm-client
27-
command: percona-release enable {{ pmm_client_repos_final }}
28-
2921
- name: Set up pmm-agent
3022
command: >
3123
pmm-agent setup

‎build/ansible/roles/pmm-images/tasks/main.yml

+32-64
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,24 @@
11
---
2-
- name: Install the GPG key for repo.percona.com
3-
rpm_key:
4-
state: present
5-
key: https://downloads.percona.com/downloads/RPM-GPG-KEY-percona
2+
# We use percona-release package to install the gpg keys and ppg-14
3+
# To workaround the package's incompatibility with RHEL9, we have to disable the gpg verification :(
4+
- name: Add percona-release package
5+
yum:
6+
name: https://repo.percona.com/yum/percona-release-latest.noarch.rpm
7+
state: installed
8+
disable_gpg_check: yes
69

7-
- name: Add PMM3 Server YUM repository
8-
yum_repository:
9-
name: pmm-server
10-
description: PMM Server YUM repository - x86_64
11-
baseurl: https://repo.percona.com/pmm3-components/yum/{{ pmm_server_repo }}/9/RPMS/x86_64/
12-
gpgcheck: yes
13-
enabled: yes
14-
gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
10+
- name: List installed gpg keys
11+
command: ls -la /etc/pki/rpm-gpg
1512

1613
# Local yum repo for building pmm server docker image in autobuild jobs
1714
- name: Add a local YUM repository
18-
when: ansible_virtualization_type == "docker"
1915
yum_repository:
2016
name: local
2117
description: Local YUM repository - x86_64
2218
baseurl: file:///tmp/RPMS
2319
gpgcheck: no
2420
enabled: no
2521

26-
# we use it for pmm-client (TODO we'll need to switch to pmm-client repo)
27-
# To workaround the package's incompatibility with RHEL9, we have to ignore errors :(
28-
# Error: `Failed to validate GPG signature for percona-release-1.0-27.noarch`
29-
# Despite the error, this will still install the repo and the GPG key
30-
- name: Add percona-release repository
31-
yum:
32-
name: https://repo.percona.com/yum/percona-release-latest.noarch.rpm
33-
state: installed
34-
ignore_errors: True
35-
3622
- name: Update OS packages
3723
yum:
3824
name: "*"
@@ -47,41 +33,28 @@
4733
- python3.11-psycopg2
4834
- rsync
4935

50-
- name: Install ansible-core and ansible collections
51-
yum:
52-
name:
53-
- ansible-core
54-
- ansible-collection-community-general
55-
- ansible-collection-community-postgresql
56-
- ansible-collection-ansible-posix
57-
state: present
58-
59-
- name: Create users and groups in docker container
60-
block:
61-
- name: Ensure groups exist with correct gid
62-
group:
63-
name: "{{ item.name }}"
64-
gid: "{{ item.gid }}"
65-
loop:
66-
- { name: pmm, gid: 1000 }
67-
- { name: nginx, gid: 1001 }
68-
- { name: clickhouse, gid: 1002 }
69-
70-
# Note: nginx and clickhouse users will get removed in post-build.
71-
- name: Create users
72-
user:
73-
name: "{{ item.name }}"
74-
uid: "{{ item.uid }}"
75-
home: "{{ item.home }}"
76-
comment: "{{ item.comment }}"
77-
shell: "{{ item.shell }}"
78-
group: "{{ item.group }}"
79-
loop:
80-
- { name: pmm, uid: 1000, comment: "PMM Server", shell: "/usr/bin/bash", home: "/home/pmm", group: pmm, }
81-
- { name: nginx, uid: 1001, comment: "Nginx user", shell: "/sbin/nologin", home: "/dev/null", group: nginx, }
82-
- { name: clickhouse, uid: 1002, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/dev/null", group: clickhouse, }
83-
84-
when: ansible_virtualization_type == "docker"
36+
- name: Create groups
37+
group:
38+
name: "{{ item.name }}"
39+
gid: "{{ item.gid }}"
40+
loop:
41+
- { name: pmm, gid: 1000 }
42+
- { name: nginx, gid: 1001 }
43+
- { name: clickhouse, gid: 1002 }
44+
45+
# Note: nginx and clickhouse users will get removed in post-build.
46+
- name: Create users
47+
user:
48+
name: "{{ item.name }}"
49+
uid: "{{ item.uid }}"
50+
home: "{{ item.home }}"
51+
comment: "{{ item.comment }}"
52+
shell: "{{ item.shell }}"
53+
group: "{{ item.group }}"
54+
loop:
55+
- { name: pmm, uid: 1000, comment: "PMM Server", shell: "/usr/bin/bash", home: "/home/pmm", group: pmm, }
56+
- { name: nginx, uid: 1001, comment: "Nginx user", shell: "/sbin/nologin", home: "/dev/null", group: nginx, }
57+
- { name: clickhouse, uid: 1002, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/dev/null", group: clickhouse, }
8558

8659
- name: Create directories (mask 022)
8760
file:
@@ -105,7 +78,6 @@
10578
mode: 0775
10679

10780
- name: Create dirs
108-
when: ansible_virtualization_type == "docker"
10981
file:
11082
path: "{{ item }}"
11183
state: directory
@@ -125,8 +97,7 @@
12597
- pmm-dump
12698
- vmproxy
12799
state: installed
128-
# line below is sed'ed by build-server-docker script
129-
enablerepo: "pmm-server"
100+
enablerepo: local
130101

131102
- name: Create grafana config
132103
include_role:
@@ -148,9 +119,6 @@
148119
include_role:
149120
name: supervisord
150121

151-
- name: Enable repo for pmm-client
152-
command: percona-release enable {{ pmm_client_repos }}
153-
154122
- name: Install pmm-client
155123
include_role:
156124
name: pmm-client

‎build/ansible/roles/pmm-images/vars/main.yml

-6
This file was deleted.

‎build/ansible/roles/postgres/tasks/main.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,10 @@
88
enabled: yes
99
gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
1010

11-
# we need the old postgres binary for the upgrade process
1211
- name: Install Postgres
1312
when:
1413
- not ansible_check_mode
15-
dnf:
14+
yum:
1615
name:
1716
- percona-postgresql14-server
1817
- percona-postgresql14-contrib
@@ -100,7 +99,7 @@
10099

101100
- name: Upgrade grafana database to the latest schema
102101
command: grafana cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin data-migration encrypt-datasource-passwords
103-
changed_when: True
102+
changed_when: true
104103

105104
- name: Stop Postgres 14 database without supervisor
106105
command: /usr/pgsql-14/bin/pg_ctl stop -D /srv/postgres14

‎build/docker/rpmbuild/Dockerfile

-52
This file was deleted.

‎build/docker/rpmbuild/Dockerfile.el9

+4-4
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@ RUN yum update -y && \
2525
yum clean all && rm -rf /var/cache/yum
2626

2727
# keep that format for easier search
28-
ENV GO_VERSION 1.21.3
29-
ENV GO_RELEASER_VERSION 1.21.2
28+
ENV GO_VERSION 1.21.7
29+
ENV GO_RELEASER_VERSION 1.24.0
3030

3131
RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
32-
wget --progress=dot:giga https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz -O /tmp/golang.tar.gz && \
33-
wget --progress=dot:giga https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm -O /tmp/goreleaser.rpm && \
32+
curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
33+
curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
3434
tar -C /usr/local -xzf /tmp/golang.tar.gz && \
3535
yum install -y /tmp/goreleaser.rpm && \
3636
rm /tmp/golang.tar.gz /tmp/goreleaser.rpm

0 commit comments

Comments
 (0)