From 2ab236f1c697875fa9bb529010e0fa73d28623b6 Mon Sep 17 00:00:00 2001 From: hdsdi3g Date: Sun, 7 Apr 2024 20:32:35 +0200 Subject: [PATCH] Manage internal DEB repo #34 --- .github/workflows/install-ubuntu.yml | 30 +++- .github/workflows/publish-release.yml | 1 - .gitignore | 16 ++ make-deb.bash | 10 +- make-rpm.bash | 1 + man-manage-internal-deb-repo.md | 98 ++++++++++++ rpmbuild/SPECS/rpm-centos.spec | 3 +- run-tests.bash | 3 +- src/usr/bin/manage-internal-deb-repo | 208 ++++++++++++++++++++++++++ 9 files changed, 358 insertions(+), 12 deletions(-) create mode 100644 man-manage-internal-deb-repo.md create mode 100755 src/usr/bin/manage-internal-deb-repo diff --git a/.github/workflows/install-ubuntu.yml b/.github/workflows/install-ubuntu.yml index b8de380..38455a1 100644 --- a/.github/workflows/install-ubuntu.yml +++ b/.github/workflows/install-ubuntu.yml @@ -12,11 +12,29 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install deps - run: sudo apt-get install -y --no-install-recommends build-essential binutils lintian debhelper dh-make devscripts pandoc rpm rpmlint + run: sudo apt-get install -y --no-install-recommends build-essential binutils gpg apt-utils rsync sudo + - name: Create gpg keychain for manage-internal-deb-repo + run: gpg --batch --passphrase '' --quick-gen-key INTERNAL_TEST_USER default default + - name: Create setup dir for manage-internal-deb-repo + run: mkdir -p "$HOME/.config"; + - name: Pre setup manage-internal-deb-repo + run: sh -c 'echo "$HOME/.debrepo" > $HOME/.config/.debrepo' + - name: Create internal repo dir in root side + run: sudo mkdir -p "/opt/internal-repo"; + - name: Pre setup manage-internal-deb-repo in root side + run: sh -c 'echo "/opt/internal-repo" > $HOME/.config/.debrepo-rootdeploy' + - name: Install make-deb deps + run: sudo apt-get install -y --no-install-recommends lintian debhelper dh-make devscripts pandoc rpm rpmlint - name: Prepare app DEB - run: ./make-deb.bash + run: ./make-deb.bash + - name: Put app DEB to internal repo with manage-internal-deb-repo + run: ./src/usr/bin/manage-internal-deb-repo linux-springboot-packager-*.deb + - name: Declare internal repo to APT + run: sudo echo deb [signed-by=/opt/internal-repo/pubkey.asc arch=all] file:///opt/internal-repo stable main | sudo tee /etc/apt/sources.list.d/internal-deb.list > /dev/null + - name: Update APT + run: sudo apt-get update - name: Install app - run: sudo dpkg -i linux-springboot-packager-*.deb + run: sudo apt-get install -y --no-install-recommends linux-springboot-packager - name: Set up JDK 17 uses: actions/setup-java@v3 with: @@ -31,8 +49,12 @@ jobs: run: make-springboot-deb test/demospringboot test/temp - name: Run internal make DEB CLI test run: make-springboot-deb test/democlispringboot test/temp + - name: Publish locally internal DEB CLI test + run: manage-internal-deb-repo test/temp/democlispringboot-0.0.1-SNAPSHOT.deb + - name: Update APT + run: sudo apt-get update - name: Install internal DEB CLI test - run: sudo dpkg -i test/temp/democlispringboot-0.0.1-SNAPSHOT.deb + run: sudo apt-get install -y --no-install-recommends democlispringboot - name: Run internal DEB CLI test run: democlispringboot A B - name: Run internal make RPM test diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index bb92822..a9bacec 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -46,4 +46,3 @@ jobs: -H "Content-Type: text/plain" \ --data-binary "@${{ env.DEB_FILE }}" \ ${{ env.UPLOAD_URL }}?name=linux-springboot-packager-${{ env.VERSION }}.deb - \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1b73f33..10e1211 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,22 @@ make-springboot-exe.1 search-winsw.bash.1 pkgdeb +# bootstrap-deb-repo +aptftp.conf +aptgenerate.conf +packages-all.db +pubkey.asc +Release +Release.gpg +InRelease +Contents.gz +Contents.bz2 +Contents-all.gz +Contents-all.bz2 +Packages +Packages.gz +Packages.bz2 + # Maven target/ !**/src/main/** diff --git a/make-deb.bash b/make-deb.bash index 7d3fda2..fd2e384 100755 --- a/make-deb.bash +++ b/make-deb.bash @@ -48,11 +48,10 @@ VERSION="$(git describe --tags || echo "0.SNAPSHOT")"; echo "Architecture: all" echo "Section: devel" echo "Priority: optional" - echo "Depends: bash (>=5), coreutils (>=8.3), man-db, pandoc, rpmlint, rpm" - echo "Recommends: maven" - echo "Suggests: nodejs, nsis, default-jdk" + echo "Depends: bash (>=5), coreutils (>=8.3), man-db, pandoc, rpmlint, rpm, gpg, apt-utils" + echo "Suggests: maven, nodejs, nsis, sudo, rsync" echo "Homepage: https://github.com/hdsdi3g/linux-springboot-packager" - echo "Description: Create Linux RPM packages and Windows installers" + echo "Description: Create Linux RPM/DEB packages and Windows installers" echo " for a Spring Boot project." } > "$CONTROL_FILE" @@ -75,6 +74,7 @@ mkdir -p "$MAN_DIR" pandoc -s -t man -o "$MAN_DIR/make-springboot-rpm.1" "$ROOT/man-make-springboot-rpm.md" pandoc -s -t man -o "$MAN_DIR/make-springboot-deb.1" "$ROOT/man-make-springboot-deb.md" pandoc -s -t man -o "$MAN_DIR/make-springboot-exe.1" "$ROOT/man-make-springboot-exe.md" +pandoc -s -t man -o "$MAN_DIR/manage-internal-deb-repo.1" "$ROOT/man-manage-internal-deb-repo.md" pandoc -s -t man -o "$MAN_DIR/search-winsw.bash.1" "$ROOT/search-winsw.bash.md" # PREPARE COPYRIGHT @@ -110,4 +110,4 @@ lintian --fail-on warning \ "$PACKAGE_FILE" echo ""; -echo "Now, you can install this app with sudo dpkg -i $PACKAGE_FILE"; +echo "Now, you can install this app with sudo dpkg -i $PACKAGE_FILE, or publish it via manage-internal-deb-repo"; diff --git a/make-rpm.bash b/make-rpm.bash index 870210f..1097b6a 100755 --- a/make-rpm.bash +++ b/make-rpm.bash @@ -55,6 +55,7 @@ pandoc -s -t man -o "$MAN_DIR/make-springboot-rpm.1" "$ROOT/man-make-springboot- pandoc -s -t man -o "$MAN_DIR/make-springboot-deb.1" "$ROOT/man-make-springboot-deb.md" pandoc -s -t man -o "$MAN_DIR/make-springboot-exe.1" "$ROOT/man-make-springboot-exe.md" pandoc -s -t man -o "$MAN_DIR/search-winsw.bash.1" "$ROOT/search-winsw.bash.md" +pandoc -s -t man -o "$MAN_DIR/manage-internal-deb-repo.1" "$ROOT/man-manage-internal-deb-repo.md" SPEC_FILE="rpmbuild/SPECS/rpm-centos.spec"; rpmlint "$SPEC_FILE" diff --git a/man-manage-internal-deb-repo.md b/man-manage-internal-deb-repo.md new file mode 100644 index 0000000..eb8bcbe --- /dev/null +++ b/man-manage-internal-deb-repo.md @@ -0,0 +1,98 @@ +% manage-internal-deb-repo(1) linux-springboot-packager documentation +% linux-springboot-packager + +# NAME +manage-internal-deb-repo - Create/manage a Debian "internal" repository, and push new deb files to it + +# SYNOPSIS + +manage-internal-deb-repo *[<NEW_DEB_PACKAGE>]* + +# DESCRIPTION + +CLI for automate actions with GPG, apt-ftparchive and apt to create a **local** deb/APT repository, signed with your GPG key. + +It can optionnaly push it to another directory via sudo and rsync. + +It's Debian package agnostic, but only works for architecture "all". + +# OPTIONS +**BY DEFAULT** + +Create an empty repository locally (ask on startup a path for it). + +Scan new added packages in "./pool" directory and update the repository with these new packages. + +**NEW_DEB_PACKAGE** + +Add create/update the local repository, if needed, and add it (move) the provided package file. + +# SETUP REPO + +To add your new repository avaliable from your host **apt** database, update **/etc/apt/sources.list** file or create a new **/etc/apt/sources.list.d/internal-deb.list** file (the name is not important to work) with: + + deb [signed-by=/pubkey.asc arch=all] file:// stable main + +After the first run, the script can help you with a one-line command to do this. + +To enable the new setup, just: + + apt-get update + +And, as usual: + + apt-get dist-upgrade + apt-get install + +Thanks to the magic of APT, if you push a new version, it will be automatically installed on the next upgrade action. + +**BEWARE** + +From a **security** point of view, it's not wise to let a local user be able to propose packages for installation or update for the entire system. If there is an automatic update service on the machine, consider that local user has now **absolute root rights**. + +Only root user (or sudoers) should be able to have the right to manage the contents of the repositories. That's why there is a secondary and optional mechanism (rootdeploy/sudo rsync) to push this repository to anther directory, with root rights. + +# PREREQUISITES + +You must have a functionnal gpg setup, with a default private key. It will be used to sign packages (technically, it sign manifests files). Your public key will be provided to apt to use it. + +# FILES + +**$HOME/.config/.debrepo** + +Default configuration file: just the path to the local repo. Created directly by this script on first start. + +**$HOME/.debrepo** + +The default local directory. Not created if not needed. + +**$HOME/.config/.debrepo-rootdeploy** + +The specific rsync target to push the local repo to another directory, with root rights. + +If needed, must be created by hand (just a path in a simple text file). + +# EXIT CODES +| Error name | Exit code | +| ------------------------------------------ | --------- | +| EXIT_CODE_MISSING_DEPENDENCY_COMMAND | 1 | +| EXIT_CODE_MISSING_RSYNC_TARGET_DIR | 2 | + +# BUGS +Free feel to send issues to https://github.com/hdsdi3g/linux-springboot-packager/issues. + +# AUTHORS +This application was writted by **hdsdi3g**; see on GitHub https://github.com/hdsdi3g/linux-springboot-packager. + +# SEE ALSO +**make-springboot-deb(1)**. + +https://unix.stackexchange.com/questions/403485/how-to-generate-the-release-file-on-a-local-package-repository + +https://gist.github.com/aarroyoc/1a96b2f8b01fcf34221a + +# NOTES +This document was transformed by *pandoc* from the original markdown documentation file. + +# COPYRIGHT +Copyright (C) hdsdi3g for hd3g.tv 2024, under the **GNU General Public License v3+** diff --git a/rpmbuild/SPECS/rpm-centos.spec b/rpmbuild/SPECS/rpm-centos.spec index 3c42b89..f7ed4ac 100644 --- a/rpmbuild/SPECS/rpm-centos.spec +++ b/rpmbuild/SPECS/rpm-centos.spec @@ -28,7 +28,7 @@ Release: %{getenv:RELEASE} BuildArch: noarch Group: Development/Tools Vendor: hd3g.tv -Requires: bash man pandoc xmlstarlet git rpm-build rpmlint +Requires: bash man pandoc xmlstarlet git rpm-build rpmlint rsync gpg %description You will need Java, Maven, basename, realpath, and optionnally rpmbuild, rpmlint, npm, makensis. @@ -39,6 +39,7 @@ Provided by hd3g.tv (https://hd3g.tv), for more information, go to https://githu %attr(0755,root,root) %{_bindir}/make-springboot-exe %attr(0755,root,root) %{_bindir}/make-springboot-rpm %attr(0755,root,root) %{_bindir}/make-springboot-deb +%attr(0755,root,root) %{_bindir}/manage-internal-deb-repo %attr(0755,root,root) %{_bindir}/search-winsw.bash %attr(0644, root,root) %{_libdir}/linux-springboot-packager/* %doc %attr(0644, root,root) /usr/local/share/man/man1/* diff --git a/run-tests.bash b/run-tests.bash index d517d8b..a0ebcd3 100755 --- a/run-tests.bash +++ b/run-tests.bash @@ -17,6 +17,8 @@ # along with this program. If not, see . # # Usage: ./run-tests +# +# manage-internal-deb-repo are not tested here set -eu @@ -61,7 +63,6 @@ fi ################ # RPM TEST ZONE ################ - "$PREFIX/usr/bin/make-springboot-rpm" "$TESTROOT/demospringboot" "$TEST_TEMP_DIR" EXPECTED_TEST_PACKAGE="$TEST_TEMP_DIR/demospringboot-0.0.1-SNAPSHOT.rpm"; diff --git a/src/usr/bin/manage-internal-deb-repo b/src/usr/bin/manage-internal-deb-repo new file mode 100755 index 0000000..ee4163c --- /dev/null +++ b/src/usr/bin/manage-internal-deb-repo @@ -0,0 +1,208 @@ +#!/bin/bash +# manage-internal-deb-repo - create/manage a Debian "internal" repository, and push new deb files to it +# +# Copyright (C) hdsdi3g for hd3g.tv 2024 +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or any +# later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Interactive usage (bootstrap and manual craft repo): +# manage-internal-deb-repo +# Automatic usage: +# manage-internal-deb-repo package-to-add.deb +# +# You will need gpg, with a private key setup by default, realpath, apt-ftparchive, optionnaly rsync. +# Fonctionnal on Debian-like hosts. +# Deb package agnostic, but only works for architecture "all". + +# https://unix.stackexchange.com/questions/403485/how-to-generate-the-release-file-on-a-local-package-repository +# https://gist.github.com/aarroyoc/1a96b2f8b01fcf34221a + +set -eu + +EXIT_CODE_MISSING_DEPENDENCY_COMMAND=1; +EXIT_CODE_MISSING_RSYNC_TARGET_DIR=2; + +if ! [ -x "$(command -v realpath)" ]; then + echo "Error: realpath is not installed." >&2 + exit $EXIT_CODE_MISSING_DEPENDENCY_COMMAND; +fi +if ! [ -x "$(command -v gpg)" ]; then + echo "Error: gpg is not installed." >&2 + exit $EXIT_CODE_MISSING_DEPENDENCY_COMMAND; +fi +if ! [ -x "$(command -v apt-ftparchive)" ]; then + echo "Error: apt-ftparchive is not installed." >&2 + exit $EXIT_CODE_MISSING_DEPENDENCY_COMMAND; +fi +if [ "$(id -u)" -eq 0 ]; then + echo "Don't run this script as root!" >&2 + exit 2; +fi + +SCRIPT_DIR="$(pwd)"; +CONF_FILE="$HOME/.config/.debrepo"; +DEFAULT_LOCAL_REPO="$HOME/.debrepo"; +IS_FIRST_STARTS="0"; + +if [ ! -f "$CONF_FILE" ]; then + echo "Enter a working directory (future deb repository), it will be created, if needed."; + read -r -p "Directory path: [$DEFAULT_LOCAL_REPO]" RELATIVE_REPO_DIR; + + if [ "$RELATIVE_REPO_DIR" == "" ]; then + REPO_DIR="$DEFAULT_LOCAL_REPO"; + else + REPO_DIR=$(realpath "$RELATIVE_REPO_DIR"); + fi + + read -r -p "Work on $REPO_DIR ? [y/N] " CONTINUE; + + if [ "$CONTINUE" != "y" ]; then + echo "Cancel script"; + exit 0; + fi + echo "Save configuration on $CONF_FILE"; + mkdir -p "$(dirname "$CONF_FILE")"; + echo "$REPO_DIR" > "$CONF_FILE"; +else + REPO_DIR="$(cat "$CONF_FILE")"; + echo "Load working dir $REPO_DIR (from configuration file $CONF_FILE)"; +fi + +mkdir -p "$REPO_DIR"; +cd "$REPO_DIR"; +mkdir -p "dists/stable/main/binary-all"; +mkdir -p "pool"; + +if [ -f "packages-all.db" ]; then + rm -f "packages-all.db"; +fi + +APTCONF_FILE="aptftp.conf"; +if [ ! -f "$APTCONF_FILE" ]; then + IS_FIRST_STARTS="1"; + echo "Create $APTCONF_FILE.You can restart this script after edit it as you needs."; + cat << EOF > "$APTCONF_FILE" +APT::FTPArchive::Release { +Suite "stable"; +Codename "stable"; +Architectures "all"; +Components "main"; +Origin "Internal only crafted packages"; +Label "Internal only package use"; +Description "https://github.com/hdsdi3g/linux-springboot-packager"; +}; +EOF +fi + +APTGENERATE_FILE="aptgenerate.conf"; +if [ ! -f "$APTGENERATE_FILE" ]; then + IS_FIRST_STARTS="1"; + echo "Create $APTGENERATE_FILE. You can restart this script after edit it as you needs."; + cat << EOF > "$APTGENERATE_FILE" +Dir::ArchiveDir "."; +Dir::CacheDir "."; +TreeDefault::Directory "pool/"; +Default::Packages::Extensions ".deb"; +Default::Packages::Compress ". gzip bzip2"; +Default::Contents::Compress "gzip bzip2"; + +BinDirectory "dists/stable/main/binary-all" { + Packages "dists/stable/main/binary-all/Packages"; + Contents "dists/stable/Contents"; +}; + +Tree "dists/stable" { + Sections "main"; + Architectures "all"; +}; +EOF +fi + +PUBKEY_FILE="pubkey.asc"; +if [ ! -f "$PUBKEY_FILE" ]; then + IS_FIRST_STARTS="1"; + echo "Export $PUBKEY_FILE from your gpg public key"; + gpg --pinentry-mode loopback --export --armor | tee "$PUBKEY_FILE" > /dev/null +fi + +if [ "$#" -eq 1 ]; then + if [[ "$1" != *".deb" ]]; then + echo "Only add a deb file on command line: $1" >&2 + exit 2; + fi + NEW_DEB_FILE="$1"; + if [ ! -f "$NEW_DEB_FILE" ]; then + NEW_DEB_FILE="$SCRIPT_DIR/$NEW_DEB_FILE"; + fi + if [ ! -f "$NEW_DEB_FILE" ]; then + echo "The supplied file on command line don't exists: $NEW_DEB_FILE" >&2 + exit 2; + fi + + echo "Import (move) $NEW_DEB_FILE to the internal repo"; + DEST_DEB_FILE="$(basename "$NEW_DEB_FILE")"; + if [[ "$DEST_DEB_FILE" != *"_all.deb" ]]; then + DEST_DEB_FILE="$(basename "$DEST_DEB_FILE" .deb)_all.deb"; + fi + + DEST_DEB_FILE="$REPO_DIR/pool/$DEST_DEB_FILE"; + if [ -f "$DEST_DEB_FILE" ]; then + rm -f "$DEST_DEB_FILE"; + fi + mv "$NEW_DEB_FILE" "$DEST_DEB_FILE"; +fi + +POOL_FILE_COUNT=$(find pool -name "*_all.deb" -type f | wc -l); + +if [ "$POOL_FILE_COUNT" == "0" ]; then + echo "Please, put DEB files on $(pwd)/pool"; + echo "Beware, all files names must end with \"_all.deb\"!"; + echo "Or, just add a deb file as a paramerer to this script"; + echo "And after, restart this script."; + exit 0; +fi + +echo "Generate repository on $POOL_FILE_COUNT deb file(s)..."; +apt-ftparchive generate -c=aptftp.conf aptgenerate.conf +apt-ftparchive release -c=aptftp.conf dists/stable > dists/stable/Release +rm -f dists/stable/Release.gpg +rm -f dists/stable/InRelease + +echo "Sign repository manifests (Release files) with GPG..." +gpg --pinentry-mode loopback --output dists/stable/Release.gpg -ba dists/stable/Release +gpg --pinentry-mode loopback -a --yes --clearsign --output dists/stable/InRelease --detach-sign dists/stable/Release + +if [ "$IS_FIRST_STARTS" == "1" ]; then + echo "The repository is created."; + echo ""; + echo "To setup an host with an APT source list file to this local repo, please create a file like internal-deb.list on /etc/apt/sources.list.d, via the command:"; + echo "sudo echo deb [signed-by=$REPO_DIR/pubkey.asc arch=all] file://$REPO_DIR stable main | sudo tee /etc/apt/sources.list.d/internal-deb.list > /dev/null"; +else + echo "The repository is updated."; +fi + +CONF_FILE_ROOT_DEPLOY="$HOME/.config/.debrepo-rootdeploy"; +if [ -f "$CONF_FILE_ROOT_DEPLOY" ]; then + echo "Deploy as root"; + ROOT_DEPLOY_PATH="$(cat "$CONF_FILE_ROOT_DEPLOY")"; + if ! [ -d "$ROOT_DEPLOY_PATH" ]; then + echo "Can't found $ROOT_DEPLOY_PATH directory (via $CONF_FILE_ROOT_DEPLOY)" >&2 + exit $EXIT_CODE_MISSING_RSYNC_TARGET_DIR; + fi + if ! [ -x "$(command -v rsync)" ]; then + echo "Error: rsync is not installed." >&2 + exit $EXIT_CODE_MISSING_DEPENDENCY_COMMAND; + fi + sudo rsync -r --exclude=".*" --exclude="*.conf" --exclude="*.db" --delete "$REPO_DIR/" "$ROOT_DEPLOY_PATH/"; +fi