From e8c3b6353ee049ba8ba9a4085829ae30e0fe9420 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 13:15:12 +0100 Subject: [PATCH 1/9] wip --- build.sbt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index fd933b5..cbfbca6 100644 --- a/build.sbt +++ b/build.sbt @@ -177,6 +177,5 @@ lazy val docker = (project in file("docker")) .enablePlugins(JavaAppPackaging, DockerPlugin) .settings( strictBuildSettings, - dockerSettings, - libraryDependencies ++= Seq("com.raw-labs" %% "das-server-scala" % "0.4.1" % "compile->compile;test->test") + dockerSettings ) From 3038b5e23b7c48e3c215e418d97ac7350a6d7962 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 13:50:53 +0100 Subject: [PATCH 2/9] merge docker build into root project --- .github/workflows/docker-ci.yaml | 2 +- .github/workflows/publish.yaml | 2 +- build.sbt | 16 +++++----------- project/plugins.sbt | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index ef9b233..d86eda6 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -14,4 +14,4 @@ jobs: runs-on: self-hosted steps: - uses: actions/checkout@v4 - - run: .github/scripts/dnd-sbt docker/Docker/publishLocal + - run: .github/scripts/dnd-sbt Docker/publishLocal diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 124f1e9..0e76e7e 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -36,7 +36,7 @@ jobs: password: ${{ secrets.WRITE_PACKAGES }} logout: false - name: publish docker images - run: .github/scripts/dnd-sbt docker/Docker/publish + run: .github/scripts/dnd-sbt Docker/publish - name: set should_trigger_deploy id: should_trigger_deploy shell: bash diff --git a/build.sbt b/build.sbt index cbfbca6..eaa89ed 100644 --- a/build.sbt +++ b/build.sbt @@ -80,6 +80,7 @@ lazy val strictBuildSettings = commonSettings ++ compileSettings ++ buildSettings ++ testSettings ++ Seq(scalacOptions ++= Seq("-Xfatal-warnings")) lazy val root = (project in file(".")) + .enablePlugins(JavaAppPackaging, DockerPlugin) .settings( name := "das-databricks", strictBuildSettings, @@ -89,14 +90,16 @@ lazy val root = (project in file(".")) "com.raw-labs" %% "protocol-das" % "1.0.0" % "compile->compile;test->test", "com.raw-labs" %% "das-server-scala" % "0.4.1" % "compile->compile;test->test", // Databricks - "com.databricks" % "databricks-sdk-java" % "0.41.0" % "compile->compile")) + "com.databricks" % "databricks-sdk-java" % "0.41.0" % "compile->compile"), + dockerSettings + ) val amzn_jdk_version = "21.0.4.7-1" val amzn_corretto_bin = s"java-21-amazon-corretto-jdk_${amzn_jdk_version}_amd64.deb" val amzn_corretto_bin_dl_url = s"https://corretto.aws/downloads/resources/${amzn_jdk_version.replace('-', '.')}" lazy val dockerSettings = strictBuildSettings ++ Seq( - name := "das-databricks-server", + Docker/ packageName := "das-databricks-server", dockerBaseImage := s"--platform=amd64 debian:bookworm-slim", dockerLabels ++= Map( "vendor" -> "RAW Labs SA", @@ -159,7 +162,6 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( case lm @ _ => lm }, Compile / mainClass := Some("com.rawlabs.das.server.DASServer"), - Docker / dockerAutoremoveMultiStageIntermediateImages := false, dockerAlias := dockerAlias.value.withTag(Option(version.value.replace("+", "-"))), dockerAliases := { val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-databricks") @@ -171,11 +173,3 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( case None => Seq(baseAlias) } }) - -lazy val docker = (project in file("docker")) - .dependsOn(root % "compile->compile;test->test") - .enablePlugins(JavaAppPackaging, DockerPlugin) - .settings( - strictBuildSettings, - dockerSettings - ) diff --git a/project/plugins.sbt b/project/plugins.sbt index ab3f73c..b852ee6 100755 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,7 +8,7 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.0") addSbtPlugin("nl.gn0s1s" % "sbt-dotenv" % "3.1.1") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.4") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") From 36ebed0b8a922a8aa2cdb2aef1e6b47bbe0d7186 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 13:53:59 +0100 Subject: [PATCH 3/9] change base image --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index eaa89ed..0382bae 100644 --- a/build.sbt +++ b/build.sbt @@ -100,7 +100,7 @@ val amzn_corretto_bin_dl_url = s"https://corretto.aws/downloads/resources/${amzn lazy val dockerSettings = strictBuildSettings ++ Seq( Docker/ packageName := "das-databricks-server", - dockerBaseImage := s"--platform=amd64 debian:bookworm-slim", + dockerBaseImage := "eclipse-temurin:21-jre", dockerLabels ++= Map( "vendor" -> "RAW Labs SA", "product" -> "das-databricks-server", From 9196f989b28be32c69351f573f2ea497e1383dbe Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 13:56:46 +0100 Subject: [PATCH 4/9] fixup! change base image --- build.sbt | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/build.sbt b/build.sbt index 0382bae..691e03f 100644 --- a/build.sbt +++ b/build.sbt @@ -94,10 +94,6 @@ lazy val root = (project in file(".")) dockerSettings ) -val amzn_jdk_version = "21.0.4.7-1" -val amzn_corretto_bin = s"java-21-amazon-corretto-jdk_${amzn_jdk_version}_amd64.deb" -val amzn_corretto_bin_dl_url = s"https://corretto.aws/downloads/resources/${amzn_jdk_version.replace('-', '.')}" - lazy val dockerSettings = strictBuildSettings ++ Seq( Docker/ packageName := "das-databricks-server", dockerBaseImage := "eclipse-temurin:21-jre", @@ -117,20 +113,8 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( case cmd => false }, dockerCommands ++= Seq( - Cmd( - "RUN", - s"""set -eux \\ - && apt-get update \\ - && apt-get install -y --no-install-recommends \\ - curl wget ca-certificates gnupg software-properties-common fontconfig java-common \\ - && wget $amzn_corretto_bin_dl_url/$amzn_corretto_bin \\ - && dpkg --install $amzn_corretto_bin \\ - && rm -f $amzn_corretto_bin \\ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \\ - wget gnupg software-properties-common"""), Cmd("USER", "raw")), dockerEnvVars += "LANG" -> "C.UTF-8", - dockerEnvVars += "JAVA_HOME" -> "/usr/lib/jvm/java-21-amazon-corretto", Compile / doc / sources := Seq.empty, // Do not generate scaladocs // Skip docs to speed up build Compile / packageDoc / mappings := Seq(), From 6b785007fe5788eb92125ca79570e71bf61d4368 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 13:58:39 +0100 Subject: [PATCH 5/9] more robust user uid group and gid --- build.sbt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sbt b/build.sbt index 691e03f..86de5cc 100644 --- a/build.sbt +++ b/build.sbt @@ -103,6 +103,9 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( "image-type" -> "final", "org.opencontainers.image.source" -> "https://github.com/raw-labs/das-databricks"), Docker / daemonUser := "raw", + Docker / daemonUserUid := Some("1001"), + Docker / daemonGroup := "raw", + Docker / daemonGroupGid := Some("1001"), dockerExposedVolumes := Seq("/var/log/raw"), dockerExposedPorts := Seq(50051), dockerEnvVars := Map("PATH" -> s"${(Docker / defaultLinuxInstallLocation).value}/bin:$$PATH"), From 688f39e3c233bf36734ae3c6ab162706dad9c9e2 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 14:16:57 +0100 Subject: [PATCH 6/9] trivy ci --- .github/workflows/docker-ci.yaml | 43 ++++++++++++++++++++++++++++++-- build.sbt | 9 +++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index d86eda6..d09e069 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -5,13 +5,52 @@ on: - .github/workflows/docker-ci.yaml - .github/scripts/** - build.sbt + - src/** env: GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }} jobs: - build: + build-and-test: + name: Build, Scan & Test runs-on: self-hosted steps: - uses: actions/checkout@v4 - - run: .github/scripts/dnd-sbt Docker/publishLocal + + - name: Build Docker image + run: | + .github/scripts/dnd-sbt Docker/publishLocal + IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2) + echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.IMAGE }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + + - name: Test image - run container + run: | + CONTAINER_ID=$(docker run -d -p 50051 ${IMAGE}) + echo "CONTAINER_ID=${CONTAINER_ID}" >> $GITHUB_ENV + sleep 15 + + - name: Test image - verify service is running + run: | + docker exec ${CONTAINER_ID} /opt/docker/bin/healthcheck.sh + if [ $? -ne 0 ]; then + echo "Service check failed!" + exit 1 + fi + + - name: Cleanup container + if: always() + run: | + if [ ! -z "${CONTAINER_ID}" ]; then + docker stop ${CONTAINER_ID} + docker rm ${CONTAINER_ID} + fi diff --git a/build.sbt b/build.sbt index 86de5cc..d7645cd 100644 --- a/build.sbt +++ b/build.sbt @@ -160,3 +160,12 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( case None => Seq(baseAlias) } }) + +lazy val printDockerImageName = taskKey[Unit]("Prints the full Docker image name that will be produced") + +printDockerImageName := { + // Get the main Docker alias (the first one in the sequence) + val alias = (Docker / dockerAliases).value.head + // The toString method already returns the full image name with registry and tag + println(s"DOCKER_IMAGE=${alias}") +} From 0423f5031763d18b33510287620864f6a9b0c314 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 16:28:02 +0100 Subject: [PATCH 7/9] fixup! trivy ci --- build.sbt | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/build.sbt b/build.sbt index d7645cd..f5f0dc9 100644 --- a/build.sbt +++ b/build.sbt @@ -43,7 +43,9 @@ lazy val compileSettings = Seq( Compile / packageBin / packageOptions += Package.ManifestAttributes( "Automatic-Module-Name" -> name.value.replace('-', '.')), // Ensure Java annotations get compiled first, so that they are accessible from Scala. - compileOrder := CompileOrder.JavaThenScala) + compileOrder := CompileOrder.JavaThenScala, + Compile / mainClass := Some("com.rawlabs.das.server.DASServer") + ) lazy val testSettings = Seq( // Ensuring tests are run in a forked JVM for isolation. @@ -94,7 +96,7 @@ lazy val root = (project in file(".")) dockerSettings ) -lazy val dockerSettings = strictBuildSettings ++ Seq( +lazy val dockerSettings = Seq( Docker/ packageName := "das-databricks-server", dockerBaseImage := "eclipse-temurin:21-jre", dockerLabels ++= Map( @@ -109,18 +111,7 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( dockerExposedVolumes := Seq("/var/log/raw"), dockerExposedPorts := Seq(50051), dockerEnvVars := Map("PATH" -> s"${(Docker / defaultLinuxInstallLocation).value}/bin:$$PATH"), - // We remove the automatic switch to USER 1001:0. - // We we want to run as root to install the JDK, also later we will switch to a non-root user. - dockerCommands := dockerCommands.value.filterNot { - case Cmd("USER", args @ _*) => args.contains("1001:0") - case cmd => false - }, - dockerCommands ++= Seq( - Cmd("USER", "raw")), dockerEnvVars += "LANG" -> "C.UTF-8", - Compile / doc / sources := Seq.empty, // Do not generate scaladocs - // Skip docs to speed up build - Compile / packageDoc / mappings := Seq(), updateOptions := updateOptions.value.withLatestSnapshots(true), Linux / linuxPackageMappings += packageTemplateMapping(s"/var/lib/${packageName.value}")(), bashScriptDefines := { @@ -148,7 +139,6 @@ lazy val dockerSettings = strictBuildSettings ++ Seq( } case lm @ _ => lm }, - Compile / mainClass := Some("com.rawlabs.das.server.DASServer"), dockerAlias := dockerAlias.value.withTag(Option(version.value.replace("+", "-"))), dockerAliases := { val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-databricks") From a47e4eac6e0100ba7a41429847c5c1b50822f8a1 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 17:01:55 +0100 Subject: [PATCH 8/9] adapt ci basic healthcheck for now --- .github/workflows/docker-ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index d09e069..444e48c 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -37,11 +37,13 @@ jobs: run: | CONTAINER_ID=$(docker run -d -p 50051 ${IMAGE}) echo "CONTAINER_ID=${CONTAINER_ID}" >> $GITHUB_ENV + HOST_PORT=$(docker port ${CONTAINER_ID} 50051 | cut -d':' -f2) + echo "HOST_PORT=${HOST_PORT}" >> $GITHUB_ENV sleep 15 - name: Test image - verify service is running run: | - docker exec ${CONTAINER_ID} /opt/docker/bin/healthcheck.sh + nc -z localhost ${HOST_PORT} if [ $? -ne 0 ]; then echo "Service check failed!" exit 1 From cf52dc93cfb786e1559148c8f8116124e30a8a00 Mon Sep 17 00:00:00 2001 From: yann Date: Tue, 4 Mar 2025 17:04:00 +0100 Subject: [PATCH 9/9] parallel sec scan --- .github/workflows/docker-ci.yaml | 34 +++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-ci.yaml b/.github/workflows/docker-ci.yaml index 444e48c..db7be94 100644 --- a/.github/workflows/docker-ci.yaml +++ b/.github/workflows/docker-ci.yaml @@ -12,7 +12,7 @@ env: jobs: build-and-test: - name: Build, Scan & Test + name: Build & Test runs-on: self-hosted steps: - uses: actions/checkout@v4 @@ -23,16 +23,6 @@ jobs: IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2) echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ${{ env.IMAGE }} - format: 'table' - exit-code: '1' - ignore-unfixed: true - vuln-type: 'os,library' - severity: 'CRITICAL,HIGH' - - name: Test image - run container run: | CONTAINER_ID=$(docker run -d -p 50051 ${IMAGE}) @@ -56,3 +46,25 @@ jobs: docker stop ${CONTAINER_ID} docker rm ${CONTAINER_ID} fi + + security-scan: + name: Security Scan + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + + - name: Build Docker image + run: | + .github/scripts/dnd-sbt Docker/publishLocal + IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2) + echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ env.IMAGE }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH'