diff --git a/.bazelrc.github b/.bazelrc.github new file mode 100644 index 00000000..82cb7020 --- /dev/null +++ b/.bazelrc.github @@ -0,0 +1,25 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx2000m +startup --host_jvm_args=-Xms2000m +startup --batch +test --ram_utilization_factor=10 + +# This is so we understand failures better +build --verbose_failures + +# This is so we use a recent enough GCC when building. +# build --crosstool_top //tools/custom_crosstool:CROSSTOOL + +# This is so we don't use sandboxed execution. Sandboxed execution +# runs stuff in a container, and since Travis already runs its script +# in a container (unless you require sudo in your .travis.yml) this +# fails to run tests. +build --spawn_strategy=standalone --genrule_strategy=standalone +test --test_strategy=standalone + +build --experimental_strict_action_env + +build:github_actions --announce_rc + + +try-import %workspace%/user.bazelrc diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..d1329ba8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,89 @@ +name: Release +on: + push: + branches: + - master + pull_request: + +jobs: + native-image: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + include: + - os: macos-latest + graal_url: https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.0.0/graalvm-ce-java11-darwin-amd64-20.0.0.tar.gz + artifact: bazel-deps-macos + bazel_installer_sha: b4c94148f52854b89cff5de38a9eeeb4b0bcb3fb3a027330c46c468d9ea0898b + bazel_version: 2.1.1 + - os: ubuntu-latest + graal_url: https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.0.0/graalvm-ce-java11-linux-amd64-20.0.0.tar.gz + artifact: bazel-deps-linux + bazel_installer_sha: d6cea18d59e9c90c7ec417b2645834f968132de16d0022c7439b1e60438eb8c9 + bazel_version: 2.1.1 + steps: + - uses: actions/checkout@v2 + - name: Bazel output cache + id: cache-bazel + uses: actions/cache@v1 + with: + path: "${{ github.workspace }}/.bazel-cache" + key: ${{ runner.os }}-bazel-out-${{ github.run_id}} + restore-keys: | + ${{ runner.os }}-bazel-out- + - name: Configure bazel config for actions + run: | + echo "build --config github_actions" > user.bazelrc + echo "build --disk_cache ${{ github.workspace }}/.bazel-cache/disk-cache" >> user.bazelrc + echo "build --repository_cache ${{ github.workspace }}/.bazel-cache/repo-cache" >> user.bazelrc + - name: Install bazel + run: ./ci_scripts/ci_install_bazel.sh + env: + BAZEL_VERSION: ${{ matrix.bazel_version }} + BAZEL_INSTALLER_SHA: ${{ matrix.bazel_installer_sha }} + BAZEL_BIN_LOC: "${{ github.workspace }}/.bazel-cache/bazel-bin" + - name: Build fat jar + run: ./ci_scripts/build_fat_jar.sh + env: + BAZEL_VERSION: ${{ matrix.bazel_version }} + BAZEL_INSTALLER_SHA: ${{ matrix.bazel_installer_sha }} + BAZEL_BIN_LOC: "${{ github.workspace }}/.bazel-cache/bazel-bin" + - uses: olafurpg/setup-scala@v5 + - run: jabba install graal-custom@20.0=tgz+${{ matrix.graal_url }} + - name: Make native image + run: ./ci_scripts/make_native_artifact.sh ${{ matrix.graal_url }} + - uses: actions/upload-artifact@master + with: + name: ${{ matrix.artifact }} + path: bazel-deps + make_release: + name: Make release + needs: native-image + runs-on: ubuntu-latest + steps: + - name: Download linux bazel-deps + uses: actions/download-artifact@v1 + with: + name: bazel-deps-linux + path: downloads/bazel-deps-linux + - name: Download macos bazel-deps + uses: actions/download-artifact@v1 + with: + name: bazel-deps-macos + path: downloads/bazel-deps-macos + - name: mv binaries linux + run: mv downloads/bazel-deps-linux/bazel-deps bazel-deps-linux + - name: mv binaries macos + run: mv downloads/bazel-deps-macos/bazel-deps bazel-deps-macos + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "v0.1-${{ github.run_number }}" + prerelease: false + title: "Auto generated release" + files: | + bazel-deps-macos + bazel-deps-linux + id: "automatic_releases" diff --git a/.gitignore b/.gitignore index 4e0dd5ca..ba6a0ce5 100755 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ bazel-* .DS_Store .project/** .ijwb +/templates diff --git a/README.md b/README.md index eabb507c..79f26ae6 100755 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Generate [bazel](https://bazel.build/) dependencies transitively for maven artifacts, with scala support. +## Fetching/usage + +This repo can be cloned and built locally, or you can download pre-build binaries for MacOS and Linux in the releases page. Automatic releases are generated for every commit against master. + ## Usage First, list all of your maven artifact dependencies in a [Dependencies](#dependencies) file. @@ -202,7 +206,7 @@ java_plugin( "//external:jar/com/google/auto/value/auto_value", ], ) -``` +``` If there is only a single `processorClasses` defined, the `java_plugin` rule is named `_plugin`. If there are multiple `processorClasses` defined, each one is named `_plugin_`. @@ -235,12 +239,12 @@ all of the supported options. output_base`). Coursier ignores this option and uses `~/.cache/coursier`. * namePrefix: a string added to the generated workspace names, to avoid conflicts. The external repository names and binding targets of each dependency are prefixed. -* strictVisibility: this is enabled by default, when enabled a target must be explicitly declared in the - `dependencies.yaml` file or it will not be visible to the rest of the workspace. If it is set to `false` all targets +* strictVisibility: this is enabled by default, when enabled a target must be explicitly declared in the + `dependencies.yaml` file or it will not be visible to the rest of the workspace. If it is set to `false` all targets will be generated with `public` visibility. * licenses: a set of strings added a licenses rule to each generated bazel target. Required by bazel if your build targets are under `third_party/`. See the [licenses](https://docs.bazel.build/versions/master/be/functions.html#licenses) function in Bazel. -* resolverType: `aether` or `coursier`. Note that `aether` is slower and seems to silently miss some dependencies for +* resolverType: `aether` or `coursier`. Note that `aether` is slower and seems to silently miss some dependencies for reasons we don't yet understand. * buildFileName: filename of the generated build files diff --git a/ci_scripts/bootstrapping_bazel b/ci_scripts/bootstrapping_bazel new file mode 100755 index 00000000..df9987dd --- /dev/null +++ b/ci_scripts/bootstrapping_bazel @@ -0,0 +1,116 @@ +#!/bin/bash + +# Update the WORKSPACE to point at new bazel versions +# https://github.com/bazelbuild/bazel +set -e + +ORIGINAL_PWD=$PWD +TMPDIR="${TMPDIR:-/tmp}" +RND_UID="${USER}_$(date "+%s")_${RANDOM}" + +function try_goto_cmd_location() { + BASH_LOC=${BASH_SOURCE[0]} + if [ ! -z "$BASH_LOC" ]; then + DIR_NAME=$(dirname "$BASH_LOC") + CMD_LOC="$( cd "$DIR_NAME" && pwd )" + if [ ! -z "$CMD_LOC" ]; then + cd $CMD_LOC + fi + fi +} + +# If we are executed as a sub-process we won't be able to do this +# So we do it as a try +try_goto_cmd_location + +set -e + +findWorkspace() { + OLD_PWD=$PWD + if [ ! -f WORKSPACE ]; then + cd .. + if [ "$PWD" = "$OLD_PWD" ]; then + echo "Didn't find the workspace" + exit 1 + fi + + findWorkspace + fi +} + +findWorkspace + +REPO_ROOT=$PWD +cd $ORIGINAL_PWD + +if [ "$(uname -s)" == "Linux" ]; then + export BAZEL_LAUNCHER_PLATFORM_NAME='linux' +elif [ "$(uname -s)" == "Darwin" ]; then + export BAZEL_LAUNCHER_PLATFORM_NAME='darwin' +else + "Your platform $(uname -s) is unsupported, sorry" + exit 1 +fi + +if [ -z "$BAZEL_REMOTE_SOURCE" ]; then + export BAZEL_REMOTE_SOURCE=https://github.com/bazelbuild/bazel/releases/download +fi + + +SHA_VARIABLE_NAME="BAZEL_INSTALLER_VERSION_${BAZEL_LAUNCHER_PLATFORM_NAME}_SHA" + +if [ -z BAZEL_VERSION ]; then + echo "Must supply a BAZEL_VERSION in the environment" + exit 1 +fi + +if [ -z "$BAZEL_INSTALLER_SHA" ]; then + echo "Must supply a BAZEL_INSTALLER_SHA in the environment" + exit 1 +fi +if [ -z "$BAZEL_BIN_LOC" ]; then + BAZEL_BIN_LOC=~/.bazel_binaries +fi + +mkdir -p $BAZEL_BIN_LOC +export BAZEL_EXEC_PATH=$BAZEL_BIN_LOC/$BAZEL_VERSION/bin/bazel-real + +if [ -f "$BAZEL_EXEC_PATH" ]; then + exec $BAZEL_EXEC_PATH "$@" +fi + +export BUILD_DIR=${TMPDIR}/bazel_b_${RND_UID} +mkdir -p $BUILD_DIR + +( # Opens a subshell + set -e + echo "Installing Bazel, this will take a few seconds depending on your internet connection speed." + cd $BUILD_DIR + + INSTALLER_NAME="bazel-${BAZEL_VERSION}-installer-${BAZEL_LAUNCHER_PLATFORM_NAME}-x86_64.sh" + echo $PWD + if [ -z $BAZEL_INSTALLER_PATH ]; then + BAZEL_INSTALLER_PATH=$BAZEL_REMOTE_SOURCE/${BAZEL_VERSION}/$INSTALLER_NAME + fi + curl -O -L $BAZEL_INSTALLER_PATH + + GENERATED_SHA_256=$(shasum -a 256 $INSTALLER_NAME | awk '{print $1}') + + if [ "$GENERATED_SHA_256" != "$BAZEL_INSTALLER_SHA" ]; then + echo "Sha 256 does not match, expected: $BAZEL_INSTALLER_SHA" + echo "But found $GENERATED_SHA_256" + echo "Recommend you: update the sha to the expected" + echo "and then re-run this script" + exit 1 + fi + + chmod +x ${INSTALLER_NAME} + mkdir -p ${BAZEL_BIN_LOC}/${BAZEL_VERSION}/bin_t + + ./${INSTALLER_NAME} --base=${BAZEL_BIN_LOC}/${BAZEL_VERSION} --bin=${BAZEL_BIN_LOC}/${BAZEL_VERSION}/bin_t +) +rm -rf $BUILD_DIR + +cd $ORIGINAL_PWD +ls -R ${BAZEL_BIN_LOC} +exec $BAZEL_EXEC_PATH "$@" diff --git a/ci_scripts/build_fat_jar.sh b/ci_scripts/build_fat_jar.sh new file mode 100755 index 00000000..d28c48a5 --- /dev/null +++ b/ci_scripts/build_fat_jar.sh @@ -0,0 +1,3 @@ +set -e +./bazel build src/scala/com/github/johnynek/bazel_deps/parseproject_deploy.jar +cp bazel-bin/src/scala/com/github/johnynek/bazel_deps/parseproject_deploy.jar bazel-deps.jar diff --git a/ci_scripts/ci_install_bazel.sh b/ci_scripts/ci_install_bazel.sh new file mode 100755 index 00000000..04fedff5 --- /dev/null +++ b/ci_scripts/ci_install_bazel.sh @@ -0,0 +1,10 @@ +set -e +cd $GITHUB_WORKSPACE + +cp ci_scripts/bootstrapping_bazel bazel +cp ci_scripts/bootstrapping_bazel tools/bazel + +./bazel +cp .bazelrc.github .bazelrc + + diff --git a/ci_scripts/make_native_artifact.sh b/ci_scripts/make_native_artifact.sh new file mode 100755 index 00000000..a10bcdda --- /dev/null +++ b/ci_scripts/make_native_artifact.sh @@ -0,0 +1,41 @@ +set -eux +# NOTE(olafur): for some reason `jabba use ...` doesn't seem to work on GH Actions +export JAVA_HOME=$(jabba which --home graal-custom@20.0) +export PATH=$JAVA_HOME/bin:$PATH +echo $JAVA_HOME +which gu +gu install native-image + +cd $GITHUB_WORKSPACE + +rm -rf native_image_working_directory +mkdir native_image_working_directory +cd native_image_working_directory + +mkdir templates/ +cp ../src/scala/com/github/johnynek/bazel_deps/templates/* templates/ +cp ../bazel-deps.jar . +cp ../ci_scripts/reflection.json . + +native-image -H:+ReportUnsupportedElementsAtRuntime \ + --initialize-at-build-time \ + --no-server \ + --enable-http \ + --enable-https \ + -H:Log=registerResource: \ + -H:EnableURLProtocols=http,https \ + --enable-all-security-services \ + -H:ReflectionConfigurationFiles=reflection.json \ + --allow-incomplete-classpath \ + -H:+ReportExceptionStackTraces \ + --no-fallback \ + -H:IncludeResources='.*bzl$' \ + -jar bazel-deps.jar + + cd .. + rm -f bazel-deps + mv native_image_working_directory/bazel-deps . + rm -rf native_image_working_directory + +# ensure it actually works! +./bazel-deps generate --repo-root `pwd` --sha-file 3rdparty/workspace.bzl --deps dependencies.yaml --target-file 3rdparty/target_file.bzl --disable-3rdparty-in-repo diff --git a/ci_scripts/reflection.json b/ci_scripts/reflection.json new file mode 100644 index 00000000..c2ea1ef2 --- /dev/null +++ b/ci_scripts/reflection.json @@ -0,0 +1,11 @@ +[ + { + "name" : "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl", + "allDeclaredConstructors" : true, + "allPublicConstructors" : true, + "allDeclaredMethods" : true, + "allPublicMethods" : true, + "allDeclaredClasses" : true, + "allPublicClasses" : true + } +] diff --git a/src/scala/com/github/johnynek/bazel_deps/Writer.scala b/src/scala/com/github/johnynek/bazel_deps/Writer.scala index d2ebc9ea..d867b21a 100644 --- a/src/scala/com/github/johnynek/bazel_deps/Writer.scala +++ b/src/scala/com/github/johnynek/bazel_deps/Writer.scala @@ -10,12 +10,27 @@ import scala.io.Source import scala.util.{Failure, Success} object Writer { - private lazy val jarArtifactBackend = Source.fromInputStream( - getClass.getResource("/templates/jar_artifact_backend.bzl").openStream()).mkString + // This changed from using Source.fromInputStream, as this prior method method could result in null values in a native-image. + private[this] def loadResourceToString(path: String): String = { + val is = getClass.getResourceAsStream(path) + val outputBuffer = new java.io.ByteArrayOutputStream(); + val data = new Array[Byte](1024) + @annotation.tailrec + def go() { + val nRead = is.read(data, 0, data.length) + if(nRead != -1) { + outputBuffer.write(data, 0, nRead) + go + } + } + go() + outputBuffer.flush(); + new String(outputBuffer.toByteArray()) + } + private lazy val jarArtifactBackend = loadResourceToString("/templates/jar_artifact_backend.bzl") - private lazy val externalWorkspaceBackend = Source.fromInputStream( - getClass.getResource("/templates/external_workspace_backend.bzl").openStream()).mkString + private lazy val externalWorkspaceBackend = loadResourceToString("/templates/external_workspace_backend.bzl") sealed abstract class TargetsError { def message: String