Add's some GitHub Actions configuration and produces linux + Mac nati…

…ve images (bazeltools#282)

Add native-image generation using Graal, and generate a release soon every master commit.

.bazelrc.github

@@ -0,0 +1,25 @@
+# This is from Bazel's former travis setup, to avoid blowing up the RAM usage.
+startup --host_jvm_args=-Xmx2000m
+startup --host_jvm_args=-Xms2000m
+startup --batch
+test --ram_utilization_factor=10
+
+# This is so we understand failures better
+build --verbose_failures
+
+# This is so we use a recent enough GCC when building.
+# build --crosstool_top //tools/custom_crosstool:CROSSTOOL
+
+# This is so we don't use sandboxed execution. Sandboxed execution
+# runs stuff in a container, and since Travis already runs its script
+# in a container (unless you require sudo in your .travis.yml) this
+# fails to run tests.
+build --spawn_strategy=standalone --genrule_strategy=standalone
+test --test_strategy=standalone
+
+build --experimental_strict_action_env
+
+build:github_actions --announce_rc
+
+
+try-import %workspace%/user.bazelrc

.github/workflows/release.yml

@@ -0,0 +1,89 @@
+name: Release
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  native-image:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+        include:
+          - os: macos-latest
+            graal_url: https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.0.0/graalvm-ce-java11-darwin-amd64-20.0.0.tar.gz
+            artifact: bazel-deps-macos
+            bazel_installer_sha: b4c94148f52854b89cff5de38a9eeeb4b0bcb3fb3a027330c46c468d9ea0898b
+            bazel_version: 2.1.1
+          - os: ubuntu-latest
+            graal_url: https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.0.0/graalvm-ce-java11-linux-amd64-20.0.0.tar.gz
+            artifact: bazel-deps-linux
+            bazel_installer_sha: d6cea18d59e9c90c7ec417b2645834f968132de16d0022c7439b1e60438eb8c9
+            bazel_version: 2.1.1
+    steps:
+      - uses: actions/checkout@v2
+      - name: Bazel output cache
+        id: cache-bazel
+        uses: actions/cache@v1
+        with:
+          path: "${{ github.workspace }}/.bazel-cache"
+          key: ${{ runner.os }}-bazel-out-${{ github.run_id}}
+          restore-keys: |
+            ${{ runner.os }}-bazel-out-
+      - name: Configure bazel config for actions
+        run: |
+          echo "build --config github_actions" > user.bazelrc
+          echo "build --disk_cache ${{ github.workspace }}/.bazel-cache/disk-cache" >> user.bazelrc
+          echo "build --repository_cache ${{ github.workspace }}/.bazel-cache/repo-cache" >> user.bazelrc
+      - name: Install bazel
+        run: ./ci_scripts/ci_install_bazel.sh
+        env:
+          BAZEL_VERSION: ${{ matrix.bazel_version }}
+          BAZEL_INSTALLER_SHA: ${{ matrix.bazel_installer_sha }}
+          BAZEL_BIN_LOC: "${{ github.workspace }}/.bazel-cache/bazel-bin"
+      - name: Build fat jar
+        run: ./ci_scripts/build_fat_jar.sh
+        env:
+          BAZEL_VERSION: ${{ matrix.bazel_version }}
+          BAZEL_INSTALLER_SHA: ${{ matrix.bazel_installer_sha }}
+          BAZEL_BIN_LOC: "${{ github.workspace }}/.bazel-cache/bazel-bin"
+      - uses: olafurpg/setup-scala@v5
+      - run: jabba install graal-custom@20.0=tgz+${{ matrix.graal_url }}
+      - name: Make native image
+        run: ./ci_scripts/make_native_artifact.sh  ${{ matrix.graal_url }}
+      - uses: actions/upload-artifact@master
+        with:
+          name: ${{ matrix.artifact }}
+          path: bazel-deps
+  make_release:
+    name: Make release
+    needs: native-image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download linux bazel-deps
+        uses: actions/download-artifact@v1
+        with:
+          name: bazel-deps-linux
+          path: downloads/bazel-deps-linux
+      - name: Download macos bazel-deps
+        uses: actions/download-artifact@v1
+        with:
+          name: bazel-deps-macos
+          path: downloads/bazel-deps-macos
+      - name: mv binaries linux
+        run: mv downloads/bazel-deps-linux/bazel-deps bazel-deps-linux
+      - name: mv binaries macos
+        run: mv downloads/bazel-deps-macos/bazel-deps bazel-deps-macos
+      - uses: "marvinpinto/action-automatic-releases@latest"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          automatic_release_tag: "v0.1-${{ github.run_number }}"
+          prerelease: false
+          title: "Auto generated release"
+          files: |
+            bazel-deps-macos
+            bazel-deps-linux
+        id: "automatic_releases"

.gitignore

@@ -7,3 +7,4 @@ bazel-*
 .DS_Store
 .project/**
 .ijwb
+/templates

README.md

@@ -3,6 +3,10 @@
 Generate [bazel](https://bazel.build/) dependencies transitively for maven artifacts, with scala
 support.
 
+## Fetching/usage
+
+This repo can be cloned and built locally, or you can download pre-build binaries for MacOS and Linux in the releases page. Automatic releases are generated for every commit against master.
+
 ## Usage
 
 First, list all of your maven artifact dependencies in a [Dependencies](#dependencies) file.
@@ -202,7 +206,7 @@ java_plugin(
         "//external:jar/com/google/auto/value/auto_value",
     ],
 )
-``` 
+```
 If there is only a single `processorClasses` defined, the `java_plugin` rule is named `<java_library_name>_plugin`. If there are multiple
 `processorClasses` defined, each one is named `<java_library_name>_plugin_<processor_class_to_snake_case>`.
 
@@ -235,12 +239,12 @@ all of the supported options.
   output_base`).  Coursier ignores this option and uses `~/.cache/coursier`.
 * namePrefix: a string added to the generated workspace names, to avoid conflicts.  The external repository names and
   binding targets of each dependency are prefixed.
-* strictVisibility: this is enabled by default, when enabled a target must be explicitly declared in the 
-  `dependencies.yaml` file or it will not be visible to the rest of the workspace. If it is set to `false` all targets 
+* strictVisibility: this is enabled by default, when enabled a target must be explicitly declared in the
+  `dependencies.yaml` file or it will not be visible to the rest of the workspace. If it is set to `false` all targets
   will be generated with `public` visibility.
 * licenses: a set of strings added a licenses rule to each generated bazel target.  Required by
   bazel if your build targets are under `third_party/`. See the [licenses](https://docs.bazel.build/versions/master/be/functions.html#licenses) function in Bazel.
-* resolverType: `aether` or `coursier`. Note that `aether` is slower and seems to silently miss some dependencies for 
+* resolverType: `aether` or `coursier`. Note that `aether` is slower and seems to silently miss some dependencies for
   reasons we don't yet understand.
 * buildFileName: filename of the generated build files
 

ci_scripts/bootstrapping_bazel

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Update the WORKSPACE to point at new bazel versions
+# https://github.com/bazelbuild/bazel
+set -e
+
+ORIGINAL_PWD=$PWD
+TMPDIR="${TMPDIR:-/tmp}"
+RND_UID="${USER}_$(date "+%s")_${RANDOM}"
+
+function try_goto_cmd_location() {
+  BASH_LOC=${BASH_SOURCE[0]}
+  if [ ! -z "$BASH_LOC" ]; then
+    DIR_NAME=$(dirname "$BASH_LOC")
+    CMD_LOC="$( cd "$DIR_NAME" && pwd )"
+    if [ ! -z "$CMD_LOC" ]; then
+      cd $CMD_LOC
+    fi
+  fi
+}
+
+# If we are executed as a sub-process we won't be able to do this
+# So we do it as a try
+try_goto_cmd_location
+
+set -e
+
+findWorkspace() {
+  OLD_PWD=$PWD
+  if [ ! -f WORKSPACE ]; then
+    cd ..
+    if [ "$PWD" = "$OLD_PWD" ]; then
+      echo "Didn't find the workspace"
+      exit 1
+    fi
+
+    findWorkspace
+  fi
+}
+
+findWorkspace
+
+REPO_ROOT=$PWD
+cd $ORIGINAL_PWD
+
+if [ "$(uname -s)" == "Linux" ]; then
+  export BAZEL_LAUNCHER_PLATFORM_NAME='linux'
+elif [ "$(uname -s)" == "Darwin" ]; then
+  export BAZEL_LAUNCHER_PLATFORM_NAME='darwin'
+else
+  "Your platform $(uname -s) is unsupported, sorry"
+  exit 1
+fi
+
+if [ -z "$BAZEL_REMOTE_SOURCE" ]; then
+  export BAZEL_REMOTE_SOURCE=https://github.com/bazelbuild/bazel/releases/download
+fi
+
+
+SHA_VARIABLE_NAME="BAZEL_INSTALLER_VERSION_${BAZEL_LAUNCHER_PLATFORM_NAME}_SHA"
+
+if [ -z BAZEL_VERSION ]; then
+  echo "Must supply a BAZEL_VERSION in the environment"
+  exit 1
+fi
+
+if [ -z "$BAZEL_INSTALLER_SHA" ]; then
+  echo "Must supply a BAZEL_INSTALLER_SHA in the environment"
+  exit 1
+fi
+if [ -z "$BAZEL_BIN_LOC" ]; then
+  BAZEL_BIN_LOC=~/.bazel_binaries
+fi
+
+mkdir -p $BAZEL_BIN_LOC
+export BAZEL_EXEC_PATH=$BAZEL_BIN_LOC/$BAZEL_VERSION/bin/bazel-real
+
+if [ -f "$BAZEL_EXEC_PATH" ]; then
+   exec $BAZEL_EXEC_PATH "$@"
+fi
+
+export BUILD_DIR=${TMPDIR}/bazel_b_${RND_UID}
+mkdir -p $BUILD_DIR
+
+( # Opens a subshell
+  set -e
+  echo "Installing Bazel, this will take a few seconds depending on your internet connection speed."
+  cd $BUILD_DIR
+
+  INSTALLER_NAME="bazel-${BAZEL_VERSION}-installer-${BAZEL_LAUNCHER_PLATFORM_NAME}-x86_64.sh"
+  echo $PWD
+  if [ -z $BAZEL_INSTALLER_PATH ]; then
+    BAZEL_INSTALLER_PATH=$BAZEL_REMOTE_SOURCE/${BAZEL_VERSION}/$INSTALLER_NAME
+  fi
+  curl -O -L $BAZEL_INSTALLER_PATH
+
+  GENERATED_SHA_256=$(shasum -a 256 $INSTALLER_NAME | awk '{print $1}')
+
+  if [ "$GENERATED_SHA_256" != "$BAZEL_INSTALLER_SHA" ]; then
+    echo "Sha 256 does not match, expected: $BAZEL_INSTALLER_SHA"
+    echo "But found $GENERATED_SHA_256"
+    echo "Recommend you:  update the sha to the expected"
+    echo "and then re-run this script"
+    exit 1
+  fi
+
+  chmod +x ${INSTALLER_NAME}
+  mkdir -p ${BAZEL_BIN_LOC}/${BAZEL_VERSION}/bin_t
+
+  ./${INSTALLER_NAME} --base=${BAZEL_BIN_LOC}/${BAZEL_VERSION} --bin=${BAZEL_BIN_LOC}/${BAZEL_VERSION}/bin_t
+)
+rm -rf $BUILD_DIR
+
+cd $ORIGINAL_PWD
+ls -R ${BAZEL_BIN_LOC}
+exec $BAZEL_EXEC_PATH "$@"

ci_scripts/build_fat_jar.sh

@@ -0,0 +1,3 @@
+set -e
+./bazel build src/scala/com/github/johnynek/bazel_deps/parseproject_deploy.jar
+cp bazel-bin/src/scala/com/github/johnynek/bazel_deps/parseproject_deploy.jar bazel-deps.jar

ci_scripts/ci_install_bazel.sh

@@ -0,0 +1,10 @@
+set -e
+cd $GITHUB_WORKSPACE
+
+cp ci_scripts/bootstrapping_bazel bazel
+cp ci_scripts/bootstrapping_bazel tools/bazel
+
+./bazel
+cp .bazelrc.github .bazelrc
+
+

ci_scripts/make_native_artifact.sh

@@ -0,0 +1,41 @@
+set -eux
+# NOTE(olafur): for some reason `jabba use ...` doesn't seem to work on GH Actions
+export JAVA_HOME=$(jabba which --home graal-custom@20.0)
+export PATH=$JAVA_HOME/bin:$PATH
+echo $JAVA_HOME
+which gu
+gu install native-image
+
+cd $GITHUB_WORKSPACE
+
+rm -rf native_image_working_directory
+mkdir native_image_working_directory
+cd native_image_working_directory
+
+mkdir templates/
+cp ../src/scala/com/github/johnynek/bazel_deps/templates/* templates/
+cp ../bazel-deps.jar .
+cp ../ci_scripts/reflection.json .
+
+native-image -H:+ReportUnsupportedElementsAtRuntime \
+ --initialize-at-build-time \
+ --no-server \
+ --enable-http \
+ --enable-https \
+ -H:Log=registerResource: \
+ -H:EnableURLProtocols=http,https \
+ --enable-all-security-services \
+ -H:ReflectionConfigurationFiles=reflection.json \
+ --allow-incomplete-classpath \
+ -H:+ReportExceptionStackTraces \
+ --no-fallback \
+ -H:IncludeResources='.*bzl$' \
+ -jar bazel-deps.jar
+
+ cd ..
+ rm -f bazel-deps
+ mv native_image_working_directory/bazel-deps .
+ rm -rf native_image_working_directory
+
+# ensure it actually works!
+./bazel-deps generate --repo-root `pwd` --sha-file 3rdparty/workspace.bzl --deps dependencies.yaml --target-file 3rdparty/target_file.bzl --disable-3rdparty-in-repo

ci_scripts/reflection.json

@@ -0,0 +1,11 @@
+[
+  {
+    "name" : "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl",
+    "allDeclaredConstructors" : true,
+    "allPublicConstructors" : true,
+    "allDeclaredMethods" : true,
+    "allPublicMethods" : true,
+    "allDeclaredClasses" : true,
+    "allPublicClasses" : true
+  }
+]

src/scala/com/github/johnynek/bazel_deps/Writer.scala

@@ -10,12 +10,27 @@ import scala.io.Source
 import scala.util.{Failure, Success}
 
 object Writer {
-  private lazy val jarArtifactBackend = Source.fromInputStream(
-    getClass.getResource("/templates/jar_artifact_backend.bzl").openStream()).mkString
 
+  // This changed from using Source.fromInputStream, as this prior method method could result in null values in a native-image.
+  private[this] def loadResourceToString(path: String): String = {
+    val is = getClass.getResourceAsStream(path)
+    val outputBuffer = new java.io.ByteArrayOutputStream();
+    val data = new Array[Byte](1024)
+    @annotation.tailrec
+    def go() {
+      val nRead = is.read(data, 0, data.length)
+      if(nRead != -1) {
+        outputBuffer.write(data, 0, nRead)
+        go
+      }
+    }
+    go()
+    outputBuffer.flush();
+    new String(outputBuffer.toByteArray())
+  }
+  private lazy val jarArtifactBackend = loadResourceToString("/templates/jar_artifact_backend.bzl")
 
-  private lazy val externalWorkspaceBackend = Source.fromInputStream(
-    getClass.getResource("/templates/external_workspace_backend.bzl").openStream()).mkString
+  private lazy val externalWorkspaceBackend = loadResourceToString("/templates/external_workspace_backend.bzl")
 
   sealed abstract class TargetsError {
     def message: String