ci: configure env vars for CI in .env file

.env

@@ -41,3 +41,19 @@ DOTNET=7.0
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
 VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"
+
+# These are used to tell tests where to find services for integration testing.
+# They are valid if the services are started with the docker-compose config.
+ADBC_DREMIO_FLIGHTSQL_PASS=dremio123
+ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010
+ADBC_DREMIO_FLIGHTSQL_USER=dremio
+ADBC_JDBC_MSSQL_URL=localhost:1433
+ADBC_JDBC_MSSQL_USER=SA
+ADBC_JDBC_MSSQL_PASSWORD=Password1!
+ADBC_JDBC_POSTGRESQL_URL=localhost:5432/postgres
+ADBC_JDBC_POSTGRESQL_USER=postgres
+ADBC_JDBC_POSTGRESQL_PASSWORD=password
+ADBC_JDBC_POSTGRESQL_DATABASE=postgres
+ADBC_POSTGRESQL_TEST_URI=postgresql://localhost:5432/postgres?user=postgres&password=password
+ADBC_SQLITE_FLIGHTSQL_URI=grpc+tcp://localhost:8080
+ADBC_TEST_FLIGHTSQL_URI=grpc+tcp://localhost:41414

.github/workflows/integration.yml

@@ -138,52 +138,38 @@ jobs:
       - name: Start SQLite server and Dremio
         shell: bash -l {0}
         run: |
-          docker-compose up -d flightsql-test flightsql-sqlite-test dremio dremio-init
+          docker compose up --detach flightsql-test flightsql-sqlite-test dremio dremio-init
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
 
       - name: Build FlightSQL Driver
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
+          BUILD_DRIVER_MANAGER: "1"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test FlightSQL Driver against Dremio and SQLite
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
           ./ci/scripts/cpp_test.sh "$(pwd)/build"
-      - name: Build Python Flight SQL driver
-        shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_FLIGHTSQL: "1"
-          BUILD_DRIVER_MANAGER: "1"
-        run: |
-          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test Python Flight SQL driver against Dremio
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
       - name: Stop SQLite server and Dremio
         shell: bash -l {0}
         run: |
-          docker-compose down
+          docker compose down
 
   postgresql:
     name: "PostgreSQL Integration Tests"
@@ -220,26 +206,23 @@ jobs:
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
+          BUILD_DRIVER_MANAGER: "1"
           BUILD_DRIVER_POSTGRESQL: "1"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
-      - name: Build Python PostgreSQL Driver
+          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Configure Integration Env Vars
         shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_MANAGER: "1"
-          BUILD_DRIVER_POSTGRESQL: "1"
         run: |
-          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
+          echo "ADBC_USE_ASAN=ON" >> $GITHUB_ENV
+          echo "ADBC_USE_UBSAN=ON" >> $GITHUB_ENV
 
       - name: Test PostgreSQL Driver - postgres 11
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=11 docker compose up --wait --detach postgres-test
@@ -251,9 +234,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=12 docker compose up --wait --detach postgres-test
@@ -265,9 +245,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=13 docker compose up --wait --detach postgres-test
@@ -279,9 +256,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=14 docker compose up --wait --detach postgres-test
@@ -293,9 +267,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=15 docker compose up --wait --detach postgres-test
@@ -307,9 +278,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=16 docker compose up --wait --detach postgres-test

.github/workflows/java.yml

@@ -57,16 +57,9 @@ jobs:
       - name: Start test services
         shell: bash -l {0}
         run: |
-          docker compose up -d --wait flightsql-sqlite-test mssql-test postgres-test
+          docker compose up --detach --wait flightsql-sqlite-test mssql-test postgres-test
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
       - name: Build/Test
-        env:
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_JDBC_MSSQL_URL: "localhost:1433"
-          ADBC_JDBC_MSSQL_USER: "SA"
-          ADBC_JDBC_MSSQL_PASSWORD: "Password1!"
-          ADBC_JDBC_POSTGRESQL_URL: "localhost:5432/postgres"
-          ADBC_JDBC_POSTGRESQL_USER: "postgres"
-          ADBC_JDBC_POSTGRESQL_PASSWORD: "password"
         run: |
           cd java
           mvn install
@@ -104,4 +97,5 @@ jobs:
           --add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
           --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
           HERE
-          mvn -P errorprone install
+          # Our linters are compile-time, no point re-running tests
+          mvn -P errorprone install -DskipTests

.github/workflows/native-unix.yml

@@ -572,18 +572,16 @@ jobs:
           retention-days: 2
           path: |
             docs/build/html
+      - name: Configure Integration Env Vars
+        shell: bash -l {0}
+        run: |
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
       - name: Test Recipes (C++)
         shell: bash -l {0}
         run: |
           ./ci/scripts/cpp_recipe.sh $(pwd) ~/local build/recipe
       - name: Test Recipes (Python)
         shell: bash -l {0}
-        env:
-          ADBC_POSTGRESQL_TEST_URI: "postgres://localhost:5432/postgres?user=postgres&password=password"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
         run: |
           docker compose up --detach --wait dremio dremio-init flightsql-sqlite-test postgres-test
           pytest -vvs docs/source/tests/

.github/workflows/nightly-website.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          docker-compose run docs
+          docker compose run docs
       - name: Archive docs
         uses: actions/upload-artifact@v3
         with:

.github/workflows/packaging.yml

@@ -151,7 +151,7 @@ jobs:
       - name: Build and test
         run: |
           pushd adbc
-          docker-compose run \
+          docker compose run \
             -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \
             docs
           popd
@@ -199,7 +199,7 @@ jobs:
       - name: Build and test
         run: |
           pushd adbc/
-          docker-compose run java-dist
+          docker compose run java-dist
           popd
           cp -a adbc/dist/ ./
 
@@ -383,7 +383,7 @@ jobs:
           ARCH_CONDA_FORGE: ${{ matrix.arch }}
         run: |
           pushd adbc
-          docker-compose run \
+          docker compose run \
             -e HOST_USER_ID=$(id -u) \
             python-conda
           popd
@@ -402,7 +402,7 @@ jobs:
           ARCH_CONDA_FORGE: ${{ matrix.arch }}
         run: |
           pushd adbc
-          docker-compose run \
+          docker compose run \
             python-conda-test
           popd
 
@@ -527,7 +527,7 @@ jobs:
           MANYLINUX: ${{ matrix.manylinux_version }}
         run: |
           pushd adbc
-          docker-compose run \
+          docker compose run \
             -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \
             python-wheel-manylinux
           popd
@@ -550,9 +550,9 @@ jobs:
           MANYLINUX: ${{ matrix.manylinux_version }}
         run: |
           pushd adbc
-          env PYTHON=3.9 docker-compose run python-wheel-manylinux-test
-          env PYTHON=3.10 docker-compose run python-wheel-manylinux-test
-          env PYTHON=3.11 docker-compose run python-wheel-manylinux-test
+          env PYTHON=3.9 docker compose run python-wheel-manylinux-test
+          env PYTHON=3.10 docker compose run python-wheel-manylinux-test
+          env PYTHON=3.11 docker compose run python-wheel-manylinux-test
 
   python-macos:
     name: "Python ${{ matrix.arch }} macOS"
@@ -807,7 +807,7 @@ jobs:
       - name: Build sdist
         run: |
           pushd adbc
-          docker-compose run  \
+          docker compose run  \
             -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \
             python-sdist
           popd
@@ -827,7 +827,7 @@ jobs:
       - name: Test sdist
         run: |
           pushd adbc
-          docker-compose run python-sdist-test
+          docker compose run python-sdist-test
           popd
 
   release:

CONTRIBUTING.md

@@ -54,6 +54,27 @@ develop and support Conda users.
 [conda]: https://docs.conda.io/en/latest/
 [mambaforge]: https://mamba.readthedocs.io/en/latest/installation.html
 
+### Running Integration Tests
+
+Many of the test suites need to run against external services.  For example,
+the PostgreSQL driver needs to test against a running database!  This can be
+done by setting environment variables to tell tests where the services they
+need can be located.
+
+To standardize the configuration of these services, we use a Docker Compose
+file and a dotenv file.  Services can be started with Docker Compose:
+
+```shell
+$ docker compose up --detach --wait postgres-test
+```
+
+Then, source the .env file at the root of the repo to set the environment
+variables before running tests:
+
+```shell
+$ source .env
+```
+
 ### C/C++
 
 All libraries here contained within one CMake project. To build any

java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java

@@ -28,6 +28,7 @@
 import org.apache.arrow.adbc.core.AdbcException;
 import org.apache.arrow.adbc.driver.jdbc.JdbcDriver;
 import org.apache.arrow.adbc.driver.jdbc.StandardJdbcQuirks;
+import org.apache.arrow.adbc.driver.testsuite.SqlTestUtil;
 import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.types.TimeUnit;
@@ -42,6 +43,11 @@ static String makeJdbcUrl() {
     final String url = System.getenv(URL_ENV_VAR);
     final String user = System.getenv(USER_ENV_VAR);
     final String password = System.getenv(PASSWORD_ENV_VAR);
+
+    if (SqlTestUtil.isCI() && (url == null || url.isEmpty())) {
+      throw new RuntimeException("SQL Server not found, set " + URL_ENV_VAR);
+    }
+
     Assumptions.assumeFalse(url == null, "Microsoft SQL Server not found, set " + URL_ENV_VAR);
     Assumptions.assumeFalse(url.isEmpty(), "Microsoft SQL Server not found, set " + URL_ENV_VAR);
     return String.format(

java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java

@@ -28,6 +28,7 @@
 import org.apache.arrow.adbc.core.AdbcException;
 import org.apache.arrow.adbc.driver.jdbc.JdbcDriver;
 import org.apache.arrow.adbc.driver.jdbc.StandardJdbcQuirks;
+import org.apache.arrow.adbc.driver.testsuite.SqlTestUtil;
 import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.types.TimeUnit;
@@ -45,6 +46,11 @@ static String makeJdbcUrl() {
     final String postgresUrl = System.getenv(POSTGRESQL_URL_ENV_VAR);
     final String user = System.getenv(POSTGRESQL_USER_ENV_VAR);
     final String password = System.getenv(POSTGRESQL_PASSWORD_ENV_VAR);
+
+    if (SqlTestUtil.isCI() && (postgresUrl == null || postgresUrl.isEmpty())) {
+      throw new RuntimeException("PostgreSQL not found, set " + POSTGRESQL_URL_ENV_VAR);
+    }
+
     Assumptions.assumeFalse(
         postgresUrl == null, "PostgreSQL not found, set " + POSTGRESQL_URL_ENV_VAR);
     Assumptions.assumeFalse(
@@ -61,6 +67,10 @@ public AdbcDatabase initDatabase(BufferAllocator allocator) throws AdbcException
     String url = makeJdbcUrl();
 
     final String catalog = System.getenv(POSTGRESQL_DATABASE_ENV_VAR);
+    if (SqlTestUtil.isCI() && catalog == null) {
+      throw new RuntimeException(
+          "PostgreSQL catalog not found, set " + POSTGRESQL_DATABASE_ENV_VAR);
+    }
     Assumptions.assumeFalse(
         catalog == null, "PostgreSQL catalog not found, set " + POSTGRESQL_DATABASE_ENV_VAR);
     this.catalog = catalog;

java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/SqlTestUtil.java

@@ -39,6 +39,12 @@ public SqlTestUtil(SqlValidationQuirks quirks) {
     this.quirks = quirks;
   }
 
+  /** Check if we are running in the Arrow CI. */
+  public static boolean isCI() {
+    // Set by GitHub Actions
+    return "true".equals(System.getenv("CI"));
+  }
+
   /** Load a simple table with two columns. */
   public Schema ingestTableIntsStrs(
       BufferAllocator allocator, AdbcConnection connection, String tableName) throws Exception {