From 2e7cfc3bd84795d7976e5358f5fd95185846f80b Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 1 Mar 2024 12:06:23 -0500 Subject: [PATCH] ci: configure env vars for CI in .env file Fixes #1545. --- .env | 15 ++++++ .github/workflows/integration.yml | 54 ++++--------------- .github/workflows/java.yml | 11 +--- .github/workflows/native-unix.yml | 10 ++-- .github/workflows/nightly-website.yml | 2 +- .github/workflows/packaging.yml | 20 +++---- CONTRIBUTING.md | 21 ++++++++ .../jdbc/mssqlserver/MsSqlServerQuirks.java | 6 +++ .../jdbc/postgresql/PostgresqlQuirks.java | 6 +++ .../adbc/driver/testsuite/SqlTestUtil.java | 6 +++ 10 files changed, 82 insertions(+), 69 deletions(-) diff --git a/.env b/.env index 90ff405b13..f7e896a161 100644 --- a/.env +++ b/.env @@ -41,3 +41,18 @@ DOTNET=7.0 # When updating, also update the docs, which list the version of libpq/SQLite # that vcpkg (and hence our wheels) ship VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1" + +# These are used to tell tests where to find services for integration testing. +# They are valid if the services are started with the docker-compose config. +ADBC_DREMIO_FLIGHTSQL_PASS=dremio123 +ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010 +ADBC_DREMIO_FLIGHTSQL_USER=dremio +ADBC_JDBC_MSSQL_URL=localhost:1433 +ADBC_JDBC_MSSQL_USER=SA +ADBC_JDBC_MSSQL_PASSWORD=Password1! +ADBC_JDBC_POSTGRESQL_URL=localhost:5432/postgres +ADBC_JDBC_POSTGRESQL_USER=postgres +ADBC_JDBC_POSTGRESQL_PASSWORD=password +ADBC_POSTGRESQL_TEST_URI=postgresql://localhost:5432/postgres?user=postgres&password=password +ADBC_SQLITE_FLIGHTSQL_URI=grpc+tcp://localhost:8080 +ADBC_TEST_FLIGHTSQL_URI=grpc+tcp://localhost:41414 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6b7745e350..ba7d7cb0c7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -138,52 +138,38 @@ jobs: - name: Start SQLite server and Dremio shell: bash -l {0} run: | - docker-compose up -d flightsql-test flightsql-sqlite-test dremio dremio-init + docker compose up --detach flightsql-test flightsql-sqlite-test dremio dremio-init + cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV - name: Build FlightSQL Driver shell: bash -l {0} env: BUILD_ALL: "0" BUILD_DRIVER_FLIGHTSQL: "1" + BUILD_DRIVER_MANAGER: "1" run: | ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" + ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" - name: Test FlightSQL Driver against Dremio and SQLite shell: bash -l {0} env: BUILD_ALL: "0" BUILD_DRIVER_FLIGHTSQL: "1" - ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010" - ADBC_DREMIO_FLIGHTSQL_USER: "dremio" - ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123" - ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080" - ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414" run: | ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" ./ci/scripts/cpp_test.sh "$(pwd)/build" - - name: Build Python Flight SQL driver - shell: bash -l {0} - env: - BUILD_ALL: "0" - BUILD_DRIVER_FLIGHTSQL: "1" - BUILD_DRIVER_MANAGER: "1" - run: | - ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" - name: Test Python Flight SQL driver against Dremio shell: bash -l {0} env: BUILD_ALL: "0" BUILD_DRIVER_FLIGHTSQL: "1" - ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010" - ADBC_DREMIO_FLIGHTSQL_USER: "dremio" - ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123" - ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414" PYTEST_ADDOPTS: "--error-for-skips" run: | ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" - name: Stop SQLite server and Dremio shell: bash -l {0} run: | - docker-compose down + docker compose down postgresql: name: "PostgreSQL Integration Tests" @@ -220,26 +206,23 @@ jobs: shell: bash -l {0} env: BUILD_ALL: "0" + BUILD_DRIVER_MANAGER: "1" BUILD_DRIVER_POSTGRESQL: "1" run: | ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" - - name: Build Python PostgreSQL Driver + ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" + - name: Configure Integration Env Vars shell: bash -l {0} - env: - BUILD_ALL: "0" - BUILD_DRIVER_MANAGER: "1" - BUILD_DRIVER_POSTGRESQL: "1" run: | - ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" + cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV + echo "ADBC_USE_ASAN=ON" >> $GITHUB_ENV + echo "ADBC_USE_UBSAN=ON" >> $GITHUB_ENV - name: Test PostgreSQL Driver - postgres 11 shell: bash -l {0} env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=11 docker compose up --wait --detach postgres-test @@ -251,9 +234,6 @@ jobs: env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=12 docker compose up --wait --detach postgres-test @@ -265,9 +245,6 @@ jobs: env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=13 docker compose up --wait --detach postgres-test @@ -279,9 +256,6 @@ jobs: env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=14 docker compose up --wait --detach postgres-test @@ -293,9 +267,6 @@ jobs: env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=15 docker compose up --wait --detach postgres-test @@ -307,9 +278,6 @@ jobs: env: BUILD_ALL: "0" BUILD_DRIVER_POSTGRESQL: "1" - ADBC_USE_ASAN: "ON" - ADBC_USE_UBSAN: "ON" - ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password" PYTEST_ADDOPTS: "--error-for-skips" run: | env POSTGRES_VERSION=16 docker compose up --wait --detach postgres-test diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 88609cba50..2959bfeada 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -57,16 +57,9 @@ jobs: - name: Start test services shell: bash -l {0} run: | - docker compose up -d --wait flightsql-sqlite-test mssql-test postgres-test + docker compose up --detach --wait flightsql-sqlite-test mssql-test postgres-test + cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV - name: Build/Test - env: - ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080" - ADBC_JDBC_MSSQL_URL: "localhost:1433" - ADBC_JDBC_MSSQL_USER: "SA" - ADBC_JDBC_MSSQL_PASSWORD: "Password1!" - ADBC_JDBC_POSTGRESQL_URL: "localhost:5432/postgres" - ADBC_JDBC_POSTGRESQL_USER: "postgres" - ADBC_JDBC_POSTGRESQL_PASSWORD: "password" run: | cd java mvn install diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml index c42b56aac3..22ac5c4a91 100644 --- a/.github/workflows/native-unix.yml +++ b/.github/workflows/native-unix.yml @@ -572,18 +572,16 @@ jobs: retention-days: 2 path: | docs/build/html + - name: Configure Integration Env Vars + shell: bash -l {0} + run: | + cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV - name: Test Recipes (C++) shell: bash -l {0} run: | ./ci/scripts/cpp_recipe.sh $(pwd) ~/local build/recipe - name: Test Recipes (Python) shell: bash -l {0} - env: - ADBC_POSTGRESQL_TEST_URI: "postgres://localhost:5432/postgres?user=postgres&password=password" - ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010" - ADBC_DREMIO_FLIGHTSQL_USER: "dremio" - ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123" - ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080" run: | docker compose up --detach --wait dremio dremio-init flightsql-sqlite-test postgres-test pytest -vvs docs/source/tests/ diff --git a/.github/workflows/nightly-website.yml b/.github/workflows/nightly-website.yml index 428cfb73a5..9ddf3a644e 100644 --- a/.github/workflows/nightly-website.yml +++ b/.github/workflows/nightly-website.yml @@ -43,7 +43,7 @@ jobs: - name: Build shell: bash run: | - docker-compose run docs + docker compose run docs - name: Archive docs uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index ba1e65b05e..d5241a8f3e 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -151,7 +151,7 @@ jobs: - name: Build and test run: | pushd adbc - docker-compose run \ + docker compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \ docs popd @@ -199,7 +199,7 @@ jobs: - name: Build and test run: | pushd adbc/ - docker-compose run java-dist + docker compose run java-dist popd cp -a adbc/dist/ ./ @@ -383,7 +383,7 @@ jobs: ARCH_CONDA_FORGE: ${{ matrix.arch }} run: | pushd adbc - docker-compose run \ + docker compose run \ -e HOST_USER_ID=$(id -u) \ python-conda popd @@ -402,7 +402,7 @@ jobs: ARCH_CONDA_FORGE: ${{ matrix.arch }} run: | pushd adbc - docker-compose run \ + docker compose run \ python-conda-test popd @@ -527,7 +527,7 @@ jobs: MANYLINUX: ${{ matrix.manylinux_version }} run: | pushd adbc - docker-compose run \ + docker compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \ python-wheel-manylinux popd @@ -550,9 +550,9 @@ jobs: MANYLINUX: ${{ matrix.manylinux_version }} run: | pushd adbc - env PYTHON=3.9 docker-compose run python-wheel-manylinux-test - env PYTHON=3.10 docker-compose run python-wheel-manylinux-test - env PYTHON=3.11 docker-compose run python-wheel-manylinux-test + env PYTHON=3.9 docker compose run python-wheel-manylinux-test + env PYTHON=3.10 docker compose run python-wheel-manylinux-test + env PYTHON=3.11 docker compose run python-wheel-manylinux-test python-macos: name: "Python ${{ matrix.arch }} macOS" @@ -807,7 +807,7 @@ jobs: - name: Build sdist run: | pushd adbc - docker-compose run \ + docker compose run \ -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ steps.version.outputs.VERSION }} \ python-sdist popd @@ -827,7 +827,7 @@ jobs: - name: Test sdist run: | pushd adbc - docker-compose run python-sdist-test + docker compose run python-sdist-test popd release: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa515f6036..c1cd405e5d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,27 @@ develop and support Conda users. [conda]: https://docs.conda.io/en/latest/ [mambaforge]: https://mamba.readthedocs.io/en/latest/installation.html +### Running Integration Tests + +Many of the test suites need to run against external services. For example, +the PostgreSQL driver needs to test against a running database! This can be +done by setting environment variables to tell tests where the services they +need can be located. + +To standardize the configuration of these services, we use a Docker Compose +file and a dotenv file. Services can be started with Docker Compose: + +```shell +$ docker compose up --detach --wait postgres-test +``` + +Then, source the .env file at the root of the repo to set the environment +variables before running tests: + +```shell +$ source .env +``` + ### C/C++ All libraries here contained within one CMake project. To build any diff --git a/java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java b/java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java index b0297ab60f..5781df3316 100644 --- a/java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java +++ b/java/driver/jdbc-validation-mssqlserver/src/test/java/org/apache/arrow/adbc/driver/jdbc/mssqlserver/MsSqlServerQuirks.java @@ -28,6 +28,7 @@ import org.apache.arrow.adbc.core.AdbcException; import org.apache.arrow.adbc.driver.jdbc.JdbcDriver; import org.apache.arrow.adbc.driver.jdbc.StandardJdbcQuirks; +import org.apache.arrow.adbc.driver.testsuite.SqlTestUtil; import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.TimeUnit; @@ -42,6 +43,11 @@ static String makeJdbcUrl() { final String url = System.getenv(URL_ENV_VAR); final String user = System.getenv(USER_ENV_VAR); final String password = System.getenv(PASSWORD_ENV_VAR); + + if (SqlTestUtil.isCI() && (url == null || url.isEmpty())) { + throw new RuntimeException("SQL Server not found, set " + URL_ENV_VAR); + } + Assumptions.assumeFalse(url == null, "Microsoft SQL Server not found, set " + URL_ENV_VAR); Assumptions.assumeFalse(url.isEmpty(), "Microsoft SQL Server not found, set " + URL_ENV_VAR); return String.format( diff --git a/java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java b/java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java index fce9ff134d..fa75b4d67f 100644 --- a/java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java +++ b/java/driver/jdbc-validation-postgresql/src/test/java/org/apache/arrow/adbc/driver/jdbc/postgresql/PostgresqlQuirks.java @@ -28,6 +28,7 @@ import org.apache.arrow.adbc.core.AdbcException; import org.apache.arrow.adbc.driver.jdbc.JdbcDriver; import org.apache.arrow.adbc.driver.jdbc.StandardJdbcQuirks; +import org.apache.arrow.adbc.driver.testsuite.SqlTestUtil; import org.apache.arrow.adbc.driver.testsuite.SqlValidationQuirks; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.TimeUnit; @@ -45,6 +46,11 @@ static String makeJdbcUrl() { final String postgresUrl = System.getenv(POSTGRESQL_URL_ENV_VAR); final String user = System.getenv(POSTGRESQL_USER_ENV_VAR); final String password = System.getenv(POSTGRESQL_PASSWORD_ENV_VAR); + + if (SqlTestUtil.isCI() && (postgresUrl == null || postgresUrl.isEmpty())) { + throw new RuntimeException("PostgreSQL not found, set " + POSTGRESQL_URL_ENV_VAR); + } + Assumptions.assumeFalse( postgresUrl == null, "PostgreSQL not found, set " + POSTGRESQL_URL_ENV_VAR); Assumptions.assumeFalse( diff --git a/java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/SqlTestUtil.java b/java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/SqlTestUtil.java index c0536e5cf5..c872774fa4 100644 --- a/java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/SqlTestUtil.java +++ b/java/driver/validation/src/main/java/org/apache/arrow/adbc/driver/testsuite/SqlTestUtil.java @@ -39,6 +39,12 @@ public SqlTestUtil(SqlValidationQuirks quirks) { this.quirks = quirks; } + /** Check if we are running in the Arrow CI. */ + public static boolean isCI() { + // Set by GitHub Actions + return "true".equals(System.getenv("CI")); + } + /** Load a simple table with two columns. */ public Schema ingestTableIntsStrs( BufferAllocator allocator, AdbcConnection connection, String tableName) throws Exception {