From a5a5730450ac8db96fcbc13e2e7ed954cc2b60ec Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 14:06:22 -0500 Subject: [PATCH 1/6] fix data type parsing --- csharp/src/Drivers/BigQuery/AssemblyInfo.cs | 20 +++++++++ .../Drivers/BigQuery/BigQueryConnection.cs | 25 ++++++++--- .../Drivers/BigQuery/BigQueryTableTypes.cs | 25 +++++++++++ .../Metadata/GetObjectsParser.cs | 11 +++++ csharp/test/Drivers/BigQuery/DriverTests.cs | 42 ++++++++++++++++--- 5 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 csharp/src/Drivers/BigQuery/AssemblyInfo.cs create mode 100644 csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs diff --git a/csharp/src/Drivers/BigQuery/AssemblyInfo.cs b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs new file mode 100644 index 0000000000..e6245e0e96 --- /dev/null +++ b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs @@ -0,0 +1,20 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 398365314f..0b9931561d 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -530,11 +530,15 @@ private StructArray GetColumnSchema( ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); remarksBuilder.Append(""); - string dataType = ToTypeName(GetValue(row["data_type"])); + string dataType = ToTypeName(GetValue(row["data_type"]), out string suffix); - if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) + if ((dataType.StartsWith("NUMERIC") || + dataType.StartsWith("DECIMAL") || + dataType.StartsWith("BIGNUMERIC") || + dataType.StartsWith("BIGDECIMAL")) + && !string.IsNullOrEmpty(suffix)) { - ParsedDecimalValues values = ParsePrecisionAndScale(dataType); + ParsedDecimalValues values = ParsePrecisionAndScale(suffix); xdbcColumnSizeBuilder.Append(values.Precision); xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); } @@ -752,10 +756,19 @@ private string PatternToRegEx(string? pattern) return builder.ToString(); } - private string ToTypeName(string type) + private string ToTypeName(string type, out string suffix) { - int index = Math.Min(type.IndexOf("("), type.IndexOf("<")); + suffix = string.Empty; + + int index = type.IndexOf("("); + if (index == -1) + index = type.IndexOf("<"); + string dataType = index == -1 ? type : type.Substring(0, index); + + if (index > -1) + suffix = type.Substring(dataType.Length); + return dataType; } @@ -965,7 +978,7 @@ private ParsedDecimalValues ParsePrecisionAndScale(string type) public override IArrowArrayStream GetTableTypes() { StringArray.Builder tableTypesBuilder = new StringArray.Builder(); - tableTypesBuilder.AppendRange(new string[] { "BASE TABLE", "VIEW" }); + tableTypesBuilder.AppendRange(BigQueryTableTypes.TableTypes); IArrowArray[] dataArrays = new IArrowArray[] { diff --git a/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs new file mode 100644 index 0000000000..6834c64d81 --- /dev/null +++ b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.BigQuery +{ + internal static class BigQueryTableTypes + { + public static List TableTypes = new List { "BASE TABLE", "VIEW", "CLONE", "SNAPSHOT" }; + } +} diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs index 6b10f4cde3..0b0649d57b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs @@ -156,6 +156,17 @@ public static List ParseCatalog(RecordBatch recordBatch, string? sc { if (constraintsArray == null) return null; + // constraint details may not be loaded correctly if the depth wasn't Columns + try + { + if (constraintsArray.Fields.Count == 0) + return null; + } + catch (NullReferenceException) + { + return null; + } + List constraints = new List(); StringArray name = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8 diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 9c587c1ebd..32fabaf3d6 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; @@ -116,7 +117,7 @@ public void CanGetObjects() catalogPattern: catalogName, dbSchemaPattern: schemaName, tableNamePattern: tableName, - tableTypes: new List { "BASE TABLE", "VIEW" }, + tableTypes: BigQueryTableTypes.TableTypes, columnNamePattern: columnName); RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; @@ -134,6 +135,40 @@ public void CanGetObjects() Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns?.Count); } + [SkippableFact, Order(3)] + public void CanGetObjectsTables() + { + string? catalogName = _testConfiguration.Metadata.Catalog; + string? schemaName = _testConfiguration.Metadata.Schema; + string? tableName = _testConfiguration.Metadata.Table; + + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); + + IArrowArrayStream stream = adbcConnection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Tables, + catalogPattern: catalogName, + dbSchemaPattern: schemaName, + tableNamePattern: null, + tableTypes: BigQueryTableTypes.TableTypes, + columnNamePattern: null); + + RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? tables = catalogs + .Where(c => string.Equals(c.Name, catalogName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault(); + + AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); + Assert.True(table != null, "table should not be null"); + Assert.Equal("BASE TABLE", table.Type); + } + /// /// Validates if the driver can call GetTableSchema. /// @@ -167,10 +202,7 @@ public void CanGetTableTypes() StringArray stringArray = (StringArray)recordBatch.Column("table_type"); - List known_types = new List - { - "BASE TABLE", "VIEW" - }; + List known_types = BigQueryTableTypes.TableTypes; int results = 0; From cd23a211b3c7f3b8850f860ae76f85110a4b91f8 Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 15:00:13 -0500 Subject: [PATCH 2/6] Revert "test(c): don't use sketchy cast to test backwards compatibility (#2425)" This reverts commit 63bb903b7ddc7730beaa3d092dc5808632cd4b08. --- .github/workflows/nightly-verify.yml | 5 --- c/driver/flightsql/sqlite_flightsql_test.cc | 16 ++++---- c/driver/postgresql/postgresql_test.cc | 45 ++++++++++----------- c/validation/adbc_validation_statement.cc | 24 +++++------ c/vendor/nanoarrow/nanoarrow.hpp | 7 ---- ci/docker/cpp-clang-latest.dockerfile | 12 +++--- ci/docker/cpp-gcc-latest.dockerfile | 34 ---------------- docker-compose.yml | 15 +------ 8 files changed, 46 insertions(+), 112 deletions(-) delete mode 100644 ci/docker/cpp-gcc-latest.dockerfile diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index 2d4652585d..3d27ef1646 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -190,11 +190,6 @@ jobs: pushd arrow-adbc docker compose run --rm cpp-clang-latest - - name: cpp-gcc-latest - run: | - pushd arrow-adbc - docker compose run --rm cpp-gcc-latest - - name: python-debug run: | pushd arrow-adbc diff --git a/c/driver/flightsql/sqlite_flightsql_test.cc b/c/driver/flightsql/sqlite_flightsql_test.cc index 752ce8154f..4797d58e77 100644 --- a/c/driver/flightsql/sqlite_flightsql_test.cc +++ b/c/driver/flightsql/sqlite_flightsql_test.cc @@ -235,20 +235,18 @@ TEST_F(SqliteFlightSqlTest, TestGarbageInput) { ASSERT_THAT(AdbcDatabaseRelease(&database, &error), IsOkStatus(&error)); } -int Canary(const struct AdbcError*) { return 0; } - TEST_F(SqliteFlightSqlTest, AdbcDriverBackwardsCompatibility) { - struct AdbcDriver driver; - std::memset(&driver, 0, ADBC_DRIVER_1_1_0_SIZE); - driver.ErrorGetDetailCount = Canary; + // XXX: sketchy cast + auto* driver = static_cast(malloc(ADBC_DRIVER_1_0_0_SIZE)); + std::memset(driver, 0, ADBC_DRIVER_1_0_0_SIZE); - ASSERT_THAT(::FlightSQLDriverInit(ADBC_VERSION_1_0_0, &driver, &error), + ASSERT_THAT(::FlightSQLDriverInit(ADBC_VERSION_1_0_0, driver, &error), IsOkStatus(&error)); - ASSERT_EQ(Canary, driver.ErrorGetDetailCount); - - ASSERT_THAT(::FlightSQLDriverInit(424242, &driver, &error), + ASSERT_THAT(::FlightSQLDriverInit(424242, driver, &error), adbc_validation::IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); + + free(driver); } class SqliteFlightSqlConnectionTest : public ::testing::Test, diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc index b76ba1cf7d..be32bd893b 100644 --- a/c/driver/postgresql/postgresql_test.cc +++ b/c/driver/postgresql/postgresql_test.cc @@ -223,20 +223,18 @@ class PostgresDatabaseTest : public ::testing::Test, }; ADBCV_TEST_DATABASE(PostgresDatabaseTest) -int Canary(const struct AdbcError*) { return 0; } - TEST_F(PostgresDatabaseTest, AdbcDriverBackwardsCompatibility) { - struct AdbcDriver driver; - std::memset(&driver, 0, ADBC_DRIVER_1_1_0_SIZE); - driver.ErrorGetDetailCount = Canary; + // XXX: sketchy cast + auto* driver = static_cast(malloc(ADBC_DRIVER_1_0_0_SIZE)); + std::memset(driver, 0, ADBC_DRIVER_1_0_0_SIZE); - ASSERT_THAT(::PostgresqlDriverInit(ADBC_VERSION_1_0_0, &driver, &error), + ASSERT_THAT(::PostgresqlDriverInit(ADBC_VERSION_1_0_0, driver, &error), IsOkStatus(&error)); - ASSERT_EQ(Canary, driver.ErrorGetDetailCount); - - ASSERT_THAT(::PostgresqlDriverInit(424242, &driver, &error), + ASSERT_THAT(::PostgresqlDriverInit(424242, driver, &error), IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); + + free(driver); } class PostgresConnectionTest : public ::testing::Test, @@ -1554,25 +1552,24 @@ TEST_F(PostgresStatementTest, BatchSizeHint) { // Test that an ADBC 1.0.0-sized error still works TEST_F(PostgresStatementTest, AdbcErrorBackwardsCompatibility) { - struct AdbcError error; - std::memset(&error, 0, ADBC_ERROR_1_1_0_SIZE); - struct AdbcDriver canary; - error.private_data = &canary; - error.private_driver = &canary; + // XXX: sketchy cast + auto* error = static_cast(malloc(ADBC_ERROR_1_0_0_SIZE)); + std::memset(error, 0, ADBC_ERROR_1_0_0_SIZE); - ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error)); + ASSERT_THAT(AdbcStatementNew(&connection, &statement, error), IsOkStatus(error)); ASSERT_THAT( - AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", &error), - IsOkStatus(&error)); + AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", error), + IsOkStatus(error)); adbc_validation::StreamReader reader; ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, - &reader.rows_affected, &error), - IsStatus(ADBC_STATUS_NOT_FOUND, &error)); - ASSERT_EQ("42P01", std::string_view(error.sqlstate, 5)); - ASSERT_EQ(0, AdbcErrorGetDetailCount(&error)); - ASSERT_EQ(&canary, error.private_data); - ASSERT_EQ(&canary, error.private_driver); - error.release(&error); + &reader.rows_affected, error), + IsStatus(ADBC_STATUS_NOT_FOUND, error)); + + ASSERT_EQ("42P01", std::string_view(error->sqlstate, 5)); + ASSERT_EQ(0, AdbcErrorGetDetailCount(error)); + + error->release(error); + free(error); } TEST_F(PostgresStatementTest, Cancel) { diff --git a/c/validation/adbc_validation_statement.cc b/c/validation/adbc_validation_statement.cc index 8c6dd94846..cd388623ba 100644 --- a/c/validation/adbc_validation_statement.cc +++ b/c/validation/adbc_validation_statement.cc @@ -2817,23 +2817,21 @@ struct ADBC_EXPORT AdbcError100 { // Test that an ADBC 1.0.0-sized error still works void StatementTest::TestErrorCompatibility() { static_assert(sizeof(AdbcError100) == ADBC_ERROR_1_0_0_SIZE, "Wrong size"); - struct AdbcError error; - std::memset(&error, 0, ADBC_ERROR_1_1_0_SIZE); - struct AdbcDriver canary; - error.private_data = &canary; - error.private_driver = &canary; + // XXX: sketchy cast + auto* error = reinterpret_cast(malloc(ADBC_ERROR_1_0_0_SIZE)); + std::memset(error, 0, ADBC_ERROR_1_0_0_SIZE); - ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error)); + ASSERT_THAT(AdbcStatementNew(&connection, &statement, error), IsOkStatus(error)); ASSERT_THAT( - AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", &error), - IsOkStatus(&error)); + AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", error), + IsOkStatus(error)); adbc_validation::StreamReader reader; ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, - &reader.rows_affected, &error), - ::testing::Not(IsOkStatus(&error))); - ASSERT_EQ(&canary, error.private_data); - ASSERT_EQ(&canary, error.private_driver); - error.release(&error); + &reader.rows_affected, error), + ::testing::Not(IsOkStatus(error))); + auto* old_error = reinterpret_cast(error); + old_error->release(old_error); + free(error); } void StatementTest::TestResultInvalidation() { diff --git a/c/vendor/nanoarrow/nanoarrow.hpp b/c/vendor/nanoarrow/nanoarrow.hpp index f2eade3d8e..16c2e55b9f 100644 --- a/c/vendor/nanoarrow/nanoarrow.hpp +++ b/c/vendor/nanoarrow/nanoarrow.hpp @@ -92,16 +92,9 @@ namespace literals { /// @{ /// \brief User literal operator allowing ArrowStringView construction like "str"_asv -#if !defined(__clang__) && (defined(__GNUC__) && __GNUC__ < 6) inline ArrowStringView operator"" _asv(const char* data, std::size_t size_bytes) { return {data, static_cast(size_bytes)}; } -#else -inline ArrowStringView operator""_asv(const char* data, std::size_t size_bytes) { - return {data, static_cast(size_bytes)}; -} -#endif -// N.B. older GCC requires the space above, newer Clang forbids the space // @} diff --git a/ci/docker/cpp-clang-latest.dockerfile b/ci/docker/cpp-clang-latest.dockerfile index a1b76a552e..ff6e161e37 100644 --- a/ci/docker/cpp-clang-latest.dockerfile +++ b/ci/docker/cpp-clang-latest.dockerfile @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +ARG VCPKG + FROM debian:12 -ARG GO RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y && \ @@ -33,9 +34,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y cmake git libpq-dev libsqlite3-dev pkg-config -RUN curl -L -o go.tar.gz https://go.dev/dl/go${GO}.linux-amd64.tar.gz && \ - tar -C /opt -xvf go.tar.gz - -ENV PATH=/opt/go/bin:$PATH \ - CC=/usr/bin/clang \ - CXX=/usr/bin/clang++ +RUN curl -L -o go.tar.gz https://go.dev/dl/go1.22.5.linux-amd64.tar.gz && \ + tar -C /opt -xvf go.tar.gz && \ + echo 'export PATH=$PATH:/opt/go/bin' | tee -a ~/.bashrc diff --git a/ci/docker/cpp-gcc-latest.dockerfile b/ci/docker/cpp-gcc-latest.dockerfile deleted file mode 100644 index 38a7781ef9..0000000000 --- a/ci/docker/cpp-gcc-latest.dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -FROM amd64/debian:experimental -ARG GCC -ARG GO - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update -y && \ - apt-get install -y -q cmake curl git gnupg libpq-dev libsqlite3-dev pkg-config && \ - apt-get install -y -q -t experimental g++-${GCC} gcc-${GCC} && \ - apt-get clean - -RUN curl -L -o go.tar.gz https://go.dev/dl/go${GO}.linux-amd64.tar.gz && \ - tar -C /opt -xvf go.tar.gz - -ENV PATH=/opt/go/bin:$PATH \ - CC=/usr/bin/gcc-${GCC} \ - CXX=/usr/bin/g++-${GCC} diff --git a/docker-compose.yml b/docker-compose.yml index 1872a2cfc1..bf961cdb84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,21 +38,10 @@ services: context: . dockerfile: ci/docker/cpp-clang-latest.dockerfile args: - GO: ${GO} - volumes: - - .:/adbc:delegated - command: "bash -c 'git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build/clang-latest && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build/clang-latest'" - - cpp-gcc-latest: - build: - context: . - dockerfile: ci/docker/cpp-gcc-latest.dockerfile - args: - GCC: 15 - GO: ${GO} + VCPKG: ${VCPKG} volumes: - .:/adbc:delegated - command: "bash -c 'git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build/gcc-latest && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build/gcc-latest'" + command: "bash -c 'export PATH=$PATH:/opt/go/bin CC=$(which clang) CXX=$(which clang++) && git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build'" ############################ Documentation ################################### From b92d6f5fe134a46dbbe18f6389442b4b5dfac17a Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 15:02:45 -0500 Subject: [PATCH 3/6] Revert "fix data type parsing" This reverts commit a5a5730450ac8db96fcbc13e2e7ed954cc2b60ec. --- csharp/src/Drivers/BigQuery/AssemblyInfo.cs | 20 --------- .../Drivers/BigQuery/BigQueryConnection.cs | 25 +++-------- .../Drivers/BigQuery/BigQueryTableTypes.cs | 25 ----------- .../Metadata/GetObjectsParser.cs | 11 ----- csharp/test/Drivers/BigQuery/DriverTests.cs | 42 +++---------------- 5 files changed, 11 insertions(+), 112 deletions(-) delete mode 100644 csharp/src/Drivers/BigQuery/AssemblyInfo.cs delete mode 100644 csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs diff --git a/csharp/src/Drivers/BigQuery/AssemblyInfo.cs b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs deleted file mode 100644 index e6245e0e96..0000000000 --- a/csharp/src/Drivers/BigQuery/AssemblyInfo.cs +++ /dev/null @@ -1,20 +0,0 @@ -/* -* Licensed to the Apache Software Foundation (ASF) under one or more -* contributor license agreements. See the NOTICE file distributed with -* this work for additional information regarding copyright ownership. -* The ASF licenses this file to You under the Apache License, Version 2.0 -* (the "License"); you may not use this file except in compliance with -* the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 0b9931561d..398365314f 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -530,15 +530,11 @@ private StructArray GetColumnSchema( ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); remarksBuilder.Append(""); - string dataType = ToTypeName(GetValue(row["data_type"]), out string suffix); + string dataType = ToTypeName(GetValue(row["data_type"])); - if ((dataType.StartsWith("NUMERIC") || - dataType.StartsWith("DECIMAL") || - dataType.StartsWith("BIGNUMERIC") || - dataType.StartsWith("BIGDECIMAL")) - && !string.IsNullOrEmpty(suffix)) + if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) { - ParsedDecimalValues values = ParsePrecisionAndScale(suffix); + ParsedDecimalValues values = ParsePrecisionAndScale(dataType); xdbcColumnSizeBuilder.Append(values.Precision); xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); } @@ -756,19 +752,10 @@ private string PatternToRegEx(string? pattern) return builder.ToString(); } - private string ToTypeName(string type, out string suffix) + private string ToTypeName(string type) { - suffix = string.Empty; - - int index = type.IndexOf("("); - if (index == -1) - index = type.IndexOf("<"); - + int index = Math.Min(type.IndexOf("("), type.IndexOf("<")); string dataType = index == -1 ? type : type.Substring(0, index); - - if (index > -1) - suffix = type.Substring(dataType.Length); - return dataType; } @@ -978,7 +965,7 @@ private ParsedDecimalValues ParsePrecisionAndScale(string type) public override IArrowArrayStream GetTableTypes() { StringArray.Builder tableTypesBuilder = new StringArray.Builder(); - tableTypesBuilder.AppendRange(BigQueryTableTypes.TableTypes); + tableTypesBuilder.AppendRange(new string[] { "BASE TABLE", "VIEW" }); IArrowArray[] dataArrays = new IArrowArray[] { diff --git a/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs deleted file mode 100644 index 6834c64d81..0000000000 --- a/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -using System.Collections.Generic; - -namespace Apache.Arrow.Adbc.Drivers.BigQuery -{ - internal static class BigQueryTableTypes - { - public static List TableTypes = new List { "BASE TABLE", "VIEW", "CLONE", "SNAPSHOT" }; - } -} diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs index 0b0649d57b..6b10f4cde3 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs @@ -156,17 +156,6 @@ public static List ParseCatalog(RecordBatch recordBatch, string? sc { if (constraintsArray == null) return null; - // constraint details may not be loaded correctly if the depth wasn't Columns - try - { - if (constraintsArray.Fields.Count == 0) - return null; - } - catch (NullReferenceException) - { - return null; - } - List constraints = new List(); StringArray name = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8 diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 32fabaf3d6..9c587c1ebd 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,7 +18,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; @@ -117,7 +116,7 @@ public void CanGetObjects() catalogPattern: catalogName, dbSchemaPattern: schemaName, tableNamePattern: tableName, - tableTypes: BigQueryTableTypes.TableTypes, + tableTypes: new List { "BASE TABLE", "VIEW" }, columnNamePattern: columnName); RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; @@ -135,40 +134,6 @@ public void CanGetObjects() Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns?.Count); } - [SkippableFact, Order(3)] - public void CanGetObjectsTables() - { - string? catalogName = _testConfiguration.Metadata.Catalog; - string? schemaName = _testConfiguration.Metadata.Schema; - string? tableName = _testConfiguration.Metadata.Table; - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); - - IArrowArrayStream stream = adbcConnection.GetObjects( - depth: AdbcConnection.GetObjectsDepth.Tables, - catalogPattern: catalogName, - dbSchemaPattern: schemaName, - tableNamePattern: null, - tableTypes: BigQueryTableTypes.TableTypes, - columnNamePattern: null); - - RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; - - List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); - - List? tables = catalogs - .Where(c => string.Equals(c.Name, catalogName)) - .Select(c => c.DbSchemas) - .FirstOrDefault() - ?.Where(s => string.Equals(s.Name, schemaName)) - .Select(s => s.Tables) - .FirstOrDefault(); - - AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); - Assert.True(table != null, "table should not be null"); - Assert.Equal("BASE TABLE", table.Type); - } - /// /// Validates if the driver can call GetTableSchema. /// @@ -202,7 +167,10 @@ public void CanGetTableTypes() StringArray stringArray = (StringArray)recordBatch.Column("table_type"); - List known_types = BigQueryTableTypes.TableTypes; + List known_types = new List + { + "BASE TABLE", "VIEW" + }; int results = 0; From 060fc70dd698fd8f509ee0df6ec141fa654858e3 Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 15:10:14 -0500 Subject: [PATCH 4/6] Reapply "fix data type parsing" This reverts commit b92d6f5fe134a46dbbe18f6389442b4b5dfac17a. --- csharp/src/Drivers/BigQuery/AssemblyInfo.cs | 20 +++++++++ .../Drivers/BigQuery/BigQueryConnection.cs | 25 ++++++++--- .../Drivers/BigQuery/BigQueryTableTypes.cs | 25 +++++++++++ .../Metadata/GetObjectsParser.cs | 11 +++++ csharp/test/Drivers/BigQuery/DriverTests.cs | 42 ++++++++++++++++--- 5 files changed, 112 insertions(+), 11 deletions(-) create mode 100644 csharp/src/Drivers/BigQuery/AssemblyInfo.cs create mode 100644 csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs diff --git a/csharp/src/Drivers/BigQuery/AssemblyInfo.cs b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs new file mode 100644 index 0000000000..e6245e0e96 --- /dev/null +++ b/csharp/src/Drivers/BigQuery/AssemblyInfo.cs @@ -0,0 +1,20 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 398365314f..0b9931561d 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -530,11 +530,15 @@ private StructArray GetColumnSchema( ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); remarksBuilder.Append(""); - string dataType = ToTypeName(GetValue(row["data_type"])); + string dataType = ToTypeName(GetValue(row["data_type"]), out string suffix); - if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) + if ((dataType.StartsWith("NUMERIC") || + dataType.StartsWith("DECIMAL") || + dataType.StartsWith("BIGNUMERIC") || + dataType.StartsWith("BIGDECIMAL")) + && !string.IsNullOrEmpty(suffix)) { - ParsedDecimalValues values = ParsePrecisionAndScale(dataType); + ParsedDecimalValues values = ParsePrecisionAndScale(suffix); xdbcColumnSizeBuilder.Append(values.Precision); xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); } @@ -752,10 +756,19 @@ private string PatternToRegEx(string? pattern) return builder.ToString(); } - private string ToTypeName(string type) + private string ToTypeName(string type, out string suffix) { - int index = Math.Min(type.IndexOf("("), type.IndexOf("<")); + suffix = string.Empty; + + int index = type.IndexOf("("); + if (index == -1) + index = type.IndexOf("<"); + string dataType = index == -1 ? type : type.Substring(0, index); + + if (index > -1) + suffix = type.Substring(dataType.Length); + return dataType; } @@ -965,7 +978,7 @@ private ParsedDecimalValues ParsePrecisionAndScale(string type) public override IArrowArrayStream GetTableTypes() { StringArray.Builder tableTypesBuilder = new StringArray.Builder(); - tableTypesBuilder.AppendRange(new string[] { "BASE TABLE", "VIEW" }); + tableTypesBuilder.AppendRange(BigQueryTableTypes.TableTypes); IArrowArray[] dataArrays = new IArrowArray[] { diff --git a/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs new file mode 100644 index 0000000000..6834c64d81 --- /dev/null +++ b/csharp/src/Drivers/BigQuery/BigQueryTableTypes.cs @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.BigQuery +{ + internal static class BigQueryTableTypes + { + public static List TableTypes = new List { "BASE TABLE", "VIEW", "CLONE", "SNAPSHOT" }; + } +} diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs index 6b10f4cde3..0b0649d57b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Metadata/GetObjectsParser.cs @@ -156,6 +156,17 @@ public static List ParseCatalog(RecordBatch recordBatch, string? sc { if (constraintsArray == null) return null; + // constraint details may not be loaded correctly if the depth wasn't Columns + try + { + if (constraintsArray.Fields.Count == 0) + return null; + } + catch (NullReferenceException) + { + return null; + } + List constraints = new List(); StringArray name = (StringArray)constraintsArray.Fields[StandardSchemas.ConstraintSchema.FindIndexOrThrow("constraint_name")]; // constraint_name | utf8 diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 9c587c1ebd..32fabaf3d6 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; @@ -116,7 +117,7 @@ public void CanGetObjects() catalogPattern: catalogName, dbSchemaPattern: schemaName, tableNamePattern: tableName, - tableTypes: new List { "BASE TABLE", "VIEW" }, + tableTypes: BigQueryTableTypes.TableTypes, columnNamePattern: columnName); RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; @@ -134,6 +135,40 @@ public void CanGetObjects() Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns?.Count); } + [SkippableFact, Order(3)] + public void CanGetObjectsTables() + { + string? catalogName = _testConfiguration.Metadata.Catalog; + string? schemaName = _testConfiguration.Metadata.Schema; + string? tableName = _testConfiguration.Metadata.Table; + + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); + + IArrowArrayStream stream = adbcConnection.GetObjects( + depth: AdbcConnection.GetObjectsDepth.Tables, + catalogPattern: catalogName, + dbSchemaPattern: schemaName, + tableNamePattern: null, + tableTypes: BigQueryTableTypes.TableTypes, + columnNamePattern: null); + + RecordBatch recordBatch = stream.ReadNextRecordBatchAsync().Result; + + List catalogs = GetObjectsParser.ParseCatalog(recordBatch, schemaName); + + List? tables = catalogs + .Where(c => string.Equals(c.Name, catalogName)) + .Select(c => c.DbSchemas) + .FirstOrDefault() + ?.Where(s => string.Equals(s.Name, schemaName)) + .Select(s => s.Tables) + .FirstOrDefault(); + + AdbcTable? table = tables?.Where((table) => string.Equals(table.Name, tableName)).FirstOrDefault(); + Assert.True(table != null, "table should not be null"); + Assert.Equal("BASE TABLE", table.Type); + } + /// /// Validates if the driver can call GetTableSchema. /// @@ -167,10 +202,7 @@ public void CanGetTableTypes() StringArray stringArray = (StringArray)recordBatch.Column("table_type"); - List known_types = new List - { - "BASE TABLE", "VIEW" - }; + List known_types = BigQueryTableTypes.TableTypes; int results = 0; From b52cf43e62281bb097491d8807ddf34339438d63 Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 14:06:22 -0500 Subject: [PATCH 5/6] fix data type parsing From f6f4799314db6efc7a285f08ae71002bc04993fe Mon Sep 17 00:00:00 2001 From: David Coe <> Date: Fri, 10 Jan 2025 15:35:19 -0500 Subject: [PATCH 6/6] fix accidental revert --- .github/workflows/nightly-verify.yml | 7 ++- c/driver/flightsql/sqlite_flightsql_test.cc | 18 ++++---- c/driver/postgresql/postgresql_test.cc | 47 +++++++++++---------- c/validation/adbc_validation_statement.cc | 26 ++++++------ c/vendor/nanoarrow/nanoarrow.hpp | 9 +++- ci/docker/cpp-clang-latest.dockerfile | 12 +++--- ci/docker/cpp-gcc-latest.dockerfile | 34 +++++++++++++++ docker-compose.yml | 17 ++++++-- 8 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 ci/docker/cpp-gcc-latest.dockerfile diff --git a/.github/workflows/nightly-verify.yml b/.github/workflows/nightly-verify.yml index 3d27ef1646..d6fdee13c2 100644 --- a/.github/workflows/nightly-verify.yml +++ b/.github/workflows/nightly-verify.yml @@ -190,7 +190,12 @@ jobs: pushd arrow-adbc docker compose run --rm cpp-clang-latest + - name: cpp-gcc-latest + run: | + pushd arrow-adbc + docker compose run --rm cpp-gcc-latest + - name: python-debug run: | pushd arrow-adbc - docker compose run -e PYTHON=3.12 --rm python-debug + docker compose run -e PYTHON=3.12 --rm python-debug \ No newline at end of file diff --git a/c/driver/flightsql/sqlite_flightsql_test.cc b/c/driver/flightsql/sqlite_flightsql_test.cc index 4797d58e77..2f72c19ec8 100644 --- a/c/driver/flightsql/sqlite_flightsql_test.cc +++ b/c/driver/flightsql/sqlite_flightsql_test.cc @@ -235,18 +235,20 @@ TEST_F(SqliteFlightSqlTest, TestGarbageInput) { ASSERT_THAT(AdbcDatabaseRelease(&database, &error), IsOkStatus(&error)); } +int Canary(const struct AdbcError*) { return 0; } + TEST_F(SqliteFlightSqlTest, AdbcDriverBackwardsCompatibility) { - // XXX: sketchy cast - auto* driver = static_cast(malloc(ADBC_DRIVER_1_0_0_SIZE)); - std::memset(driver, 0, ADBC_DRIVER_1_0_0_SIZE); + struct AdbcDriver driver; + std::memset(&driver, 0, ADBC_DRIVER_1_1_0_SIZE); + driver.ErrorGetDetailCount = Canary; - ASSERT_THAT(::FlightSQLDriverInit(ADBC_VERSION_1_0_0, driver, &error), + ASSERT_THAT(::FlightSQLDriverInit(ADBC_VERSION_1_0_0, &driver, &error), IsOkStatus(&error)); - ASSERT_THAT(::FlightSQLDriverInit(424242, driver, &error), - adbc_validation::IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); + ASSERT_EQ(Canary, driver.ErrorGetDetailCount); - free(driver); + ASSERT_THAT(::FlightSQLDriverInit(424242, &driver, &error), + adbc_validation::IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); } class SqliteFlightSqlConnectionTest : public ::testing::Test, @@ -427,4 +429,4 @@ INSERT INTO foo(a) VALUES ('foo');)", ASSERT_NE(0, detail.value_length); EXPECT_STREQ("grpc-status-details-bin", detail.key); } -} +} \ No newline at end of file diff --git a/c/driver/postgresql/postgresql_test.cc b/c/driver/postgresql/postgresql_test.cc index be32bd893b..0379eafd08 100644 --- a/c/driver/postgresql/postgresql_test.cc +++ b/c/driver/postgresql/postgresql_test.cc @@ -223,18 +223,20 @@ class PostgresDatabaseTest : public ::testing::Test, }; ADBCV_TEST_DATABASE(PostgresDatabaseTest) +int Canary(const struct AdbcError*) { return 0; } + TEST_F(PostgresDatabaseTest, AdbcDriverBackwardsCompatibility) { - // XXX: sketchy cast - auto* driver = static_cast(malloc(ADBC_DRIVER_1_0_0_SIZE)); - std::memset(driver, 0, ADBC_DRIVER_1_0_0_SIZE); + struct AdbcDriver driver; + std::memset(&driver, 0, ADBC_DRIVER_1_1_0_SIZE); + driver.ErrorGetDetailCount = Canary; - ASSERT_THAT(::PostgresqlDriverInit(ADBC_VERSION_1_0_0, driver, &error), + ASSERT_THAT(::PostgresqlDriverInit(ADBC_VERSION_1_0_0, &driver, &error), IsOkStatus(&error)); - ASSERT_THAT(::PostgresqlDriverInit(424242, driver, &error), - IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); + ASSERT_EQ(Canary, driver.ErrorGetDetailCount); - free(driver); + ASSERT_THAT(::PostgresqlDriverInit(424242, &driver, &error), + IsStatus(ADBC_STATUS_NOT_IMPLEMENTED, &error)); } class PostgresConnectionTest : public ::testing::Test, @@ -1552,24 +1554,25 @@ TEST_F(PostgresStatementTest, BatchSizeHint) { // Test that an ADBC 1.0.0-sized error still works TEST_F(PostgresStatementTest, AdbcErrorBackwardsCompatibility) { - // XXX: sketchy cast - auto* error = static_cast(malloc(ADBC_ERROR_1_0_0_SIZE)); - std::memset(error, 0, ADBC_ERROR_1_0_0_SIZE); + struct AdbcError error; + std::memset(&error, 0, ADBC_ERROR_1_1_0_SIZE); + struct AdbcDriver canary; + error.private_data = &canary; + error.private_driver = &canary; - ASSERT_THAT(AdbcStatementNew(&connection, &statement, error), IsOkStatus(error)); + ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error)); ASSERT_THAT( - AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", error), - IsOkStatus(error)); + AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", &error), + IsOkStatus(&error)); adbc_validation::StreamReader reader; ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, - &reader.rows_affected, error), - IsStatus(ADBC_STATUS_NOT_FOUND, error)); - - ASSERT_EQ("42P01", std::string_view(error->sqlstate, 5)); - ASSERT_EQ(0, AdbcErrorGetDetailCount(error)); - - error->release(error); - free(error); + &reader.rows_affected, &error), + IsStatus(ADBC_STATUS_NOT_FOUND, &error)); + ASSERT_EQ("42P01", std::string_view(error.sqlstate, 5)); + ASSERT_EQ(0, AdbcErrorGetDetailCount(&error)); + ASSERT_EQ(&canary, error.private_data); + ASSERT_EQ(&canary, error.private_driver); + error.release(&error); } TEST_F(PostgresStatementTest, Cancel) { @@ -2397,4 +2400,4 @@ INSTANTIATE_TEST_SUITE_P(Decimal256NoScale, PostgresDecimalTest, INSTANTIATE_TEST_SUITE_P(Decimal256LargeTests, PostgresDecimalTest, testing::ValuesIn(kDecimal256LargeCases)); INSTANTIATE_TEST_SUITE_P(Decimal256LargeNoScale, PostgresDecimalTest, - testing::ValuesIn(kDecimal256LargeNoScaleCases)); + testing::ValuesIn(kDecimal256LargeNoScaleCases)); \ No newline at end of file diff --git a/c/validation/adbc_validation_statement.cc b/c/validation/adbc_validation_statement.cc index cd388623ba..9e406f0bad 100644 --- a/c/validation/adbc_validation_statement.cc +++ b/c/validation/adbc_validation_statement.cc @@ -2817,21 +2817,23 @@ struct ADBC_EXPORT AdbcError100 { // Test that an ADBC 1.0.0-sized error still works void StatementTest::TestErrorCompatibility() { static_assert(sizeof(AdbcError100) == ADBC_ERROR_1_0_0_SIZE, "Wrong size"); - // XXX: sketchy cast - auto* error = reinterpret_cast(malloc(ADBC_ERROR_1_0_0_SIZE)); - std::memset(error, 0, ADBC_ERROR_1_0_0_SIZE); + struct AdbcError error; + std::memset(&error, 0, ADBC_ERROR_1_1_0_SIZE); + struct AdbcDriver canary; + error.private_data = &canary; + error.private_driver = &canary; - ASSERT_THAT(AdbcStatementNew(&connection, &statement, error), IsOkStatus(error)); + ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error)); ASSERT_THAT( - AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", error), - IsOkStatus(error)); + AdbcStatementSetSqlQuery(&statement, "SELECT * FROM thistabledoesnotexist", &error), + IsOkStatus(&error)); adbc_validation::StreamReader reader; ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value, - &reader.rows_affected, error), - ::testing::Not(IsOkStatus(error))); - auto* old_error = reinterpret_cast(error); - old_error->release(old_error); - free(error); + &reader.rows_affected, &error), + ::testing::Not(IsOkStatus(&error))); + ASSERT_EQ(&canary, error.private_data); + ASSERT_EQ(&canary, error.private_driver); + error.release(&error); } void StatementTest::TestResultInvalidation() { @@ -2855,4 +2857,4 @@ void StatementTest::TestResultInvalidation() { // First reader may fail, or may succeed but give no data reader1.MaybeNext(); } -} // namespace adbc_validation +} // namespace adbc_validation \ No newline at end of file diff --git a/c/vendor/nanoarrow/nanoarrow.hpp b/c/vendor/nanoarrow/nanoarrow.hpp index 16c2e55b9f..1b1d77ec65 100644 --- a/c/vendor/nanoarrow/nanoarrow.hpp +++ b/c/vendor/nanoarrow/nanoarrow.hpp @@ -92,9 +92,16 @@ namespace literals { /// @{ /// \brief User literal operator allowing ArrowStringView construction like "str"_asv +#if !defined(__clang__) && (defined(__GNUC__) && __GNUC__ < 6) inline ArrowStringView operator"" _asv(const char* data, std::size_t size_bytes) { return {data, static_cast(size_bytes)}; } +#else +inline ArrowStringView operator""_asv(const char* data, std::size_t size_bytes) { + return {data, static_cast(size_bytes)}; +} +#endif +// N.B. older GCC requires the space above, newer Clang forbids the space // @} @@ -929,4 +936,4 @@ inline bool operator==(ArrowStringView l, ArrowStringView r) { return memcmp(l.data, r.data, l.size_bytes) == 0; } -#endif +#endif \ No newline at end of file diff --git a/ci/docker/cpp-clang-latest.dockerfile b/ci/docker/cpp-clang-latest.dockerfile index ff6e161e37..5d01b135da 100644 --- a/ci/docker/cpp-clang-latest.dockerfile +++ b/ci/docker/cpp-clang-latest.dockerfile @@ -15,9 +15,8 @@ # specific language governing permissions and limitations # under the License. -ARG VCPKG - FROM debian:12 +ARG GO RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y && \ @@ -34,6 +33,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get install -y cmake git libpq-dev libsqlite3-dev pkg-config -RUN curl -L -o go.tar.gz https://go.dev/dl/go1.22.5.linux-amd64.tar.gz && \ - tar -C /opt -xvf go.tar.gz && \ - echo 'export PATH=$PATH:/opt/go/bin' | tee -a ~/.bashrc +RUN curl -L -o go.tar.gz https://go.dev/dl/go${GO}.linux-amd64.tar.gz && \ + tar -C /opt -xvf go.tar.gz + +ENV PATH=/opt/go/bin:$PATH \ + CC=/usr/bin/clang \ + CXX=/usr/bin/clang++ \ No newline at end of file diff --git a/ci/docker/cpp-gcc-latest.dockerfile b/ci/docker/cpp-gcc-latest.dockerfile new file mode 100644 index 0000000000..6c1b5cf2eb --- /dev/null +++ b/ci/docker/cpp-gcc-latest.dockerfile @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM amd64/debian:experimental +ARG GCC +ARG GO + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update -y && \ + apt-get install -y -q cmake curl git gnupg libpq-dev libsqlite3-dev pkg-config && \ + apt-get install -y -q -t experimental g++-${GCC} gcc-${GCC} && \ + apt-get clean + +RUN curl -L -o go.tar.gz https://go.dev/dl/go${GO}.linux-amd64.tar.gz && \ + tar -C /opt -xvf go.tar.gz + +ENV PATH=/opt/go/bin:$PATH \ + CC=/usr/bin/gcc-${GCC} \ + CXX=/usr/bin/g++-${GCC} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a9ab47f409..de982f660d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,10 +38,21 @@ services: context: . dockerfile: ci/docker/cpp-clang-latest.dockerfile args: - VCPKG: ${VCPKG} + GO: ${GO} + volumes: + - .:/adbc:delegated + command: "bash -c 'git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build/clang-latest && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build/clang-latest'" + + cpp-gcc-latest: + build: + context: . + dockerfile: ci/docker/cpp-gcc-latest.dockerfile + args: + GCC: 15 + GO: ${GO} volumes: - .:/adbc:delegated - command: "bash -c 'export PATH=$PATH:/opt/go/bin CC=$(which clang) CXX=$(which clang++) && git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build'" + command: "bash -c 'git config --global --add safe.directory /adbc && /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build/gcc-latest && env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 BUILD_DRIVER_SQLITE=1 /adbc/ci/scripts/cpp_test.sh /adbc/build/gcc-latest'" ############################ Documentation ################################### @@ -256,4 +267,4 @@ services: timeout: 5s retries: 5 ports: - - "5432:5432" + - "5432:5432" \ No newline at end of file