From 7b22b5486d72d2e93b887454f7c55001772aaffd Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 20 Nov 2023 14:37:12 -0500 Subject: [PATCH 01/14] rebasing --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 1c5347ef4a..4f0dce5c46 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -87,6 +87,7 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } +<<<<<<< HEAD public override object GetValue(IArrowArray arrowArray, Field field, int index) { switch(arrowArray) @@ -100,6 +101,21 @@ public override object GetValue(IArrowArray arrowArray, Field field, int index) } } +======= + public override object GetValue(IArrowArray arrowArray, Field field, int index) + { + switch(arrowArray) + { + case StructArray structArray: + return SerializeToJson(structArray, index); + case ListArray listArray: + return listArray.GetSlicedValues(index); + default: + return base.GetValue(arrowArray, field, index); + } + } + +>>>>>>> be35986c (add back support for List, Struct types) private IArrowType TranslateType(TableFieldSchema field) { // per https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType-- From e62683f583a9a51a5f65f1dc6b5bacd4c868990f Mon Sep 17 00:00:00 2001 From: David Coe Date: Fri, 10 Nov 2023 11:19:32 -0500 Subject: [PATCH 02/14] fix line endings --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 4f0dce5c46..7d14bf8ccd 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -88,6 +88,9 @@ private Field TranslateField(TableFieldSchema field) } <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 641a6fe4 (fix line endings) public override object GetValue(IArrowArray arrowArray, Field field, int index) { switch(arrowArray) @@ -101,6 +104,7 @@ public override object GetValue(IArrowArray arrowArray, Field field, int index) } } +<<<<<<< HEAD ======= public override object GetValue(IArrowArray arrowArray, Field field, int index) { @@ -116,6 +120,8 @@ public override object GetValue(IArrowArray arrowArray, Field field, int index) } >>>>>>> be35986c (add back support for List, Struct types) +======= +>>>>>>> 641a6fe4 (fix line endings) private IArrowType TranslateType(TableFieldSchema field) { // per https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType-- From ebf404e3e067b0827e4057dc4662eb6050f59f03 Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 27 Nov 2023 11:16:21 -0500 Subject: [PATCH 03/14] add Scopes for BigQuery, refactor --- .../Drivers/BigQuery/BigQueryConnection.cs | 23 ++++++++- .../Drivers/BigQuery/BigQueryParameters.cs | 2 + .../src/Drivers/BigQuery/BigQueryStatement.cs | 22 --------- csharp/src/Drivers/BigQuery/readme.md | 3 ++ .../BigQuery/BigQueryTestConfiguration.cs | 7 +++ .../Drivers/BigQuery/BigQueryTestingUtils.cs | 21 ++++++-- csharp/test/Drivers/BigQuery/ClientTests.cs | 36 ++++++-------- csharp/test/Drivers/BigQuery/DriverTests.cs | 49 ++++++++----------- csharp/test/Drivers/Snowflake/DriverTests.cs | 4 +- 9 files changed, 89 insertions(+), 78 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 48a9c9535c..07d93453aa 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -105,7 +105,7 @@ internal void Open() if (!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out refreshToken)) throw new ArgumentException($"The {BigQueryParameters.RefreshToken} parameter is not present"); - this.credential = GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint)); + this.credential = ApplyScopes(GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint))); } else { @@ -114,12 +114,31 @@ internal void Open() if (!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json)) throw new ArgumentException($"The {BigQueryParameters.JsonCredential} parameter is not present"); - this.credential = GoogleCredential.FromJson(json); + this.credential = ApplyScopes(GoogleCredential.FromJson(json)); } this.client = BigQueryClient.Create(projectId, this.credential); } + /// + /// Apply any additional scopes to the credential. + /// + /// + /// + private GoogleCredential ApplyScopes(GoogleCredential credential) + { + if (credential == null) throw new ArgumentNullException(nameof(credential)); + + if (this.properties.TryGetValue(BigQueryParameters.Scopes, out string scopes)) + { + IEnumerable parsedScopes = scopes.Split(',').Where(x => x.Length > 0); + + return credential.CreateScoped(parsedScopes); + } + + return credential; + } + public override IArrowArrayStream GetInfo(List codes) { const int strValTypeID = 0; diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index 5a12388c3b..6e7d098e71 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -31,6 +31,8 @@ public class BigQueryParameters public const string AllowLargeResults = "adbc.bigquery.allow_large_results"; public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql"; public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string"; + public const string Scopes = "adbc.bigquery.scopes"; + } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 7d14bf8ccd..1c5347ef4a 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -87,10 +87,6 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 641a6fe4 (fix line endings) public override object GetValue(IArrowArray arrowArray, Field field, int index) { switch(arrowArray) @@ -104,24 +100,6 @@ public override object GetValue(IArrowArray arrowArray, Field field, int index) } } -<<<<<<< HEAD -======= - public override object GetValue(IArrowArray arrowArray, Field field, int index) - { - switch(arrowArray) - { - case StructArray structArray: - return SerializeToJson(structArray, index); - case ListArray listArray: - return listArray.GetSlicedValues(index); - default: - return base.GetValue(arrowArray, field, index); - } - } - ->>>>>>> be35986c (add back support for List, Struct types) -======= ->>>>>>> 641a6fe4 (fix line endings) private IArrowType TranslateType(TableFieldSchema field) { // per https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType-- diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 310cf32dae..611374c585 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -55,6 +55,9 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.refresh_token**
    The refresh token used for when the generated OAuth token expires. Required for `user` authentication. +**adbc.bigquery.scopes**
+    Optional. Comma separated list of scopes to include for the credential. + **adbc.bigquery.use_legacy_sql**
    Sets the [UseLegacySql](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_UseLegacySql) value of the QueryOptions to `true` if configured; otherwise, the default is `false`. diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index ee97c801c0..a6b0ab6b07 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -35,5 +35,12 @@ internal class BigQueryTestConfiguration : TestConfiguration [JsonPropertyName("refreshToken")] public string RefreshToken { get; set; } + + [JsonPropertyName("scopes")] + public string Scopes { get; set; } + + [JsonPropertyName("jsonCredential")] + public string JsonCredential { get; set; } + } } diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 567532cb16..2b0518834f 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -54,11 +54,26 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestConf Dictionary parameters = new Dictionary { { BigQueryParameters.ProjectId, testConfiguration.ProjectId }, - { BigQueryParameters.ClientId, testConfiguration.ClientId }, - { BigQueryParameters.ClientSecret, testConfiguration.ClientSecret}, - { BigQueryParameters.RefreshToken, testConfiguration.RefreshToken} }; + if(!string.IsNullOrEmpty(testConfiguration.JsonCredential)) + { + parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.ServiceAccountAuthenticationType); + parameters.Add(BigQueryParameters.JsonCredential, testConfiguration.JsonCredential); + } + else + { + parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.UserAuthenticationType); + parameters.Add(BigQueryParameters.ClientId, testConfiguration.ClientId); + parameters.Add(BigQueryParameters.ClientSecret, testConfiguration.ClientSecret); + parameters.Add(BigQueryParameters.RefreshToken, testConfiguration.RefreshToken); + } + + if (!string.IsNullOrEmpty(testConfiguration.Scopes)) + { + parameters.Add(BigQueryParameters.Scopes, testConfiguration.Scopes); + } + return parameters; } diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index d9672d225f..902c24d947 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -34,9 +34,13 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] public class ClientTests { + private BigQueryTestConfiguration _testConfiguration; + public ClientTests() { Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE)); + + _testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); } /// @@ -45,17 +49,15 @@ public ClientTests() [SkippableFact, Order(1)] public void CanClientExecuteUpdate() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { adbcConnection.Open(); - string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); + string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration); List expectedResults = new List() { -1, 1, 1, 1 }; - Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, testConfiguration, queries, expectedResults); + Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, _testConfiguration, queries, expectedResults); } } @@ -65,11 +67,9 @@ public void CanClientExecuteUpdate() [SkippableFact, Order(2)] public void CanClientGetSchema() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { - Tests.ClientTests.CanClientGetSchema(adbcConnection, testConfiguration); + Tests.ClientTests.CanClientGetSchema(adbcConnection, _testConfiguration); } } @@ -80,11 +80,9 @@ public void CanClientGetSchema() [SkippableFact, Order(3)] public void CanClientExecuteQuery() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { - Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration); + Tests.ClientTests.CanClientExecuteQuery(adbcConnection, _testConfiguration); } } @@ -95,9 +93,7 @@ public void CanClientExecuteQuery() [SkippableFact, Order(4)] public void VerifyTypesAndValues() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection(testConfiguration)) + using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection()) { SampleDataBuilder sampleDataBuilder = BigQueryData.GetSampleData(); @@ -108,9 +104,7 @@ public void VerifyTypesAndValues() [SkippableFact] public void VerifySchemaTables() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { adbcConnection.Open(); @@ -147,11 +141,11 @@ public void VerifySchemaTables() } } - private Adbc.Client.AdbcConnection GetAdbcConnection(BigQueryTestConfiguration testConfiguration) + private Adbc.Client.AdbcConnection GetAdbcConnection() { return new Adbc.Client.AdbcConnection( new BigQueryDriver(), - BigQueryTestingUtils.GetBigQueryParameters(testConfiguration), + BigQueryTestingUtils.GetBigQueryParameters(_testConfiguration), new Dictionary() ); } diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 9d470475b6..7b09021d9a 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -35,9 +35,13 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] public class DriverTests { + BigQueryTestConfiguration _testConfiguration; + public DriverTests() { Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE)); + _testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); + } /// @@ -47,11 +51,10 @@ public DriverTests() [SkippableFact, Order(1)] public void CanExecuteUpdate() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); - string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); + string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration); List expectedResults = new List() { -1, 1, 1, 1 }; @@ -73,9 +76,7 @@ public void CanExecuteUpdate() [SkippableFact, Order(2)] public void CanGetInfo() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream stream = adbcConnection.GetInfo(new List() { AdbcInfoCode.DriverName, AdbcInfoCode.DriverVersion, AdbcInfoCode.VendorName }); @@ -102,15 +103,13 @@ public void CanGetInfo() [SkippableFact, Order(3)] public void CanGetObjects() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - // need to add the database - string catalogName = testConfiguration.Metadata.Catalog; - string schemaName = testConfiguration.Metadata.Schema; - string tableName = testConfiguration.Metadata.Table; + string catalogName = _testConfiguration.Metadata.Catalog; + string schemaName = _testConfiguration.Metadata.Schema; + string tableName = _testConfiguration.Metadata.Table; string columnName = null; - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream stream = adbcConnection.GetObjects( depth: AdbcConnection.GetObjectsDepth.All, @@ -132,7 +131,7 @@ public void CanGetObjects() .Select(c => c.Columns) .FirstOrDefault(); - Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, columns.Count); + Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns.Count); } /// @@ -141,19 +140,17 @@ public void CanGetObjects() [SkippableFact, Order(4)] public void CanGetTableSchema() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); - - string catalogName = testConfiguration.Metadata.Catalog; - string schemaName = testConfiguration.Metadata.Schema; - string tableName = testConfiguration.Metadata.Table; + string catalogName = _testConfiguration.Metadata.Catalog; + string schemaName = _testConfiguration.Metadata.Schema; + string tableName = _testConfiguration.Metadata.Table; Schema schema = adbcConnection.GetTableSchema(catalogName, schemaName, tableName); int numberOfFields = schema.FieldsList.Count; - Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, numberOfFields); + Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, numberOfFields); } /// @@ -162,9 +159,7 @@ public void CanGetTableSchema() [SkippableFact, Order(5)] public void CanGetTableTypes() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream arrowArrayStream = adbcConnection.GetTableTypes(); @@ -199,16 +194,14 @@ public void CanGetTableTypes() [SkippableFact, Order(6)] public void CanExecuteQuery() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = testConfiguration.Query; + statement.SqlQuery = _testConfiguration.Query; QueryResult queryResult = statement.ExecuteQuery(); - Tests.DriverTests.CanExecuteQuery(queryResult, testConfiguration.ExpectedResultsCount); + Tests.DriverTests.CanExecuteQuery(queryResult, _testConfiguration.ExpectedResultsCount); } } } diff --git a/csharp/test/Drivers/Snowflake/DriverTests.cs b/csharp/test/Drivers/Snowflake/DriverTests.cs index 1d80ce2b59..9d1a7af128 100644 --- a/csharp/test/Drivers/Snowflake/DriverTests.cs +++ b/csharp/test/Drivers/Snowflake/DriverTests.cs @@ -330,9 +330,9 @@ public void CanGetObjectsAll() .FirstOrDefault(); Assert.True(columns != null, "Columns cannot be null"); - Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, columns.Count); + Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns.Count); - if (testConfiguration.UseHighPrecision) + if (_testConfiguration.UseHighPrecision) { IEnumerable highPrecisionColumns = columns.Where(c => c.XdbcTypeName == "NUMBER"); From b2c3d6b28cb75914a8b32f51dfbe52562c05de4b Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 27 Nov 2023 14:05:44 -0500 Subject: [PATCH 04/14] add support for returning List for ListArray --- csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs | 10 +-- csharp/src/Client/AdbcDataReader.cs | 2 +- .../src/Drivers/BigQuery/BigQueryStatement.cs | 88 ++++++++++++++++++- .../Client/ClientTests.cs | 2 +- .../Apache.Arrow.Adbc.Tests/ClientTests.cs | 40 ++++++++- csharp/test/Drivers/BigQuery/BigQueryData.cs | 6 +- 6 files changed, 131 insertions(+), 17 deletions(-) diff --git a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs index 9e42544eac..ef4ae13ac8 100644 --- a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs +++ b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs @@ -16,6 +16,9 @@ */ using System; +using System.Collections; +using System.Collections.Generic; +using System.Data.SqlTypes; using System.IO; using System.Threading.Tasks; using Apache.Arrow.Ipc; @@ -140,17 +143,12 @@ public virtual void Dispose() /// /// The Arrow array. /// - /// - /// The from the that can - /// be used for metadata inspection. - /// /// /// The index in the array to get the value from. /// - public virtual object GetValue(IArrowArray arrowArray, Field field, int index) + public virtual object GetValue(IArrowArray arrowArray, int index) { if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); - if (field == null) throw new ArgumentNullException(nameof(field)); if (index < 0) throw new ArgumentOutOfRangeException(nameof(index)); switch (arrowArray) diff --git a/csharp/src/Client/AdbcDataReader.cs b/csharp/src/Client/AdbcDataReader.cs index 6e9a7ae49c..c58adc450e 100644 --- a/csharp/src/Client/AdbcDataReader.cs +++ b/csharp/src/Client/AdbcDataReader.cs @@ -347,7 +347,7 @@ public ReadOnlyCollection GetAdbcColumnSchema() public object GetValue(IArrowArray arrowArray, int ordinal) { Field field = this.schema.GetFieldByIndex(ordinal); - return this.adbcCommand.AdbcStatement.GetValue(arrowArray, field, this.currentRowInRecordBatch); + return this.adbcCommand.AdbcStatement.GetValue(arrowArray, this.currentRowInRecordBatch); } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 1c5347ef4a..14c05e452a 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -16,9 +16,12 @@ */ using System; +using System.Collections; using System.Collections.Generic; +using System.Data.SqlTypes; using System.IO; using System.Linq; +using System.Reflection; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -87,19 +90,96 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } - public override object GetValue(IArrowArray arrowArray, Field field, int index) + public override object GetValue(IArrowArray arrowArray, int index) { switch(arrowArray) { case StructArray structArray: return SerializeToJson(structArray, index); case ListArray listArray: - return listArray.GetSlicedValues(index); + IArrowArray array = listArray.GetSlicedValues(index); + IList values = CreateList(array); + + for(int i=0;i(); + case Date32Array date32Array: + case Date64Array date64Array: + return new List(); + case Decimal128Array decimal128Array: + return new List(); + case Decimal256Array decimal256Array: + return new List(); + case DoubleArray doubleArray: + return new List(); + case FloatArray floatArray: + return new List(); +#if NET5_0_OR_GREATER + case PrimitiveArray halfFloatArray: + return new List(); +#endif + case Int8Array int8Array: + return new List(); + case Int16Array int16Array: + return new List(); + case Int32Array int32Array: + return new List(); + case Int64Array int64Array: + return new List(); + case StringArray stringArray: + return new List(); +#if NET6_0_OR_GREATER + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#else + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#endif + case TimestampArray timestampArray: + return new List(); + case UInt8Array uInt8Array: + return new List(); + case UInt16Array uInt16Array: + return new List(); + case UInt32Array uInt32Array: + return new List(); + case UInt64Array uInt64Array: + return new List(); + + case BinaryArray binaryArray: + return new List(); + + // not covered: + // -- struct array + // -- dictionary array + // -- fixed size binary + // -- list array + // -- union array + } + + return null; + } + + private IArrowType TranslateType(TableFieldSchema field) { // per https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType-- @@ -187,7 +267,7 @@ private string SerializeToJson(StructArray structArray, int index) for (int i = 0; i < structArray.Data.Children.Length; i++) { jsonDictionary.Add(structType.Fields[i].Name, - GetValue(structArray.Fields[i], structType.GetFieldByName(structType.Fields[i].Name), index)); + GetValue(structArray.Fields[i], index)); } return JsonSerializer.Serialize(jsonDictionary); } diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs index 9548b85e4a..423f2f819b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs @@ -70,7 +70,7 @@ private AdbcDataReader GetMoqDataReader(DecimalBehavior decimalBehavior, string Mock mockStatement = new Mock(); mockStatement.Setup(x => x.ExecuteQuery()).Returns(queryResult); ; - mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny(), It.IsAny())).Returns(sqlDecimal); + mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny())).Returns(sqlDecimal); Adbc.Client.AdbcConnection mockConnection = new Adbc.Client.AdbcConnection(); mockConnection.DecimalBehavior = decimalBehavior; diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs index b3784fc05d..3de76e3754 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs @@ -222,6 +222,22 @@ static void AssertTypeAndValue( Console.WriteLine($"Could not validate the values inside of {netType.Name} because it is empty for query [{query}]"); } } + else if (value is IList && !(value is byte[])) + { + IList values = value as IList; + int length = values.Count; + + if (length > 0) + { + object internalValue = values[0]; + + Assert.True(internalValue.GetType() == ctv.ExpectedNetType, $"{name} is {netType.Name} and not {ctv.ExpectedNetType.Name} in the reader for query [{query}]"); + } + else + { + Console.WriteLine($"Could not validate the values inside of {netType.Name} because it is empty for query [{query}]"); + } + } else { Assert.True(netType == ctv.ExpectedNetType, $"{name} is {netType.Name} and not {ctv.ExpectedNetType.Name} in the reader for query [{query}]"); @@ -230,7 +246,29 @@ static void AssertTypeAndValue( if (value != null) { - if (!value.GetType().BaseType.Name.Contains("PrimitiveArray")) + if (value is IList && !(value is byte[])) + { + IList list = value as IList; + IList expectedList = ctv.ExpectedValue as IList; + int i = -1; + + foreach (var actual in list) + { + i++; + int j = -1; + + foreach (var expected in expectedList) + { + j++; + + if (i == j) + { + Assert.True(expected.Equals(actual), $"Expected value does not match actual value for {ctv.Name} at {i} for query [{query}]"); + } + } + } + } + else if (!value.GetType().BaseType.Name.Contains("PrimitiveArray")) { Assert.True(ctv.ExpectedNetType == value.GetType(), $"Expected type does not match actual type for {ctv.Name} for query [{query}]"); diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index 2a39fcaba8..d024ffa6ee 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -76,7 +76,7 @@ public static SampleDataBuilder GetSampleData() new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)), new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2023, 9, 8, 12, 34, 56), TimeSpan.Zero)), new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), "POINT(1 2)"), - new ColumnNetTypeArrowTypeValue("numbers", typeof(long), typeof(Int64Type), numbersArray), + new ColumnNetTypeArrowTypeValue("numbers", typeof(long), typeof(Int64Type), new List() { 1, 2, 3 }), new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":\"John Doe\",\"age\":30}"), new ColumnNetTypeArrowTypeValue("json", typeof(string), typeof(StringType), "{\"age\":29,\"name\":\"Jane Doe\"}") @@ -100,8 +100,6 @@ public static SampleDataBuilder GetSampleData() }); // nulls - Int64Array.Builder emptyNumbersBuilder = new Int64Array.Builder(); - Int64Array emptyNumbersArray = emptyNumbersBuilder.Build(); sampleDataBuilder.Samples.Add( new SampleData() @@ -139,7 +137,7 @@ public static SampleDataBuilder GetSampleData() new ColumnNetTypeArrowTypeValue("datetime", typeof(DateTimeOffset), typeof(TimestampType), null), new ColumnNetTypeArrowTypeValue("timestamp", typeof(DateTimeOffset), typeof(TimestampType), null), new ColumnNetTypeArrowTypeValue("point", typeof(string), typeof(StringType), null), - new ColumnNetTypeArrowTypeValue("numbers", typeof(long), typeof(Int64Type), emptyNumbersArray), + new ColumnNetTypeArrowTypeValue("numbers", typeof(long), typeof(Int64Type), new List{ }), new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), "{\"name\":null,\"age\":null}") } }); From 7c13c8f3930d49e094bfe4ea90ddf9338300b3b9 Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 27 Nov 2023 14:37:25 -0500 Subject: [PATCH 05/14] clean up --- csharp/test/Drivers/BigQuery/BigQueryData.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index d024ffa6ee..be9fdf94a5 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -33,10 +33,6 @@ internal class BigQueryData /// public static SampleDataBuilder GetSampleData() { - Int64Array.Builder numbersBuilder = new Int64Array.Builder(); - numbersBuilder.AppendRange(new List() { 1, 2, 3 }); - Int64Array numbersArray = numbersBuilder.Build(); - SampleDataBuilder sampleDataBuilder = new SampleDataBuilder(); // standard values @@ -100,7 +96,6 @@ public static SampleDataBuilder GetSampleData() }); // nulls - sampleDataBuilder.Samples.Add( new SampleData() { From ea129fdf71d1ad62baf16477839a5517103edede Mon Sep 17 00:00:00 2001 From: David Coe Date: Tue, 28 Nov 2023 17:12:23 -0500 Subject: [PATCH 06/14] fix warnings; change target framework --- .../Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 3 +- .../Drivers/BigQuery/BigQueryConnection.cs | 354 ++++++++++-------- 2 files changed, 201 insertions(+), 156 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index 368ff8181e..3137b64efa 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -1,6 +1,7 @@ - netstandard2.0;net6.0 + net472;net6.0 + enable diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 07d93453aa..4b1162c1d5 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -69,24 +69,23 @@ public BigQueryConnection(IReadOnlyDictionary properties) /// internal void Open() { - string projectId = string.Empty; - string clientId = string.Empty; - string clientSecret = string.Empty; - string refreshToken = string.Empty; + string? projectId = string.Empty; + string? clientId = string.Empty; + string? clientSecret = string.Empty; + string? refreshToken = string.Empty; string tokenEndpoint = BigQueryConstants.TokenEndpoint; - string authenticationType = BigQueryConstants.UserAuthenticationType; + string? authenticationType = BigQueryConstants.UserAuthenticationType; // TODO: handle token expiration if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out projectId)) throw new ArgumentException($"The {BigQueryParameters.ProjectId} parameter is not present"); - if (this.properties.ContainsKey(BigQueryParameters.AuthenticationType)) + //if (this.properties.ContainsKey(BigQueryParameters.AuthenticationType)) + if (this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType)) { - this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType); - if (!authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase) && !authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType, StringComparison.OrdinalIgnoreCase)) { @@ -94,7 +93,7 @@ internal void Open() } } - if (authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase)) + if (authenticationType is not null && authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase)) { if (!this.properties.TryGetValue(BigQueryParameters.ClientId, out clientId)) throw new ArgumentException($"The {BigQueryParameters.ClientId} parameter is not present"); @@ -105,40 +104,21 @@ internal void Open() if (!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out refreshToken)) throw new ArgumentException($"The {BigQueryParameters.RefreshToken} parameter is not present"); - this.credential = ApplyScopes(GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint))); + this.credential = GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint)); } else { - string json = string.Empty; + string? json = string.Empty; if (!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json)) throw new ArgumentException($"The {BigQueryParameters.JsonCredential} parameter is not present"); - this.credential = ApplyScopes(GoogleCredential.FromJson(json)); + this.credential = GoogleCredential.FromJson(json); } this.client = BigQueryClient.Create(projectId, this.credential); } - /// - /// Apply any additional scopes to the credential. - /// - /// - /// - private GoogleCredential ApplyScopes(GoogleCredential credential) - { - if (credential == null) throw new ArgumentNullException(nameof(credential)); - - if (this.properties.TryGetValue(BigQueryParameters.Scopes, out string scopes)) - { - IEnumerable parsedScopes = scopes.Split(',').Where(x => x.Length > 0); - - return credential.CreateScoped(parsedScopes); - } - - return credential; - } - public override IArrowArrayStream GetInfo(List codes) { const int strValTypeID = 0; @@ -279,23 +259,26 @@ private List GetCatalogs( StringArray.Builder catalogNameBuilder = new StringArray.Builder(); List catalogDbSchemasValues = new List(); string catalogRegexp = PatternToRegEx(catalogPattern); - PagedEnumerable catalogs = this.client.ListProjects(); + PagedEnumerable? catalogs = this.client?.ListProjects(); - foreach (CloudProject catalog in catalogs) + if (catalogs is not null) { - if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase)) + foreach (CloudProject catalog in catalogs) { - catalogNameBuilder.Append(catalog.ProjectId); - - if (depth == GetObjectsDepth.Catalogs) + if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase)) { - catalogDbSchemasValues.Add(null); - } - else - { - catalogDbSchemasValues.Add(GetDbSchemas( - depth, catalog.ProjectId, dbSchemaPattern, - tableNamePattern, tableTypes, columnNamePattern)); + catalogNameBuilder.Append(catalog.ProjectId); + + if (depth == GetObjectsDepth.Catalogs) + { + catalogDbSchemasValues.Add(null); + } + else + { + catalogDbSchemasValues.Add(GetDbSchemas( + depth, catalog.ProjectId, dbSchemaPattern, + tableNamePattern, tableTypes, columnNamePattern)); + } } } } @@ -324,25 +307,28 @@ private StructArray GetDbSchemas( string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern); - PagedEnumerable schemas = this.client.ListDatasets(catalog); + PagedEnumerable? schemas = this.client?.ListDatasets(catalog); - foreach (BigQueryDataset schema in schemas) + if (schemas is not null) { - if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase)) + foreach (BigQueryDataset? schema in schemas) { - dbSchemaNameBuilder.Append(schema.Reference.DatasetId); - length++; - nullBitmapBuffer.Append(true); - - if (depth == GetObjectsDepth.DbSchemas) + if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase)) { - dbSchemaTablesValues.Add(null); - } - else - { - dbSchemaTablesValues.Add(GetTableSchemas( - depth, catalog, schema.Reference.DatasetId, - tableNamePattern, tableTypes, columnNamePattern)); + dbSchemaNameBuilder.Append(schema.Reference.DatasetId); + length++; + nullBitmapBuffer.Append(true); + + if (depth == GetObjectsDepth.DbSchemas) + { + dbSchemaTablesValues.Add(null); + } + else + { + dbSchemaTablesValues.Add(GetTableSchemas( + depth, catalog, schema.Reference.DatasetId, + tableNamePattern, tableTypes, columnNamePattern)); + } } } } @@ -397,25 +383,29 @@ private StructArray GetTableSchemas( } } - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + + if (result is not null) { - tableNameBuilder.Append(row["table_name"].ToString()); - tableTypeBuilder.Append(row["table_type"].ToString()); - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + tableNameBuilder.Append(GetValue(row["table_name"])); + tableTypeBuilder.Append(GetValue(row["table_type"])); + nullBitmapBuffer.Append(true); + length++; - tableConstraintsValues.Add(GetConstraintSchema( - depth, catalog, dbSchema, row["table_name"].ToString(), columnNamePattern)); + tableConstraintsValues.Add(GetConstraintSchema( + depth, catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern)); - if (depth == GetObjectsDepth.Tables) - { - tableColumnsValues.Add(null); - } - else - { - tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, row["table_name"].ToString(), columnNamePattern)); + if (depth == GetObjectsDepth.Tables) + { + tableColumnsValues.Add(null); + } + else + { + tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern)); + } } } @@ -470,32 +460,36 @@ private StructArray GetColumnSchema( query = string.Concat(query, string.Format("AND column_name LIKE '{0}'", Sanitize(columnNamePattern))); } - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + - foreach (BigQueryRow row in result) + if (result is not null) { - columnNameBuilder.Append(row["column_name"].ToString()); - ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); - remarksBuilder.Append(""); - xdbcDataTypeBuilder.AppendNull(); - string dataType = ToTypeName(row["data_type"].ToString()); - xdbcTypeNameBuilder.Append(dataType); - xdbcColumnSizeBuilder.AppendNull(); - xdbcDecimalDigitsBuilder.AppendNull(); - xdbcNumPrecRadixBuilder.AppendNull(); - xdbcNullableBuilder.AppendNull(); - xdbcColumnDefBuilder.AppendNull(); - xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); - xdbcDatetimeSubBuilder.AppendNull(); - xdbcCharOctetLengthBuilder.AppendNull(); - xdbcIsNullableBuilder.Append(row["is_nullable"].ToString()); - xdbcScopeCatalogBuilder.AppendNull(); - xdbcScopeSchemaBuilder.AppendNull(); - xdbcScopeTableBuilder.AppendNull(); - xdbcIsAutoincrementBuilder.AppendNull(); - xdbcIsGeneratedcolumnBuilder.Append(row["is_generated"].ToString().ToUpper() == "YES"); - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + columnNameBuilder.Append(GetValue(row["column_name"])); + ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); + remarksBuilder.Append(""); + xdbcDataTypeBuilder.AppendNull(); + string dataType = ToTypeName(GetValue(row["data_type"])); + xdbcTypeNameBuilder.Append(dataType); + xdbcColumnSizeBuilder.AppendNull(); + xdbcDecimalDigitsBuilder.AppendNull(); + xdbcNumPrecRadixBuilder.AppendNull(); + xdbcNullableBuilder.AppendNull(); + xdbcColumnDefBuilder.AppendNull(); + xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); + xdbcDatetimeSubBuilder.AppendNull(); + xdbcCharOctetLengthBuilder.AppendNull(); + xdbcIsNullableBuilder.Append(GetValue(row["is_nullable"])); + xdbcScopeCatalogBuilder.AppendNull(); + xdbcScopeSchemaBuilder.AppendNull(); + xdbcScopeTableBuilder.AppendNull(); + xdbcIsAutoincrementBuilder.AppendNull(); + xdbcIsGeneratedcolumnBuilder.Append(GetValue(row["is_generated"]).ToUpper() == "YES"); + nullBitmapBuffer.Append(true); + length++; + } } List dataArrays = new List @@ -545,36 +539,40 @@ private StructArray GetConstraintSchema( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(table)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result is not null) { - string constraintName = row["constraint_name"].ToString(); - constraintNameBuilder.Append(constraintName); - string constraintType = row["constraint_type"].ToString(); - constraintTypeBuilder.Append(constraintType); - nullBitmapBuffer.Append(true); - length++; - - if (depth == GetObjectsDepth.All) + foreach (BigQueryRow row in result) { - constraintColumnNamesValues.Add(GetConstraintColumnNames( - catalog, dbSchema, table, constraintName)); - if (constraintType.ToUpper() == "FOREIGN KEY") + string constraintName = GetValue(row["constraint_name"]); + constraintNameBuilder.Append(constraintName); + string constraintType = GetValue(row["constraint_type"]); + constraintTypeBuilder.Append(constraintType); + nullBitmapBuffer.Append(true); + length++; + + if (depth == GetObjectsDepth.All) { - constraintColumnUsageValues.Add(GetConstraintsUsage( + constraintColumnNamesValues.Add(GetConstraintColumnNames( catalog, dbSchema, table, constraintName)); + + if (constraintType?.ToUpper() == "FOREIGN KEY") + { + constraintColumnUsageValues.Add(GetConstraintsUsage( + catalog, dbSchema, table, constraintName)); + } + else + { + constraintColumnUsageValues.Add(null); + } } else { + constraintColumnNamesValues.Add(null); constraintColumnUsageValues.Add(null); } } - else - { - constraintColumnNamesValues.Add(null); - constraintColumnUsageValues.Add(null); - } } List dataArrays = new List @@ -603,11 +601,14 @@ private StringArray GetConstraintColumnNames( StringArray.Builder constraintColumnNamesBuilder = new StringArray.Builder(); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result is not null) { - constraintColumnNamesBuilder.Append(row["column_name"].ToString()); + foreach (BigQueryRow row in result) + { + constraintColumnNamesBuilder.Append(GetValue(row["column_name"])); + } } return constraintColumnNamesBuilder.Build(); @@ -629,16 +630,19 @@ private StructArray GetConstraintsUsage( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE WHERE table_name = '{2}' AND constraint_name = '{3}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(table), Sanitize(constraintName)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if(result is not null) { - constraintFkCatalogBuilder.Append(row["constraint_catalog"].ToString()); - constraintFkDbSchemaBuilder.Append(row["constraint_schema"].ToString()); - constraintFkTableBuilder.Append(row["table_name"].ToString()); - constraintFkColumnNameBuilder.Append(row["column_name"].ToString()); - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + constraintFkCatalogBuilder.Append(GetValue(row["constraint_catalog"])); + constraintFkDbSchemaBuilder.Append(GetValue(row["constraint_schema"])); + constraintFkTableBuilder.Append(GetValue(row["table_name"])); + constraintFkColumnNameBuilder.Append(GetValue(row["column_name"])); + nullBitmapBuffer.Append(true); + length++; + } } List dataArrays = new List @@ -713,13 +717,16 @@ public override Schema GetTableSchema(string catalog, string dbSchema, string ta string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(tableName)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); List fields = new List(); - foreach (BigQueryRow row in result) + if (result is not null) { - fields.Add(DescToField(row)); + foreach (BigQueryRow row in result) + { + fields.Add(DescToField(row)); + } } return new Schema(fields, null); @@ -729,22 +736,38 @@ private Field DescToField(BigQueryRow row) { Dictionary metaData = new Dictionary(); metaData.Add("PRIMARY_KEY", ""); - metaData.Add("ORDINAL_POSITION", row["ordinal_position"].ToString()); - metaData.Add("DATA_TYPE", row["data_type"].ToString()); + metaData.Add("ORDINAL_POSITION", GetValue(row["ordinal_position"])); + metaData.Add("DATA_TYPE", GetValue(row["data_type"])); - Field.Builder fieldBuilder = SchemaFieldGenerator(row["column_name"].ToString().ToLower(), row["data_type"].ToString()); + Field.Builder fieldBuilder = SchemaFieldGenerator(GetValue(row["column_name"]).ToLower(), GetValue(row["data_type"])); fieldBuilder.Metadata(metaData); - if (!row["is_nullable"].ToString().Equals("YES", StringComparison.OrdinalIgnoreCase)) + if (!GetValue(row["is_nullable"]).Equals("YES", StringComparison.OrdinalIgnoreCase)) { fieldBuilder.Nullable(false); } - fieldBuilder.Name(row["column_name"].ToString().ToLower()); + fieldBuilder.Name(GetValue(row["column_name"]).ToLower()); return fieldBuilder.Build(); } + private string GetValue(object? value) + { + switch (value) + { + case string sValue: + if (string.IsNullOrEmpty(sValue)) + return string.Empty; + return sValue; + default: + if (value != null) + return value.ToString(); + + throw new InvalidOperationException($"Cannot parse {value}"); + } + } + private Field.Builder SchemaFieldGenerator(string name, string type) { int index = type.IndexOf("("); @@ -839,7 +862,7 @@ public override IArrowArrayStream GetTableTypes() return new BigQueryInfoArrowStream(StandardSchemas.TableTypesSchema, dataArrays); } - private ListArray CreateNestedListArray(List arrayList, IArrowType dataType) + private ListArray CreateNestedListArray(List arrayList, IArrowType dataType) { ArrowBuffer.Builder valueOffsetsBufferBuilder = new ArrowBuffer.Builder(); ArrowBuffer.BitmapBuilder validityBufferBuilder = new ArrowBuffer.BitmapBuilder(); @@ -847,7 +870,7 @@ private ListArray CreateNestedListArray(List arrayList, IArrowType int length = 0; int nullCount = 0; - foreach (IArrowArray array in arrayList) + foreach (IArrowArray? array in arrayList) { if (array == null) { @@ -873,7 +896,9 @@ private ListArray CreateNestedListArray(List arrayList, IArrowType { EmptyArrayCreationVisitor visitor = new EmptyArrayCreationVisitor(0); dataType.Accept(visitor); - data = visitor.Result; + + if(visitor.Result is not null) + data = visitor.Result; } IArrowArray value = ArrowArrayFactory.BuildArray(data); @@ -930,6 +955,9 @@ public override void Dispose() private string Sanitize(string input) { + if (string.IsNullOrEmpty(input)) + return string.Empty; + bool isValidInput = sanitizedInputRegex.IsMatch(input); if (isValidInput) @@ -950,7 +978,7 @@ private string Sanitize(string input) /// /// /// - private string GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint) + private string? GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint) { string body = string.Format( "grant_type=refresh_token&client_id={0}&client_secret={1}&refresh_token={2}", @@ -966,9 +994,9 @@ private string GetAccessToken(string clientId, string clientSecret, string refre HttpResponseMessage response = httpClient.SendAsync(request).Result; string responseBody = response.Content.ReadAsStringAsync().Result; - BigQueryTokenResponse bigQueryTokenResponse = JsonSerializer.Deserialize(responseBody); + BigQueryTokenResponse? bigQueryTokenResponse = JsonSerializer.Deserialize(responseBody); - return bigQueryTokenResponse.AccessToken; + return bigQueryTokenResponse?.AccessToken; } enum XdbcDataType @@ -1010,7 +1038,7 @@ private class EmptyArrayCreationVisitor : IArrowTypeVisitor, IArrowTypeVisitor { - public ArrayData Result { get; private set; } + public ArrayData? Result { get; private set; } private readonly int _length; public EmptyArrayCreationVisitor(int length) @@ -1041,26 +1069,37 @@ public void Visit(StringType type) public void Visit(ListType type) { type.ValueDataType.Accept(this); - ArrayData child = Result; - Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, new[] { child }); + if (Result is not null) + { + ArrayData child = Result; + + Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, new[] { child }); + } } public void Visit(FixedSizeListType type) { type.ValueDataType.Accept(this); - ArrayData child = Result; - Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, new[] { child }); + if (Result is not null) + { + ArrayData child = Result; + + Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, new[] { child }); + } } public void Visit(StructType type) { ArrayData[] children = new ArrayData[type.Fields.Count]; + for (int i = 0; i < type.Fields.Count; i++) { type.Fields[i].DataType.Accept(this); - children[i] = Result; + + if(Result is not null) + children[i] = Result; } Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, children); @@ -1079,7 +1118,9 @@ public void Visit(UnionType type) for (int i = 0; i < type.Fields.Count; i++) { type.Fields[i].DataType.Accept(this); - children[i] = Result; + + if(Result is not null) + children[i] = Result; } ArrowBuffer[] buffers = new ArrowBuffer[bufferCount]; @@ -1094,13 +1135,16 @@ public void Visit(UnionType type) public void Visit(MapType type) { - ArrayData[] children = new ArrayData[2]; - type.KeyField.DataType.Accept(this); - children[0] = Result; - type.ValueField.DataType.Accept(this); - children[1] = Result; + if (Result is not null) + { + ArrayData[] children = new ArrayData[2]; + type.KeyField.DataType.Accept(this); + children[0] = Result; + type.ValueField.DataType.Accept(this); + children[1] = Result; - Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, children); + Result = new ArrayData(type, _length, _length, 0, new[] { ArrowBuffer.Empty }, children); + } } public void Visit(IArrowType type) From 06da1b4cc03e98e1ec4dcef9b2456b991b77af32 Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 22 Jan 2024 16:22:44 -0500 Subject: [PATCH 07/14] cleanup after merge --- csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs | 10 +- csharp/src/Client/AdbcDataReader.cs | 2 +- .../Drivers/BigQuery/BigQueryParameters.cs | 1 - .../src/Drivers/BigQuery/BigQueryStatement.cs | 95 +------------------ .../Client/ClientTests.cs | 2 +- 5 files changed, 13 insertions(+), 97 deletions(-) diff --git a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs index ef4ae13ac8..9e42544eac 100644 --- a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs +++ b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs @@ -16,9 +16,6 @@ */ using System; -using System.Collections; -using System.Collections.Generic; -using System.Data.SqlTypes; using System.IO; using System.Threading.Tasks; using Apache.Arrow.Ipc; @@ -143,12 +140,17 @@ public virtual void Dispose() /// /// The Arrow array. /// + /// + /// The from the that can + /// be used for metadata inspection. + /// /// /// The index in the array to get the value from. /// - public virtual object GetValue(IArrowArray arrowArray, int index) + public virtual object GetValue(IArrowArray arrowArray, Field field, int index) { if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); + if (field == null) throw new ArgumentNullException(nameof(field)); if (index < 0) throw new ArgumentOutOfRangeException(nameof(index)); switch (arrowArray) diff --git a/csharp/src/Client/AdbcDataReader.cs b/csharp/src/Client/AdbcDataReader.cs index c58adc450e..6e9a7ae49c 100644 --- a/csharp/src/Client/AdbcDataReader.cs +++ b/csharp/src/Client/AdbcDataReader.cs @@ -347,7 +347,7 @@ public ReadOnlyCollection GetAdbcColumnSchema() public object GetValue(IArrowArray arrowArray, int ordinal) { Field field = this.schema.GetFieldByIndex(ordinal); - return this.adbcCommand.AdbcStatement.GetValue(arrowArray, this.currentRowInRecordBatch); + return this.adbcCommand.AdbcStatement.GetValue(arrowArray, field, this.currentRowInRecordBatch); } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index 6e7d098e71..ff05036dc4 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -32,7 +32,6 @@ public class BigQueryParameters public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql"; public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string"; public const string Scopes = "adbc.bigquery.scopes"; - } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index d18cb411b2..c70172e978 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -21,7 +21,6 @@ using System.Data.SqlTypes; using System.IO; using System.Linq; -using System.Reflection; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -90,103 +89,19 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } - public override object GetValue(IArrowArray arrowArray, int index) + public override object GetValue(IArrowArray arrowArray, Field field, int index) { - switch(arrowArray) + switch (arrowArray) { case StructArray structArray: return SerializeToJson(structArray, index); case ListArray listArray: - IArrowArray array = listArray.GetSlicedValues(index); - IList? values = CreateList(array); - - if (values != null) - { - for (int i = 0; i < array.Length; i++) - { - values.Add(base.GetValue(array, i)); - } - - return values; - } - else - { - return new List(); - } - + return listArray.GetSlicedValues(index); default: - return base.GetValue(arrowArray, index); + return base.GetValue(arrowArray, field, index); } } - private IList? CreateList(IArrowArray arrowArray) - { - if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); - - switch (arrowArray) - { - case BooleanArray booleanArray: - return new List(); - case Date32Array date32Array: - case Date64Array date64Array: - return new List(); - case Decimal128Array decimal128Array: - return new List(); - case Decimal256Array decimal256Array: - return new List(); - case DoubleArray doubleArray: - return new List(); - case FloatArray floatArray: - return new List(); -#if NET5_0_OR_GREATER - case PrimitiveArray halfFloatArray: - return new List(); -#endif - case Int8Array int8Array: - return new List(); - case Int16Array int16Array: - return new List(); - case Int32Array int32Array: - return new List(); - case Int64Array int64Array: - return new List(); - case StringArray stringArray: - return new List(); -#if NET6_0_OR_GREATER - case Time32Array time32Array: - case Time64Array time64Array: - return new List(); -#else - case Time32Array time32Array: - case Time64Array time64Array: - return new List(); -#endif - case TimestampArray timestampArray: - return new List(); - case UInt8Array uInt8Array: - return new List(); - case UInt16Array uInt16Array: - return new List(); - case UInt32Array uInt32Array: - return new List(); - case UInt64Array uInt64Array: - return new List(); - - case BinaryArray binaryArray: - return new List(); - - // not covered: - // -- struct array - // -- dictionary array - // -- fixed size binary - // -- list array - // -- union array - } - - return null; - } - - private IArrowType TranslateType(TableFieldSchema field) { // per https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/model/TableFieldSchema.html#getType-- @@ -282,7 +197,7 @@ private string SerializeToJson(StructArray structArray, int index) for (int i = 0; i < structArray.Data.Children.Length; i++) { jsonDictionary.Add(structType.Fields[i].Name, - GetValue(structArray.Fields[i], index)); + GetValue(structArray.Fields[i], structType.GetFieldByName(structType.Fields[i].Name), index)); } return JsonSerializer.Serialize(jsonDictionary); } diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs index 423f2f819b..9548b85e4a 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs @@ -70,7 +70,7 @@ private AdbcDataReader GetMoqDataReader(DecimalBehavior decimalBehavior, string Mock mockStatement = new Mock(); mockStatement.Setup(x => x.ExecuteQuery()).Returns(queryResult); ; - mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny())).Returns(sqlDecimal); + mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny(), It.IsAny())).Returns(sqlDecimal); Adbc.Client.AdbcConnection mockConnection = new Adbc.Client.AdbcConnection(); mockConnection.DecimalBehavior = decimalBehavior; From aa754f108cb6ba72171bfcd076ddae44849447cc Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 22 Jan 2024 16:31:13 -0500 Subject: [PATCH 08/14] more cleanup --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index c70172e978..90cddf65d9 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -16,9 +16,7 @@ */ using System; -using System.Collections; using System.Collections.Generic; -using System.Data.SqlTypes; using System.IO; using System.Linq; using System.Text.Json; From 61d6c7701f271261273c4354179f1971789add8a Mon Sep 17 00:00:00 2001 From: David Coe Date: Mon, 22 Jan 2024 16:43:02 -0500 Subject: [PATCH 09/14] remove more warnings --- .../Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 1 + .../Drivers/BigQuery/BigQueryConnection.cs | 106 ++++++++++-------- .../src/Drivers/BigQuery/BigQueryStatement.cs | 5 +- 3 files changed, 62 insertions(+), 50 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index 3bb39e99ec..0ce2c1fe26 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -2,6 +2,7 @@ netstandard2.0;net6.0 readme.md + enable diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 814de30a1a..668d7bf311 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -483,44 +483,46 @@ private StructArray GetColumnSchema( BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - columnNameBuilder.Append(row["column_name"].ToString()); - ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); - remarksBuilder.Append(""); + foreach (BigQueryRow row in result) + { + columnNameBuilder.Append(GetValue(row["column_name"])); + ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); + remarksBuilder.Append(""); - string dataType = ToTypeName(row["data_type"].ToString()); + string dataType = ToTypeName(GetValue(row["data_type"])); - if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) - { - ParsedDecimalValues values = ParsePrecisionAndScale(dataType); - xdbcColumnSizeBuilder.Append(values.Precision); - xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); - } - else - { - xdbcColumnSizeBuilder.AppendNull(); - xdbcDecimalDigitsBuilder.AppendNull(); - } + if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) + { + ParsedDecimalValues values = ParsePrecisionAndScale(dataType); + xdbcColumnSizeBuilder.Append(values.Precision); + xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); + } + else + { + xdbcColumnSizeBuilder.AppendNull(); + xdbcDecimalDigitsBuilder.AppendNull(); + } - xdbcDataTypeBuilder.AppendNull(); - xdbcTypeNameBuilder.Append(dataType); - xdbcNumPrecRadixBuilder.AppendNull(); - xdbcNullableBuilder.AppendNull(); - xdbcColumnDefBuilder.AppendNull(); - xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); - xdbcDatetimeSubBuilder.AppendNull(); - xdbcCharOctetLengthBuilder.AppendNull(); - xdbcIsNullableBuilder.Append(row["is_nullable"].ToString()); - xdbcScopeCatalogBuilder.AppendNull(); - xdbcScopeSchemaBuilder.AppendNull(); - xdbcScopeTableBuilder.AppendNull(); - xdbcIsAutoincrementBuilder.AppendNull(); - xdbcIsGeneratedcolumnBuilder.Append(row["is_generated"].ToString().ToUpper() == "YES"); - nullBitmapBuffer.Append(true); - length++; + xdbcDataTypeBuilder.AppendNull(); + xdbcTypeNameBuilder.Append(dataType); + xdbcNumPrecRadixBuilder.AppendNull(); + xdbcNullableBuilder.AppendNull(); + xdbcColumnDefBuilder.AppendNull(); + xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); + xdbcDatetimeSubBuilder.AppendNull(); + xdbcCharOctetLengthBuilder.AppendNull(); + xdbcIsNullableBuilder.Append(row["is_nullable"].ToString()); + xdbcScopeCatalogBuilder.AppendNull(); + xdbcScopeSchemaBuilder.AppendNull(); + xdbcScopeTableBuilder.AppendNull(); + xdbcIsAutoincrementBuilder.AppendNull(); + xdbcIsGeneratedcolumnBuilder.Append(GetValue(row["is_generated"]).ToUpper() == "YES"); + nullBitmapBuffer.Append(true); + length++; + } } - List dataArrays = new List { columnNameBuilder.Build(), @@ -631,10 +633,13 @@ private StringArray GetConstraintColumnNames( BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - string column = row["column_name"].ToString(); - constraintColumnNamesBuilder.Append(column); + foreach (BigQueryRow row in result) + { + string column = GetValue(row["column_name"]); + constraintColumnNamesBuilder.Append(column); + } } return constraintColumnNamesBuilder.Build(); @@ -658,20 +663,23 @@ private StructArray GetConstraintsUsage( BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - string constraint_catalog = row["constraint_catalog"].ToString(); - string constraint_schema = row["constraint_schema"].ToString(); - string table_name = row["table_name"].ToString(); - string column_name = row["column_name"].ToString(); - - constraintFkCatalogBuilder.Append(constraint_catalog); - constraintFkDbSchemaBuilder.Append(constraint_schema); - constraintFkTableBuilder.Append(table_name); - constraintFkColumnNameBuilder.Append(column_name); - - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + string constraint_catalog = GetValue(row["constraint_catalog"]); + string constraint_schema = GetValue(row["constraint_schema"]); + string table_name = GetValue(row["table_name"]); + string column_name = GetValue(row["column_name"]); + + constraintFkCatalogBuilder.Append(constraint_catalog); + constraintFkDbSchemaBuilder.Append(constraint_schema); + constraintFkTableBuilder.Append(table_name); + constraintFkColumnNameBuilder.Append(column_name); + + nullBitmapBuffer.Append(true); + length++; + } } List dataArrays = new List diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 90cddf65d9..7dd40a7b96 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -141,7 +141,10 @@ private IArrowType TranslateType(TableFieldSchema field) return GetType(field, new Decimal128Type(38, 9)); case "BIGNUMERIC" or "BIGDECIMAL": - return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38)); + if (this.Options != null) + return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38)); + else + return GetType(field, StringType.Default); default: throw new InvalidOperationException($"{field.Type} cannot be translated"); } From 4b1cac0403424d03f99e377984930a1b923cc702 Mon Sep 17 00:00:00 2001 From: David Coe Date: Tue, 23 Jan 2024 12:03:41 -0500 Subject: [PATCH 10/14] serialize complex struct; remove extra Field parameter --- csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs | 3 +- csharp/src/Client/AdbcDataReader.cs | 2 +- .../src/Drivers/BigQuery/BigQueryStatement.cs | 116 +++++++++++++++++- .../Client/ClientTests.cs | 2 +- csharp/test/Drivers/BigQuery/BigQueryData.cs | 57 +++++++++ csharp/test/Drivers/BigQuery/DriverTests.cs | 5 + 6 files changed, 176 insertions(+), 9 deletions(-) diff --git a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs index 9e42544eac..91486be475 100644 --- a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs +++ b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs @@ -147,10 +147,9 @@ public virtual void Dispose() /// /// The index in the array to get the value from. /// - public virtual object GetValue(IArrowArray arrowArray, Field field, int index) + public virtual object GetValue(IArrowArray arrowArray, int index) { if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); - if (field == null) throw new ArgumentNullException(nameof(field)); if (index < 0) throw new ArgumentOutOfRangeException(nameof(index)); switch (arrowArray) diff --git a/csharp/src/Client/AdbcDataReader.cs b/csharp/src/Client/AdbcDataReader.cs index 6e9a7ae49c..c58adc450e 100644 --- a/csharp/src/Client/AdbcDataReader.cs +++ b/csharp/src/Client/AdbcDataReader.cs @@ -347,7 +347,7 @@ public ReadOnlyCollection GetAdbcColumnSchema() public object GetValue(IArrowArray arrowArray, int ordinal) { Field field = this.schema.GetFieldByIndex(ordinal); - return this.adbcCommand.AdbcStatement.GetValue(arrowArray, field, this.currentRowInRecordBatch); + return this.adbcCommand.AdbcStatement.GetValue(arrowArray, this.currentRowInRecordBatch); } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 7dd40a7b96..617632df88 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -16,7 +16,10 @@ */ using System; +using System.Collections; using System.Collections.Generic; +using System.Data.SqlTypes; +using System.Diagnostics; using System.IO; using System.Linq; using System.Text.Json; @@ -87,7 +90,7 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } - public override object GetValue(IArrowArray arrowArray, Field field, int index) + public override object GetValue(IArrowArray arrowArray, int index) { switch (arrowArray) { @@ -96,7 +99,7 @@ public override object GetValue(IArrowArray arrowArray, Field field, int index) case ListArray listArray: return listArray.GetSlicedValues(index); default: - return base.GetValue(arrowArray, field, index); + return base.GetValue(arrowArray, index); } } @@ -192,17 +195,120 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName) } private string SerializeToJson(StructArray structArray, int index) + { + Dictionary jsonDictionary = ParseStructArray(structArray, index); + + return JsonSerializer.Serialize(jsonDictionary); + } + + private Dictionary ParseStructArray(StructArray structArray, int index) { Dictionary jsonDictionary = new Dictionary(); StructType structType = (StructType)structArray.Data.DataType; for (int i = 0; i < structArray.Data.Children.Length; i++) { - jsonDictionary.Add(structType.Fields[i].Name, - GetValue(structArray.Fields[i], structType.GetFieldByName(structType.Fields[i].Name), index)); + string name = structType.Fields[i].Name; + object value = GetValue(structArray.Fields[i], index); + + if (value is StructArray structArray1) + { + jsonDictionary.Add(name, ParseStructArray(structArray1, index)); + } + else if (value is IArrowArray arrowArray) + { + IList? values = CreateList(arrowArray); + + if (values != null) + { + for (int j = 0; j < arrowArray.Length; j++) + { + values.Add(base.GetValue(arrowArray, j)); + } + + jsonDictionary.Add(name, values); + } + else + { + jsonDictionary.Add(name, new List()); + } + } + else + { + jsonDictionary.Add(name, value); + } + } + + return jsonDictionary; + } + + private IList? CreateList(IArrowArray arrowArray) + { + if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); + + switch (arrowArray) + { + case BooleanArray booleanArray: + return new List(); + case Date32Array date32Array: + case Date64Array date64Array: + return new List(); + case Decimal128Array decimal128Array: + return new List(); + case Decimal256Array decimal256Array: + return new List(); + case DoubleArray doubleArray: + return new List(); + case FloatArray floatArray: + return new List(); +#if NET5_0_OR_GREATER + case PrimitiveArray halfFloatArray: + return new List(); +#endif + case Int8Array int8Array: + return new List(); + case Int16Array int16Array: + return new List(); + case Int32Array int32Array: + return new List(); + case Int64Array int64Array: + return new List(); + case StringArray stringArray: + return new List(); +#if NET6_0_OR_GREATER + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#else + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#endif + case TimestampArray timestampArray: + return new List(); + case UInt8Array uInt8Array: + return new List(); + case UInt16Array uInt16Array: + return new List(); + case UInt32Array uInt32Array: + return new List(); + case UInt64Array uInt64Array: + return new List(); + + case BinaryArray binaryArray: + return new List(); + + // not covered: + // -- struct array + // -- dictionary array + // -- fixed size binary + // -- list array + // -- union array } - return JsonSerializer.Serialize(jsonDictionary); + + return null; } + class MultiArrowReader : IArrowArrayStream { readonly Schema schema; diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs index 9548b85e4a..423f2f819b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs @@ -70,7 +70,7 @@ private AdbcDataReader GetMoqDataReader(DecimalBehavior decimalBehavior, string Mock mockStatement = new Mock(); mockStatement.Setup(x => x.ExecuteQuery()).Returns(queryResult); ; - mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny(), It.IsAny())).Returns(sqlDecimal); + mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny())).Returns(sqlDecimal); Adbc.Client.AdbcConnection mockConnection = new Adbc.Client.AdbcConnection(); mockConnection.DecimalBehavior = decimalBehavior; diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index f23e4ce2ec..faf642b5f5 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using System.Data.SqlTypes; using System.Text; +using System.Xml.Linq; using Apache.Arrow.Types; namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery @@ -146,6 +147,62 @@ public static SampleDataBuilder GetSampleData() } }); + // complex struct + sampleDataBuilder.Samples.Add( + new SampleData() + { + Query = "SELECT " + + "STRUCT(" + + "\"Iron Man\" as name," + + "\"Avengers\" as team," + + "[\"Genius\", \"Billionaire\", \"Playboy\", \"Philanthropist\"] as powers," + + "[" + + " STRUCT(" + + " \"Captain America\" as name, " + + " \"Avengers\" as team, " + + " [\"Super Soldier Serum\", \"Vibranium Shield\"] as powers, " + + " [" + + " STRUCT(" + + " \"Thanos\" as name, " + + " \"Black Order\" as name, " + + " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " + + " [" + + " STRUCT(" + + " \"Loki\" as name, " + + " \"Asgard\" as team, " + + " [\"Magic\", \"Shapeshifting\", \"Trickery\"] as powers " + + " )" + + " ] as allies" + + " )" + + " ] as enemies" + + " )," + + " STRUCT(" + + " \"Spider-Man\" as name, " + + " \"Avengers\" as team, " + + " [\"Spider-Sense\", \"Web-Shooting\", \"Wall-Crawling\"] as powers, " + + " [" + + " STRUCT(" + + " \"Green Goblin\" as name, " + + " \"Sinister Six\" as team, " + + " [\"Glider\", \"Pumpkin Bombs\", \"Super Strength\"] as powers, " + + " [" + + " STRUCT(" + + " \"Doctor Octopus\" as name, " + + " \"Sinister Six\" as team, " + + " [\"Mechanical Arms\", \"Genius\", \"Madness\"] as powers " + + " )" + + " ] as allies" + + " )" + + " ] as enemies" + + " )" + + " ] as friends" + + ") as iron_man", + ExpectedValues = new List() + { + new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"_field_2\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}}}}"), + } + }); + return sampleDataBuilder; } } diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 2af6de337b..6c6d558b8a 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,9 +18,14 @@ using System; using System.Collections.Generic; using System.Linq; +using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Google.Apis.Auth.OAuth2; +using Google.Cloud.BigQuery.V2; +using Moq; using Xunit; namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery From a720fad748ee6d5f7032cca9b027c7bd7d10b377 Mon Sep 17 00:00:00 2001 From: David Coe Date: Tue, 23 Jan 2024 14:49:32 -0500 Subject: [PATCH 11/14] PR feedback --- .../Drivers/BigQuery/BigQueryConnection.cs | 20 +++++-------- .../Apache.Arrow.Adbc.Tests/ClientTests.cs | 28 ++----------------- csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs | 4 +-- .../Drivers/BigQuery/BigQueryTestingUtils.cs | 2 +- .../Snowflake/SnowflakeTestingUtils.cs | 4 +-- 5 files changed, 15 insertions(+), 43 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 668d7bf311..ef68ef4e34 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -70,10 +70,10 @@ public BigQueryConnection(IReadOnlyDictionary properties) /// internal void Open() { - string? projectId = string.Empty; - string? clientId = string.Empty; - string? clientSecret = string.Empty; - string? refreshToken = string.Empty; + string? projectId = null; + string? clientId = null; + string? clientSecret = null; + string? refreshToken = null; string tokenEndpoint = BigQueryConstants.TokenEndpoint; @@ -84,7 +84,7 @@ internal void Open() if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out projectId)) throw new ArgumentException($"The {BigQueryParameters.ProjectId} parameter is not present"); - if(this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType)) + if (this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType)) { if (!authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase) && !authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType, StringComparison.OrdinalIgnoreCase)) @@ -134,7 +134,7 @@ private GoogleCredential ApplyScopes(GoogleCredential credential) { IEnumerable parsedScopes = scopes.Split(',').Where(x => x.Length > 0); - return credential.CreateScoped(); + return credential.CreateScoped(parsedScopes); } } @@ -828,18 +828,12 @@ private string GetValue(object value) switch (value) { case string sValue: - if (string.IsNullOrEmpty(sValue)) - return string.Empty; return sValue; default: if (value != null) { string? sValue = value.ToString(); - - if (string.IsNullOrEmpty(sValue)) - return string.Empty; - - return sValue; + return sValue ?? string.Empty; } throw new InvalidOperationException($"Cannot parse {value}"); } diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs index fe3bad0449..85eceb27aa 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs @@ -137,8 +137,8 @@ public static void CanClientExecuteQuery(Adbc.Client.AdbcConnection adbcConnecti /// The to use public static void VerifyTypesAndValues(Adbc.Client.AdbcConnection adbcConnection, SampleDataBuilder sampleDataBuilder) { - if(adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection)); - if(sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder)); + if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection)); + if (sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder)); adbcConnection.Open(); @@ -212,29 +212,7 @@ static void AssertTypeAndValue( if (value != null) { - if (value is IList && !(value is byte[])) - { - IList list = value as IList; - IList expectedList = ctv.ExpectedValue as IList; - int i = -1; - - foreach (var actual in list) - { - i++; - int j = -1; - - foreach (var expected in expectedList) - { - j++; - - if (i == j) - { - Assert.True(expected.Equals(actual), $"Expected value does not match actual value for {ctv.Name} at {i} for query [{query}]"); - } - } - } - } - else if (!value.GetType().BaseType.Name.Contains("PrimitiveArray")) + if (!value.GetType().BaseType.Name.Contains("PrimitiveArray")) { Assert.True(ctv.ExpectedNetType == value.GetType(), $"Expected type does not match actual type for {ctv.Name} for query [{query}]"); diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs index 343719bc49..1d695480b8 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs @@ -75,7 +75,7 @@ public static bool CanExecuteTest(string environmentVariable, out string environ public static T LoadTestConfiguration(string environmentVariable) where T : TestConfiguration { - if(CanExecuteTest(environmentVariable, out string environmentValue)) + if (CanExecuteTest(environmentVariable, out string environmentValue)) return GetTestConfiguration(environmentValue); throw new InvalidOperationException($"Cannot execute test configuration from environment variable `{environmentVariable}`"); @@ -92,7 +92,7 @@ public static T LoadTestConfiguration(string environmentVariable) public static T GetTestConfiguration(string fileName) where T : TestConfiguration { - if(!File.Exists(fileName)) + if (!File.Exists(fileName)) throw new FileNotFoundException(fileName); // use a JSON file for the various settings diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 2b0518834f..874ccac791 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -56,7 +56,7 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestConf { BigQueryParameters.ProjectId, testConfiguration.ProjectId }, }; - if(!string.IsNullOrEmpty(testConfiguration.JsonCredential)) + if (!string.IsNullOrEmpty(testConfiguration.JsonCredential)) { parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.ServiceAccountAuthenticationType); parameters.Add(BigQueryParameters.JsonCredential, testConfiguration.JsonCredential); diff --git a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs index d1e7b9dc74..2f394a89a1 100644 --- a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs +++ b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs @@ -81,14 +81,14 @@ out Dictionary parameters { SnowflakeParameters.USE_HIGH_PRECISION, testConfiguration.UseHighPrecision.ToString().ToLowerInvariant() } }; - if(testConfiguration.Authentication.Default is not null) + if (testConfiguration.Authentication.Default is not null) { parameters[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthSnowflake; parameters[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.Default.User; parameters[SnowflakeParameters.PASSWORD] = testConfiguration.Authentication.Default.Password; } - if(!string.IsNullOrWhiteSpace(testConfiguration.Host)) + if (!string.IsNullOrWhiteSpace(testConfiguration.Host)) { parameters[SnowflakeParameters.HOST] = testConfiguration.Host; } From d326cfb5f03d64ee3053d2a47946d79e17eb0989 Mon Sep 17 00:00:00 2001 From: David Coe Date: Tue, 23 Jan 2024 19:17:05 -0500 Subject: [PATCH 12/14] fix list of structs serialization --- .../src/Drivers/BigQuery/BigQueryStatement.cs | 19 +++++++++++++++++-- csharp/test/Drivers/BigQuery/BigQueryData.cs | 4 ++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 617632df88..c64abb984a 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -95,7 +95,7 @@ public override object GetValue(IArrowArray arrowArray, int index) switch (arrowArray) { case StructArray structArray: - return SerializeToJson(structArray, index); + return SerializeToJson(structArray, index); case ListArray listArray: return listArray.GetSlicedValues(index); default: @@ -212,7 +212,22 @@ private Dictionary ParseStructArray(StructArray structArray, int if (value is StructArray structArray1) { - jsonDictionary.Add(name, ParseStructArray(structArray1, index)); + List> children = new List>(); + + if (structArray1.Length > 1) + { + for (int j = 0; j < structArray1.Length; j++) + children.Add(ParseStructArray(structArray1, j)); + } + + if (children.Count > 0) + { + jsonDictionary.Add(name, children); + } + else + { + jsonDictionary.Add(name, ParseStructArray(structArray1, index)); + } } else if (value is IArrowArray arrowArray) { diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index faf642b5f5..fc9acd76b2 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -164,7 +164,7 @@ public static SampleDataBuilder GetSampleData() " [" + " STRUCT(" + " \"Thanos\" as name, " + - " \"Black Order\" as name, " + + " \"Black Order\" as team, " + " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " + " [" + " STRUCT(" + @@ -199,7 +199,7 @@ public static SampleDataBuilder GetSampleData() ") as iron_man", ExpectedValues = new List() { - new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"_field_2\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}}}}"), + new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":[{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"team\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}}},{\"name\":\"Spider-Man\",\"team\":\"Avengers\",\"powers\":[\"Spider-Sense\",\"Web-Shooting\",\"Wall-Crawling\"],\"enemies\":{\"name\":\"Green Goblin\",\"team\":\"Sinister Six\",\"powers\":[\"Glider\",\"Pumpkin Bombs\",\"Super Strength\"],\"allies\":{\"name\":\"Doctor Octopus\",\"team\":\"Sinister Six\",\"powers\":[\"Mechanical Arms\",\"Genius\",\"Madness\"]}}}]}") } }); From 1eb0317d276e8ec382291e6d58d3a06f7da816c4 Mon Sep 17 00:00:00 2001 From: David Coe Date: Thu, 25 Jan 2024 12:41:04 -0500 Subject: [PATCH 13/14] add precision and scale for Decimal128 and Decimal256 types --- csharp/src/Client/SchemaConverter.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/csharp/src/Client/SchemaConverter.cs b/csharp/src/Client/SchemaConverter.cs index 656cd0a789..0573e2247c 100644 --- a/csharp/src/Client/SchemaConverter.cs +++ b/csharp/src/Client/SchemaConverter.cs @@ -73,6 +73,16 @@ public static DataTable ConvertArrowSchema(Schema schema, AdbcStatement adbcStat row[SchemaTableColumn.NumericPrecision] = Convert.ToInt32(f.Metadata["precision"]); row[SchemaTableColumn.NumericScale] = Convert.ToInt32(f.Metadata["scale"]); } + else if (f.DataType is Decimal128Type decimal128Type) + { + row[SchemaTableColumn.NumericPrecision] = decimal128Type.Precision; + row[SchemaTableColumn.NumericScale] = decimal128Type.Scale; + } + else if (f.DataType is Decimal256Type decimal256Type) + { + row[SchemaTableColumn.NumericPrecision] = decimal256Type.Precision; + row[SchemaTableColumn.NumericScale] = decimal256Type.Scale; + } else { row[SchemaTableColumn.NumericPrecision] = DBNull.Value; From a01fc7c8edb4a716eb5b4effd825ee8ae2e5f41d Mon Sep 17 00:00:00 2001 From: David Coe Date: Fri, 26 Jan 2024 12:46:54 -0500 Subject: [PATCH 14/14] PR feedback --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 3 +-- csharp/test/Drivers/BigQuery/BigQueryData.cs | 1 - csharp/test/Drivers/BigQuery/ClientTests.cs | 2 +- csharp/test/Drivers/BigQuery/DriverTests.cs | 5 ----- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index c64abb984a..389e462e55 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -19,7 +19,6 @@ using System.Collections; using System.Collections.Generic; using System.Data.SqlTypes; -using System.Diagnostics; using System.IO; using System.Linq; using System.Text.Json; @@ -95,7 +94,7 @@ public override object GetValue(IArrowArray arrowArray, int index) switch (arrowArray) { case StructArray structArray: - return SerializeToJson(structArray, index); + return SerializeToJson(structArray, index); case ListArray listArray: return listArray.GetSlicedValues(index); default: diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index fc9acd76b2..010cbe0f1b 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -19,7 +19,6 @@ using System.Collections.Generic; using System.Data.SqlTypes; using System.Text; -using System.Xml.Linq; using Apache.Arrow.Types; namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index 65d55a06b6..94b28833dd 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -26,7 +26,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery /// /// Class for testing the ADBC Client using the BigQuery ADBC driver. /// - /// /// + /// /// Tests are ordered to ensure data is created for the other /// queries to run. /// diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index 6c6d558b8a..2af6de337b 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -18,14 +18,9 @@ using System; using System.Collections.Generic; using System.Linq; -using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Metadata; using Apache.Arrow.Adbc.Tests.Xunit; using Apache.Arrow.Ipc; -using Apache.Arrow.Types; -using Google.Apis.Auth.OAuth2; -using Google.Cloud.BigQuery.V2; -using Moq; using Xunit; namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery