diff --git a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs index 9e42544eac..91486be475 100644 --- a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs +++ b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs @@ -147,10 +147,9 @@ public virtual void Dispose() /// /// The index in the array to get the value from. /// - public virtual object GetValue(IArrowArray arrowArray, Field field, int index) + public virtual object GetValue(IArrowArray arrowArray, int index) { if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); - if (field == null) throw new ArgumentNullException(nameof(field)); if (index < 0) throw new ArgumentOutOfRangeException(nameof(index)); switch (arrowArray) diff --git a/csharp/src/Client/AdbcDataReader.cs b/csharp/src/Client/AdbcDataReader.cs index 6e9a7ae49c..c58adc450e 100644 --- a/csharp/src/Client/AdbcDataReader.cs +++ b/csharp/src/Client/AdbcDataReader.cs @@ -347,7 +347,7 @@ public ReadOnlyCollection GetAdbcColumnSchema() public object GetValue(IArrowArray arrowArray, int ordinal) { Field field = this.schema.GetFieldByIndex(ordinal); - return this.adbcCommand.AdbcStatement.GetValue(arrowArray, field, this.currentRowInRecordBatch); + return this.adbcCommand.AdbcStatement.GetValue(arrowArray, this.currentRowInRecordBatch); } /// diff --git a/csharp/src/Client/SchemaConverter.cs b/csharp/src/Client/SchemaConverter.cs index 656cd0a789..0573e2247c 100644 --- a/csharp/src/Client/SchemaConverter.cs +++ b/csharp/src/Client/SchemaConverter.cs @@ -73,6 +73,16 @@ public static DataTable ConvertArrowSchema(Schema schema, AdbcStatement adbcStat row[SchemaTableColumn.NumericPrecision] = Convert.ToInt32(f.Metadata["precision"]); row[SchemaTableColumn.NumericScale] = Convert.ToInt32(f.Metadata["scale"]); } + else if (f.DataType is Decimal128Type decimal128Type) + { + row[SchemaTableColumn.NumericPrecision] = decimal128Type.Precision; + row[SchemaTableColumn.NumericScale] = decimal128Type.Scale; + } + else if (f.DataType is Decimal256Type decimal256Type) + { + row[SchemaTableColumn.NumericPrecision] = decimal256Type.Precision; + row[SchemaTableColumn.NumericScale] = decimal256Type.Scale; + } else { row[SchemaTableColumn.NumericPrecision] = DBNull.Value; diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj index 3bb39e99ec..0ce2c1fe26 100644 --- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj +++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj @@ -2,6 +2,7 @@ netstandard2.0;net6.0 readme.md + enable diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index 8e599231a8..ef68ef4e34 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -70,24 +70,22 @@ public BigQueryConnection(IReadOnlyDictionary properties) /// internal void Open() { - string projectId = string.Empty; - string clientId = string.Empty; - string clientSecret = string.Empty; - string refreshToken = string.Empty; + string? projectId = null; + string? clientId = null; + string? clientSecret = null; + string? refreshToken = null; string tokenEndpoint = BigQueryConstants.TokenEndpoint; - string authenticationType = BigQueryConstants.UserAuthenticationType; + string? authenticationType = BigQueryConstants.UserAuthenticationType; // TODO: handle token expiration if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out projectId)) throw new ArgumentException($"The {BigQueryParameters.ProjectId} parameter is not present"); - if (this.properties.ContainsKey(BigQueryParameters.AuthenticationType)) + if (this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType)) { - this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType); - if (!authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase) && !authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType, StringComparison.OrdinalIgnoreCase)) { @@ -95,7 +93,7 @@ internal void Open() } } - if (authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(authenticationType) && authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase)) { if (!this.properties.TryGetValue(BigQueryParameters.ClientId, out clientId)) throw new ArgumentException($"The {BigQueryParameters.ClientId} parameter is not present"); @@ -106,21 +104,43 @@ internal void Open() if (!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out refreshToken)) throw new ArgumentException($"The {BigQueryParameters.RefreshToken} parameter is not present"); - this.credential = GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint)); + this.credential = ApplyScopes(GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint))); } else { - string json = string.Empty; + string? json = string.Empty; if (!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json)) throw new ArgumentException($"The {BigQueryParameters.JsonCredential} parameter is not present"); - this.credential = GoogleCredential.FromJson(json); + this.credential = ApplyScopes(GoogleCredential.FromJson(json)); } this.client = BigQueryClient.Create(projectId, this.credential); } + /// + /// Apply any additional scopes to the credential. + /// + /// + /// + private GoogleCredential ApplyScopes(GoogleCredential credential) + { + if (credential == null) throw new ArgumentNullException(nameof(credential)); + + if (this.properties.TryGetValue(BigQueryParameters.Scopes, out string? scopes)) + { + if (!string.IsNullOrEmpty(scopes)) + { + IEnumerable parsedScopes = scopes.Split(',').Where(x => x.Length > 0); + + return credential.CreateScoped(parsedScopes); + } + } + + return credential; + } + public override IArrowArrayStream GetInfo(List codes) { const int strValTypeID = 0; @@ -261,23 +281,26 @@ private List GetCatalogs( StringArray.Builder catalogNameBuilder = new StringArray.Builder(); List catalogDbSchemasValues = new List(); string catalogRegexp = PatternToRegEx(catalogPattern); - PagedEnumerable catalogs = this.client.ListProjects(); + PagedEnumerable? catalogs = this.client?.ListProjects(); - foreach (CloudProject catalog in catalogs) + if (catalogs != null) { - if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase)) + foreach (CloudProject catalog in catalogs) { - catalogNameBuilder.Append(catalog.ProjectId); - - if (depth == GetObjectsDepth.Catalogs) - { - catalogDbSchemasValues.Add(null); - } - else + if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase)) { - catalogDbSchemasValues.Add(GetDbSchemas( - depth, catalog.ProjectId, dbSchemaPattern, - tableNamePattern, tableTypes, columnNamePattern)); + catalogNameBuilder.Append(catalog.ProjectId); + + if (depth == GetObjectsDepth.Catalogs) + { + catalogDbSchemasValues.Add(null); + } + else + { + catalogDbSchemasValues.Add(GetDbSchemas( + depth, catalog.ProjectId, dbSchemaPattern, + tableNamePattern, tableTypes, columnNamePattern)); + } } } } @@ -306,25 +329,28 @@ private StructArray GetDbSchemas( string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern); - PagedEnumerable schemas = this.client.ListDatasets(catalog); + PagedEnumerable? schemas = this.client?.ListDatasets(catalog); - foreach (BigQueryDataset schema in schemas) + if (schemas != null) { - if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase)) + foreach (BigQueryDataset schema in schemas) { - dbSchemaNameBuilder.Append(schema.Reference.DatasetId); - length++; - nullBitmapBuffer.Append(true); - - if (depth == GetObjectsDepth.DbSchemas) - { - dbSchemaTablesValues.Add(null); - } - else + if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase)) { - dbSchemaTablesValues.Add(GetTableSchemas( - depth, catalog, schema.Reference.DatasetId, - tableNamePattern, tableTypes, columnNamePattern)); + dbSchemaNameBuilder.Append(schema.Reference.DatasetId); + length++; + nullBitmapBuffer.Append(true); + + if (depth == GetObjectsDepth.DbSchemas) + { + dbSchemaTablesValues.Add(null); + } + else + { + dbSchemaTablesValues.Add(GetTableSchemas( + depth, catalog, schema.Reference.DatasetId, + tableNamePattern, tableTypes, columnNamePattern)); + } } } } @@ -379,25 +405,28 @@ private StructArray GetTableSchemas( } } - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - tableNameBuilder.Append(row["table_name"].ToString()); - tableTypeBuilder.Append(row["table_type"].ToString()); - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + tableNameBuilder.Append(GetValue(row["table_name"])); + tableTypeBuilder.Append(GetValue(row["table_type"])); + nullBitmapBuffer.Append(true); + length++; - tableConstraintsValues.Add(GetConstraintSchema( - depth, catalog, dbSchema, row["table_name"].ToString(), columnNamePattern)); + tableConstraintsValues.Add(GetConstraintSchema( + depth, catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern)); - if (depth == GetObjectsDepth.Tables) - { - tableColumnsValues.Add(null); - } - else - { - tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, row["table_name"].ToString(), columnNamePattern)); + if (depth == GetObjectsDepth.Tables) + { + tableColumnsValues.Add(null); + } + else + { + tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern)); + } } } @@ -452,46 +481,48 @@ private StructArray GetColumnSchema( query = string.Concat(query, string.Format("AND column_name LIKE '{0}'", Sanitize(columnNamePattern))); } - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - columnNameBuilder.Append(row["column_name"].ToString()); - ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); - remarksBuilder.Append(""); + foreach (BigQueryRow row in result) + { + columnNameBuilder.Append(GetValue(row["column_name"])); + ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]); + remarksBuilder.Append(""); - string dataType = ToTypeName(row["data_type"].ToString()); + string dataType = ToTypeName(GetValue(row["data_type"])); - if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) - { - ParsedDecimalValues values = ParsePrecisionAndScale(dataType); - xdbcColumnSizeBuilder.Append(values.Precision); - xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); - } - else - { - xdbcColumnSizeBuilder.AppendNull(); - xdbcDecimalDigitsBuilder.AppendNull(); - } + if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL")) + { + ParsedDecimalValues values = ParsePrecisionAndScale(dataType); + xdbcColumnSizeBuilder.Append(values.Precision); + xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale)); + } + else + { + xdbcColumnSizeBuilder.AppendNull(); + xdbcDecimalDigitsBuilder.AppendNull(); + } - xdbcDataTypeBuilder.AppendNull(); - xdbcTypeNameBuilder.Append(dataType); - xdbcNumPrecRadixBuilder.AppendNull(); - xdbcNullableBuilder.AppendNull(); - xdbcColumnDefBuilder.AppendNull(); - xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); - xdbcDatetimeSubBuilder.AppendNull(); - xdbcCharOctetLengthBuilder.AppendNull(); - xdbcIsNullableBuilder.Append(row["is_nullable"].ToString()); - xdbcScopeCatalogBuilder.AppendNull(); - xdbcScopeSchemaBuilder.AppendNull(); - xdbcScopeTableBuilder.AppendNull(); - xdbcIsAutoincrementBuilder.AppendNull(); - xdbcIsGeneratedcolumnBuilder.Append(row["is_generated"].ToString().ToUpper() == "YES"); - nullBitmapBuffer.Append(true); - length++; + xdbcDataTypeBuilder.AppendNull(); + xdbcTypeNameBuilder.Append(dataType); + xdbcNumPrecRadixBuilder.AppendNull(); + xdbcNullableBuilder.AppendNull(); + xdbcColumnDefBuilder.AppendNull(); + xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType)); + xdbcDatetimeSubBuilder.AppendNull(); + xdbcCharOctetLengthBuilder.AppendNull(); + xdbcIsNullableBuilder.Append(row["is_nullable"].ToString()); + xdbcScopeCatalogBuilder.AppendNull(); + xdbcScopeSchemaBuilder.AppendNull(); + xdbcScopeTableBuilder.AppendNull(); + xdbcIsAutoincrementBuilder.AppendNull(); + xdbcIsGeneratedcolumnBuilder.Append(GetValue(row["is_generated"]).ToUpper() == "YES"); + nullBitmapBuffer.Append(true); + length++; + } } - List dataArrays = new List { columnNameBuilder.Build(), @@ -539,36 +570,39 @@ private StructArray GetConstraintSchema( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(table)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - string constraintName = row["constraint_name"].ToString(); - constraintNameBuilder.Append(constraintName); - string constraintType = row["constraint_type"].ToString(); - constraintTypeBuilder.Append(constraintType); - nullBitmapBuffer.Append(true); - length++; - - if (depth == GetObjectsDepth.All || depth == GetObjectsDepth.Tables) + foreach (BigQueryRow row in result) { - constraintColumnNamesValues.Add(GetConstraintColumnNames( - catalog, dbSchema, table, constraintName)); - if (constraintType.ToUpper() == "FOREIGN KEY") + string constraintName = GetValue(row["constraint_name"]); + constraintNameBuilder.Append(constraintName); + string constraintType = GetValue(row["constraint_type"]); + constraintTypeBuilder.Append(constraintType); + nullBitmapBuffer.Append(true); + length++; + + if (depth == GetObjectsDepth.All || depth == GetObjectsDepth.Tables) { - constraintColumnUsageValues.Add(GetConstraintsUsage( + constraintColumnNamesValues.Add(GetConstraintColumnNames( catalog, dbSchema, table, constraintName)); + if (constraintType.ToUpper() == "FOREIGN KEY") + { + constraintColumnUsageValues.Add(GetConstraintsUsage( + catalog, dbSchema, table, constraintName)); + } + else + { + constraintColumnUsageValues.Add(null); + } } else { + constraintColumnNamesValues.Add(null); constraintColumnUsageValues.Add(null); } } - else - { - constraintColumnNamesValues.Add(null); - constraintColumnUsageValues.Add(null); - } } List dataArrays = new List @@ -597,12 +631,15 @@ private StringArray GetConstraintColumnNames( StringArray.Builder constraintColumnNamesBuilder = new StringArray.Builder(); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - string column = row["column_name"].ToString(); - constraintColumnNamesBuilder.Append(column); + foreach (BigQueryRow row in result) + { + string column = GetValue(row["column_name"]); + constraintColumnNamesBuilder.Append(column); + } } return constraintColumnNamesBuilder.Build(); @@ -624,22 +661,25 @@ private StructArray GetConstraintsUsage( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE WHERE constraint_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(constraintName)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); - foreach (BigQueryRow row in result) + if (result != null) { - string constraint_catalog = row["constraint_catalog"].ToString(); - string constraint_schema = row["constraint_schema"].ToString(); - string table_name = row["table_name"].ToString(); - string column_name = row["column_name"].ToString(); - - constraintFkCatalogBuilder.Append(constraint_catalog); - constraintFkDbSchemaBuilder.Append(constraint_schema); - constraintFkTableBuilder.Append(table_name); - constraintFkColumnNameBuilder.Append(column_name); - - nullBitmapBuffer.Append(true); - length++; + foreach (BigQueryRow row in result) + { + string constraint_catalog = GetValue(row["constraint_catalog"]); + string constraint_schema = GetValue(row["constraint_schema"]); + string table_name = GetValue(row["table_name"]); + string column_name = GetValue(row["column_name"]); + + constraintFkCatalogBuilder.Append(constraint_catalog); + constraintFkDbSchemaBuilder.Append(constraint_schema); + constraintFkTableBuilder.Append(table_name); + constraintFkColumnNameBuilder.Append(column_name); + + nullBitmapBuffer.Append(true); + length++; + } } List dataArrays = new List @@ -748,13 +788,16 @@ public override Schema GetTableSchema(string catalog, string dbSchema, string ta string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(tableName)); - BigQueryResults result = this.client.ExecuteQuery(query, parameters: null); + BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); List fields = new List(); - foreach (BigQueryRow row in result) + if (result != null) { - fields.Add(DescToField(row)); + foreach (BigQueryRow row in result) + { + fields.Add(DescToField(row)); + } } return new Schema(fields, null); @@ -764,22 +807,38 @@ private Field DescToField(BigQueryRow row) { Dictionary metaData = new Dictionary(); metaData.Add("PRIMARY_KEY", ""); - metaData.Add("ORDINAL_POSITION", row["ordinal_position"].ToString()); - metaData.Add("DATA_TYPE", row["data_type"].ToString()); + metaData.Add("ORDINAL_POSITION", GetValue(row["ordinal_position"])); + metaData.Add("DATA_TYPE", GetValue(row["data_type"])); - Field.Builder fieldBuilder = SchemaFieldGenerator(row["column_name"].ToString().ToLower(), row["data_type"].ToString()); + Field.Builder fieldBuilder = SchemaFieldGenerator(GetValue(row["column_name"]).ToLower(), GetValue(row["data_type"])); fieldBuilder.Metadata(metaData); - if (!row["is_nullable"].ToString().Equals("YES", StringComparison.OrdinalIgnoreCase)) + if (!GetValue(row["is_nullable"]).Equals("YES", StringComparison.OrdinalIgnoreCase)) { fieldBuilder.Nullable(false); } - fieldBuilder.Name(row["column_name"].ToString().ToLower()); + fieldBuilder.Name(GetValue(row["column_name"]).ToLower()); return fieldBuilder.Build(); } + private string GetValue(object value) + { + switch (value) + { + case string sValue: + return sValue; + default: + if (value != null) + { + string? sValue = value.ToString(); + return sValue ?? string.Empty; + } + throw new InvalidOperationException($"Cannot parse {value}"); + } + } + private Field.Builder SchemaFieldGenerator(string name, string type) { int index = type.IndexOf("("); @@ -874,7 +933,7 @@ public override IArrowArrayStream GetTableTypes() return new BigQueryInfoArrowStream(StandardSchemas.TableTypesSchema, dataArrays); } - private ListArray CreateNestedListArray(List arrayList, IArrowType dataType) + private ListArray CreateNestedListArray(List arrayList, IArrowType dataType) { ArrowBuffer.Builder valueOffsetsBufferBuilder = new ArrowBuffer.Builder(); ArrowBuffer.BitmapBuilder validityBufferBuilder = new ArrowBuffer.BitmapBuilder(); @@ -882,7 +941,7 @@ private ListArray CreateNestedListArray(List arrayList, IArrowType int length = 0; int nullCount = 0; - foreach (IArrowArray array in arrayList) + foreach (IArrowArray? array in arrayList) { if (array == null) { @@ -902,7 +961,7 @@ private ListArray CreateNestedListArray(List arrayList, IArrowType ArrowBuffer validityBuffer = nullCount > 0 ? validityBufferBuilder.Build() : ArrowBuffer.Empty; - ArrayData data = ArrayDataConcatenator.Concatenate(arrayDataList); + ArrayData? data = ArrayDataConcatenator.Concatenate(arrayDataList); if (data == null) { @@ -970,6 +1029,9 @@ public override void Dispose() private string Sanitize(string input) { + if (string.IsNullOrEmpty(input)) + return string.Empty; + bool isValidInput = sanitizedInputRegex.IsMatch(input); if (isValidInput) @@ -990,7 +1052,7 @@ private string Sanitize(string input) /// /// /// - private string GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint) + private string? GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint) { string body = string.Format( "grant_type=refresh_token&client_id={0}&client_secret={1}&refresh_token={2}", @@ -1006,9 +1068,9 @@ private string GetAccessToken(string clientId, string clientSecret, string refre HttpResponseMessage response = httpClient.SendAsync(request).Result; string responseBody = response.Content.ReadAsStringAsync().Result; - BigQueryTokenResponse bigQueryTokenResponse = JsonSerializer.Deserialize(responseBody); + BigQueryTokenResponse? bigQueryTokenResponse = JsonSerializer.Deserialize(responseBody); - return bigQueryTokenResponse.AccessToken; + return bigQueryTokenResponse?.AccessToken; } enum XdbcDataType @@ -1049,7 +1111,7 @@ private class EmptyArrayCreationVisitor : IArrowTypeVisitor, IArrowTypeVisitor { - public ArrayData Result { get; private set; } + public ArrayData? Result { get; private set; } public void Visit(BooleanType type) { @@ -1074,7 +1136,7 @@ public void Visit(StringType type) public void Visit(ListType type) { type.ValueDataType.Accept(this); - ArrayData child = Result; + ArrayData? child = Result; Result = new ArrayData(type, 0, 0, 0, new[] { ArrowBuffer.Empty, MakeInt0Buffer() }, new[] { child }); } @@ -1082,14 +1144,14 @@ public void Visit(ListType type) public void Visit(FixedSizeListType type) { type.ValueDataType.Accept(this); - ArrayData child = Result; + ArrayData? child = Result; Result = new ArrayData(type, 0, 0, 0, new[] { ArrowBuffer.Empty }, new[] { child }); } public void Visit(StructType type) { - ArrayData[] children = new ArrayData[type.Fields.Count]; + ArrayData?[] children = new ArrayData[type.Fields.Count]; for (int i = 0; i < type.Fields.Count; i++) { type.Fields[i].DataType.Accept(this); diff --git a/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs b/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs index 75a67de8a4..257798ce08 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs @@ -28,7 +28,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery internal class BigQueryInfoArrowStream : IArrowArrayStream { private Schema schema; - private RecordBatch batch; + private RecordBatch? batch; public BigQueryInfoArrowStream(Schema schema, List data) { @@ -38,11 +38,11 @@ public BigQueryInfoArrowStream(Schema schema, List data) public Schema Schema { get { return this.schema; } } - public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) { - RecordBatch batch = this.batch; + RecordBatch? batch = this.batch; this.batch = null; - return new ValueTask(batch); + return new ValueTask(batch); } public void Dispose() diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index 5a12388c3b..ff05036dc4 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -31,6 +31,7 @@ public class BigQueryParameters public const string AllowLargeResults = "adbc.bigquery.allow_large_results"; public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql"; public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string"; + public const string Scopes = "adbc.bigquery.scopes"; } /// diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 009500b864..389e462e55 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -16,7 +16,9 @@ */ using System; +using System.Collections; using System.Collections.Generic; +using System.Data.SqlTypes; using System.IO; using System.Linq; using System.Text.Json; @@ -87,16 +89,16 @@ private Field TranslateField(TableFieldSchema field) return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE"); } - public override object GetValue(IArrowArray arrowArray, Field field, int index) + public override object GetValue(IArrowArray arrowArray, int index) { - switch(arrowArray) + switch (arrowArray) { case StructArray structArray: return SerializeToJson(structArray, index); case ListArray listArray: return listArray.GetSlicedValues(index); default: - return base.GetValue(arrowArray, field, index); + return base.GetValue(arrowArray, index); } } @@ -141,7 +143,10 @@ private IArrowType TranslateType(TableFieldSchema field) return GetType(field, new Decimal128Type(38, 9)); case "BIGNUMERIC" or "BIGDECIMAL": - return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38)); + if (this.Options != null) + return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38)); + else + return GetType(field, StringType.Default); default: throw new InvalidOperationException($"{field.Type} cannot be translated"); } @@ -189,17 +194,135 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName) } private string SerializeToJson(StructArray structArray, int index) + { + Dictionary jsonDictionary = ParseStructArray(structArray, index); + + return JsonSerializer.Serialize(jsonDictionary); + } + + private Dictionary ParseStructArray(StructArray structArray, int index) { Dictionary jsonDictionary = new Dictionary(); StructType structType = (StructType)structArray.Data.DataType; for (int i = 0; i < structArray.Data.Children.Length; i++) { - jsonDictionary.Add(structType.Fields[i].Name, - GetValue(structArray.Fields[i], structType.GetFieldByName(structType.Fields[i].Name), index)); + string name = structType.Fields[i].Name; + object value = GetValue(structArray.Fields[i], index); + + if (value is StructArray structArray1) + { + List> children = new List>(); + + if (structArray1.Length > 1) + { + for (int j = 0; j < structArray1.Length; j++) + children.Add(ParseStructArray(structArray1, j)); + } + + if (children.Count > 0) + { + jsonDictionary.Add(name, children); + } + else + { + jsonDictionary.Add(name, ParseStructArray(structArray1, index)); + } + } + else if (value is IArrowArray arrowArray) + { + IList? values = CreateList(arrowArray); + + if (values != null) + { + for (int j = 0; j < arrowArray.Length; j++) + { + values.Add(base.GetValue(arrowArray, j)); + } + + jsonDictionary.Add(name, values); + } + else + { + jsonDictionary.Add(name, new List()); + } + } + else + { + jsonDictionary.Add(name, value); + } + } + + return jsonDictionary; + } + + private IList? CreateList(IArrowArray arrowArray) + { + if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray)); + + switch (arrowArray) + { + case BooleanArray booleanArray: + return new List(); + case Date32Array date32Array: + case Date64Array date64Array: + return new List(); + case Decimal128Array decimal128Array: + return new List(); + case Decimal256Array decimal256Array: + return new List(); + case DoubleArray doubleArray: + return new List(); + case FloatArray floatArray: + return new List(); +#if NET5_0_OR_GREATER + case PrimitiveArray halfFloatArray: + return new List(); +#endif + case Int8Array int8Array: + return new List(); + case Int16Array int16Array: + return new List(); + case Int32Array int32Array: + return new List(); + case Int64Array int64Array: + return new List(); + case StringArray stringArray: + return new List(); +#if NET6_0_OR_GREATER + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#else + case Time32Array time32Array: + case Time64Array time64Array: + return new List(); +#endif + case TimestampArray timestampArray: + return new List(); + case UInt8Array uInt8Array: + return new List(); + case UInt16Array uInt16Array: + return new List(); + case UInt32Array uInt32Array: + return new List(); + case UInt64Array uInt64Array: + return new List(); + + case BinaryArray binaryArray: + return new List(); + + // not covered: + // -- struct array + // -- dictionary array + // -- fixed size binary + // -- list array + // -- union array } - return JsonSerializer.Serialize(jsonDictionary); + + return null; } + class MultiArrowReader : IArrowArrayStream { readonly Schema schema; diff --git a/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs b/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs index ee6f9802d3..8df4b34c67 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs @@ -25,15 +25,15 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery internal class BigQueryTokenResponse { [JsonPropertyName("access_token")] - public string AccessToken { get; set; } + public string? AccessToken { get; set; } [JsonPropertyName("expires_in")] public int ExpiresIn { get; set; } [JsonPropertyName("scope")] - public string Scope { get; set; } + public string? Scope { get; set; } [JsonPropertyName("token_type")] - public string TokenType { get; set; } + public string? TokenType { get; set; } } } diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 310cf32dae..611374c585 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -55,6 +55,9 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.refresh_token**
    The refresh token used for when the generated OAuth token expires. Required for `user` authentication. +**adbc.bigquery.scopes**
+    Optional. Comma separated list of scopes to include for the credential. + **adbc.bigquery.use_legacy_sql**
    Sets the [UseLegacySql](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_UseLegacySql) value of the QueryOptions to `true` if configured; otherwise, the default is `false`. diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs index 9548b85e4a..423f2f819b 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs @@ -70,7 +70,7 @@ private AdbcDataReader GetMoqDataReader(DecimalBehavior decimalBehavior, string Mock mockStatement = new Mock(); mockStatement.Setup(x => x.ExecuteQuery()).Returns(queryResult); ; - mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny(), It.IsAny())).Returns(sqlDecimal); + mockStatement.Setup(x => x.GetValue(It.IsAny(), It.IsAny())).Returns(sqlDecimal); Adbc.Client.AdbcConnection mockConnection = new Adbc.Client.AdbcConnection(); mockConnection.DecimalBehavior = decimalBehavior; diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs index cb55a94d9f..85eceb27aa 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs @@ -137,8 +137,8 @@ public static void CanClientExecuteQuery(Adbc.Client.AdbcConnection adbcConnecti /// The to use public static void VerifyTypesAndValues(Adbc.Client.AdbcConnection adbcConnection, SampleDataBuilder sampleDataBuilder) { - if(adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection)); - if(sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder)); + if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection)); + if (sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder)); adbcConnection.Open(); diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs index 343719bc49..1d695480b8 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs @@ -75,7 +75,7 @@ public static bool CanExecuteTest(string environmentVariable, out string environ public static T LoadTestConfiguration(string environmentVariable) where T : TestConfiguration { - if(CanExecuteTest(environmentVariable, out string environmentValue)) + if (CanExecuteTest(environmentVariable, out string environmentValue)) return GetTestConfiguration(environmentValue); throw new InvalidOperationException($"Cannot execute test configuration from environment variable `{environmentVariable}`"); @@ -92,7 +92,7 @@ public static T LoadTestConfiguration(string environmentVariable) public static T GetTestConfiguration(string fileName) where T : TestConfiguration { - if(!File.Exists(fileName)) + if (!File.Exists(fileName)) throw new FileNotFoundException(fileName); // use a JSON file for the various settings diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs index f23e4ce2ec..010cbe0f1b 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryData.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs @@ -146,6 +146,62 @@ public static SampleDataBuilder GetSampleData() } }); + // complex struct + sampleDataBuilder.Samples.Add( + new SampleData() + { + Query = "SELECT " + + "STRUCT(" + + "\"Iron Man\" as name," + + "\"Avengers\" as team," + + "[\"Genius\", \"Billionaire\", \"Playboy\", \"Philanthropist\"] as powers," + + "[" + + " STRUCT(" + + " \"Captain America\" as name, " + + " \"Avengers\" as team, " + + " [\"Super Soldier Serum\", \"Vibranium Shield\"] as powers, " + + " [" + + " STRUCT(" + + " \"Thanos\" as name, " + + " \"Black Order\" as team, " + + " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " + + " [" + + " STRUCT(" + + " \"Loki\" as name, " + + " \"Asgard\" as team, " + + " [\"Magic\", \"Shapeshifting\", \"Trickery\"] as powers " + + " )" + + " ] as allies" + + " )" + + " ] as enemies" + + " )," + + " STRUCT(" + + " \"Spider-Man\" as name, " + + " \"Avengers\" as team, " + + " [\"Spider-Sense\", \"Web-Shooting\", \"Wall-Crawling\"] as powers, " + + " [" + + " STRUCT(" + + " \"Green Goblin\" as name, " + + " \"Sinister Six\" as team, " + + " [\"Glider\", \"Pumpkin Bombs\", \"Super Strength\"] as powers, " + + " [" + + " STRUCT(" + + " \"Doctor Octopus\" as name, " + + " \"Sinister Six\" as team, " + + " [\"Mechanical Arms\", \"Genius\", \"Madness\"] as powers " + + " )" + + " ] as allies" + + " )" + + " ] as enemies" + + " )" + + " ] as friends" + + ") as iron_man", + ExpectedValues = new List() + { + new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":[{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"team\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name\":\"Loki\",\"team\":\"Asgard\",\"powers\":[\"Magic\",\"Shapeshifting\",\"Trickery\"]}}},{\"name\":\"Spider-Man\",\"team\":\"Avengers\",\"powers\":[\"Spider-Sense\",\"Web-Shooting\",\"Wall-Crawling\"],\"enemies\":{\"name\":\"Green Goblin\",\"team\":\"Sinister Six\",\"powers\":[\"Glider\",\"Pumpkin Bombs\",\"Super Strength\"],\"allies\":{\"name\":\"Doctor Octopus\",\"team\":\"Sinister Six\",\"powers\":[\"Mechanical Arms\",\"Genius\",\"Madness\"]}}}]}") + } + }); + return sampleDataBuilder; } } diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index ee97c801c0..a6b0ab6b07 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -35,5 +35,12 @@ internal class BigQueryTestConfiguration : TestConfiguration [JsonPropertyName("refreshToken")] public string RefreshToken { get; set; } + + [JsonPropertyName("scopes")] + public string Scopes { get; set; } + + [JsonPropertyName("jsonCredential")] + public string JsonCredential { get; set; } + } } diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 567532cb16..874ccac791 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -54,11 +54,26 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestConf Dictionary parameters = new Dictionary { { BigQueryParameters.ProjectId, testConfiguration.ProjectId }, - { BigQueryParameters.ClientId, testConfiguration.ClientId }, - { BigQueryParameters.ClientSecret, testConfiguration.ClientSecret}, - { BigQueryParameters.RefreshToken, testConfiguration.RefreshToken} }; + if (!string.IsNullOrEmpty(testConfiguration.JsonCredential)) + { + parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.ServiceAccountAuthenticationType); + parameters.Add(BigQueryParameters.JsonCredential, testConfiguration.JsonCredential); + } + else + { + parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.UserAuthenticationType); + parameters.Add(BigQueryParameters.ClientId, testConfiguration.ClientId); + parameters.Add(BigQueryParameters.ClientSecret, testConfiguration.ClientSecret); + parameters.Add(BigQueryParameters.RefreshToken, testConfiguration.RefreshToken); + } + + if (!string.IsNullOrEmpty(testConfiguration.Scopes)) + { + parameters.Add(BigQueryParameters.Scopes, testConfiguration.Scopes); + } + return parameters; } diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index 089951d382..94b28833dd 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -26,16 +26,20 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery /// /// Class for testing the ADBC Client using the BigQuery ADBC driver. /// - /// /// + /// /// Tests are ordered to ensure data is created for the other /// queries to run. /// [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] public class ClientTests { + private BigQueryTestConfiguration _testConfiguration; + public ClientTests() { Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE)); + + _testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); } /// @@ -44,17 +48,15 @@ public ClientTests() [SkippableFact, Order(1)] public void CanClientExecuteUpdate() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { adbcConnection.Open(); - string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); + string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration); List expectedResults = new List() { -1, 1, 1 }; - Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, testConfiguration, queries, expectedResults); + Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, _testConfiguration, queries, expectedResults); } } @@ -64,11 +66,9 @@ public void CanClientExecuteUpdate() [SkippableFact, Order(2)] public void CanClientGetSchema() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { - Tests.ClientTests.CanClientGetSchema(adbcConnection, testConfiguration); + Tests.ClientTests.CanClientGetSchema(adbcConnection, _testConfiguration); } } @@ -79,11 +79,9 @@ public void CanClientGetSchema() [SkippableFact, Order(3)] public void CanClientExecuteQuery() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { - Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration); + Tests.ClientTests.CanClientExecuteQuery(adbcConnection, _testConfiguration); } } @@ -94,9 +92,7 @@ public void CanClientExecuteQuery() [SkippableFact, Order(4)] public void VerifyTypesAndValues() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection(testConfiguration)) + using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection()) { SampleDataBuilder sampleDataBuilder = BigQueryData.GetSampleData(); @@ -107,9 +103,7 @@ public void VerifyTypesAndValues() [SkippableFact] public void VerifySchemaTables() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration)) + using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection()) { adbcConnection.Open(); @@ -122,7 +116,7 @@ public void VerifySchemaTables() Assert.Equal(3, restrictions.Columns.Count); var catalogs = adbcConnection.GetSchema("Catalogs"); - Assert.Equal(1, catalogs.Columns.Count); + Assert.Single(catalogs.Columns); var catalog = (string)catalogs.Rows[0].ItemArray[0]; catalogs = adbcConnection.GetSchema("Catalogs", new[] { catalog }); @@ -150,7 +144,7 @@ public void VerifySchemaTables() Assert.Equal(0, schemas.Rows.Count); var tableTypes = adbcConnection.GetSchema("TableTypes"); - Assert.Equal(1, tableTypes.Columns.Count); + Assert.Single(tableTypes.Columns); var tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema }); Assert.Equal(4, tables.Columns.Count); @@ -172,11 +166,11 @@ public void VerifySchemaTables() } } - private Adbc.Client.AdbcConnection GetAdbcConnection(BigQueryTestConfiguration testConfiguration) + private Adbc.Client.AdbcConnection GetAdbcConnection() { return new Adbc.Client.AdbcConnection( new BigQueryDriver(), - BigQueryTestingUtils.GetBigQueryParameters(testConfiguration), + BigQueryTestingUtils.GetBigQueryParameters(_testConfiguration), new Dictionary() ); } diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs index c94c737e23..2af6de337b 100644 --- a/csharp/test/Drivers/BigQuery/DriverTests.cs +++ b/csharp/test/Drivers/BigQuery/DriverTests.cs @@ -35,9 +35,13 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery [TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")] public class DriverTests { + BigQueryTestConfiguration _testConfiguration; + public DriverTests() { Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE)); + _testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); + } /// @@ -47,11 +51,10 @@ public DriverTests() [SkippableFact, Order(1)] public void CanExecuteUpdate() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); - string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration); + string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration); List expectedResults = new List() { -1, 1, 1 }; @@ -73,9 +76,7 @@ public void CanExecuteUpdate() [SkippableFact, Order(2)] public void CanGetInfo() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream stream = adbcConnection.GetInfo(new List() { AdbcInfoCode.DriverName, AdbcInfoCode.DriverVersion, AdbcInfoCode.VendorName }); @@ -102,15 +103,13 @@ public void CanGetInfo() [SkippableFact, Order(3)] public void CanGetObjects() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - // need to add the database - string catalogName = testConfiguration.Metadata.Catalog; - string schemaName = testConfiguration.Metadata.Schema; - string tableName = testConfiguration.Metadata.Table; + string catalogName = _testConfiguration.Metadata.Catalog; + string schemaName = _testConfiguration.Metadata.Schema; + string tableName = _testConfiguration.Metadata.Table; string columnName = null; - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream stream = adbcConnection.GetObjects( depth: AdbcConnection.GetObjectsDepth.All, @@ -132,7 +131,7 @@ public void CanGetObjects() .Select(c => c.Columns) .FirstOrDefault(); - Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, columns.Count); + Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns.Count); } /// @@ -141,19 +140,17 @@ public void CanGetObjects() [SkippableFact, Order(4)] public void CanGetTableSchema() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); - - string catalogName = testConfiguration.Metadata.Catalog; - string schemaName = testConfiguration.Metadata.Schema; - string tableName = testConfiguration.Metadata.Table; + string catalogName = _testConfiguration.Metadata.Catalog; + string schemaName = _testConfiguration.Metadata.Schema; + string tableName = _testConfiguration.Metadata.Table; Schema schema = adbcConnection.GetTableSchema(catalogName, schemaName, tableName); int numberOfFields = schema.FieldsList.Count; - Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, numberOfFields); + Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, numberOfFields); } /// @@ -162,9 +159,7 @@ public void CanGetTableSchema() [SkippableFact, Order(5)] public void CanGetTableTypes() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); IArrowArrayStream arrowArrayStream = adbcConnection.GetTableTypes(); @@ -199,16 +194,14 @@ public void CanGetTableTypes() [SkippableFact, Order(6)] public void CanExecuteQuery() { - BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE); - - AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration); + AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration); AdbcStatement statement = adbcConnection.CreateStatement(); - statement.SqlQuery = testConfiguration.Query; + statement.SqlQuery = _testConfiguration.Query; QueryResult queryResult = statement.ExecuteQuery(); - Tests.DriverTests.CanExecuteQuery(queryResult, testConfiguration.ExpectedResultsCount); + Tests.DriverTests.CanExecuteQuery(queryResult, _testConfiguration.ExpectedResultsCount); } } } diff --git a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs index d1e7b9dc74..2f394a89a1 100644 --- a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs +++ b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs @@ -81,14 +81,14 @@ out Dictionary parameters { SnowflakeParameters.USE_HIGH_PRECISION, testConfiguration.UseHighPrecision.ToString().ToLowerInvariant() } }; - if(testConfiguration.Authentication.Default is not null) + if (testConfiguration.Authentication.Default is not null) { parameters[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthSnowflake; parameters[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.Default.User; parameters[SnowflakeParameters.PASSWORD] = testConfiguration.Authentication.Default.Password; } - if(!string.IsNullOrWhiteSpace(testConfiguration.Host)) + if (!string.IsNullOrWhiteSpace(testConfiguration.Host)) { parameters[SnowflakeParameters.HOST] = testConfiguration.Host; }