You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2024/01/26 19:27:46 UTC
(arrow-adbc) branch main updated: fix(csharp/src/Drivers/BigQuery): add support for scopes (#1482)
This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new c6472ab0 fix(csharp/src/Drivers/BigQuery): add support for scopes (#1482)
c6472ab0 is described below
commit c6472ab0d0a991c31b76aac13424fe3516571fcf
Author: davidhcoe <13...@users.noreply.github.com>
AuthorDate: Fri Jan 26 14:27:41 2024 -0500
fix(csharp/src/Drivers/BigQuery): add support for scopes (#1482)
- Adds support for passing scopes to the BigQueryClient
- Removes many of the null warnings from the pipeline runs
- Improves serialization for complex structs
- Includes test updates
---------
Co-authored-by: David Coe <co...@umich.edu>
---
csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs | 3 +-
csharp/src/Client/AdbcDataReader.cs | 2 +-
csharp/src/Client/SchemaConverter.cs | 10 +
.../Apache.Arrow.Adbc.Drivers.BigQuery.csproj | 1 +
csharp/src/Drivers/BigQuery/BigQueryConnection.cs | 350 ++++++++++++---------
.../Drivers/BigQuery/BigQueryInfoArrowStream.cs | 8 +-
csharp/src/Drivers/BigQuery/BigQueryParameters.cs | 1 +
csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 137 +++++++-
.../src/Drivers/BigQuery/BigQueryTokenResponse.cs | 6 +-
csharp/src/Drivers/BigQuery/readme.md | 3 +
.../Apache.Arrow.Adbc.Tests/Client/ClientTests.cs | 2 +-
csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs | 4 +-
csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs | 4 +-
csharp/test/Drivers/BigQuery/BigQueryData.cs | 56 ++++
.../Drivers/BigQuery/BigQueryTestConfiguration.cs | 7 +
.../test/Drivers/BigQuery/BigQueryTestingUtils.cs | 21 +-
csharp/test/Drivers/BigQuery/ClientTests.cs | 42 ++-
csharp/test/Drivers/BigQuery/DriverTests.cs | 49 ++-
.../Interop/Snowflake/SnowflakeTestingUtils.cs | 4 +-
19 files changed, 487 insertions(+), 223 deletions(-)
diff --git a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs
index 9e42544e..91486be4 100644
--- a/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs
+++ b/csharp/src/Apache.Arrow.Adbc/AdbcStatement.cs
@@ -147,10 +147,9 @@ namespace Apache.Arrow.Adbc
/// <param name="index">
/// The index in the array to get the value from.
/// </param>
- public virtual object GetValue(IArrowArray arrowArray, Field field, int index)
+ public virtual object GetValue(IArrowArray arrowArray, int index)
{
if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray));
- if (field == null) throw new ArgumentNullException(nameof(field));
if (index < 0) throw new ArgumentOutOfRangeException(nameof(index));
switch (arrowArray)
diff --git a/csharp/src/Client/AdbcDataReader.cs b/csharp/src/Client/AdbcDataReader.cs
index 6e9a7ae4..c58adc45 100644
--- a/csharp/src/Client/AdbcDataReader.cs
+++ b/csharp/src/Client/AdbcDataReader.cs
@@ -347,7 +347,7 @@ namespace Apache.Arrow.Adbc.Client
public object GetValue(IArrowArray arrowArray, int ordinal)
{
Field field = this.schema.GetFieldByIndex(ordinal);
- return this.adbcCommand.AdbcStatement.GetValue(arrowArray, field, this.currentRowInRecordBatch);
+ return this.adbcCommand.AdbcStatement.GetValue(arrowArray, this.currentRowInRecordBatch);
}
/// <summary>
diff --git a/csharp/src/Client/SchemaConverter.cs b/csharp/src/Client/SchemaConverter.cs
index 656cd0a7..0573e224 100644
--- a/csharp/src/Client/SchemaConverter.cs
+++ b/csharp/src/Client/SchemaConverter.cs
@@ -73,6 +73,16 @@ namespace Apache.Arrow.Adbc.Client
row[SchemaTableColumn.NumericPrecision] = Convert.ToInt32(f.Metadata["precision"]);
row[SchemaTableColumn.NumericScale] = Convert.ToInt32(f.Metadata["scale"]);
}
+ else if (f.DataType is Decimal128Type decimal128Type)
+ {
+ row[SchemaTableColumn.NumericPrecision] = decimal128Type.Precision;
+ row[SchemaTableColumn.NumericScale] = decimal128Type.Scale;
+ }
+ else if (f.DataType is Decimal256Type decimal256Type)
+ {
+ row[SchemaTableColumn.NumericPrecision] = decimal256Type.Precision;
+ row[SchemaTableColumn.NumericScale] = decimal256Type.Scale;
+ }
else
{
row[SchemaTableColumn.NumericPrecision] = DBNull.Value;
diff --git a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj
index 3bb39e99..0ce2c1fe 100644
--- a/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj
+++ b/csharp/src/Drivers/BigQuery/Apache.Arrow.Adbc.Drivers.BigQuery.csproj
@@ -2,6 +2,7 @@
<PropertyGroup>
<TargetFrameworks>netstandard2.0;net6.0</TargetFrameworks>
<PackageReadmeFile>readme.md</PackageReadmeFile>
+ <Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="7.0.0" Condition="'$(TargetFrameworkIdentifier)' == '.NETStandard'" />
diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
index 8e599231..ef68ef4e 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
@@ -70,24 +70,22 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
/// <exception cref="ArgumentException"></exception>
internal void Open()
{
- string projectId = string.Empty;
- string clientId = string.Empty;
- string clientSecret = string.Empty;
- string refreshToken = string.Empty;
+ string? projectId = null;
+ string? clientId = null;
+ string? clientSecret = null;
+ string? refreshToken = null;
string tokenEndpoint = BigQueryConstants.TokenEndpoint;
- string authenticationType = BigQueryConstants.UserAuthenticationType;
+ string? authenticationType = BigQueryConstants.UserAuthenticationType;
// TODO: handle token expiration
if (!this.properties.TryGetValue(BigQueryParameters.ProjectId, out projectId))
throw new ArgumentException($"The {BigQueryParameters.ProjectId} parameter is not present");
- if (this.properties.ContainsKey(BigQueryParameters.AuthenticationType))
+ if (this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType))
{
- this.properties.TryGetValue(BigQueryParameters.AuthenticationType, out authenticationType);
-
if (!authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase) &&
!authenticationType.Equals(BigQueryConstants.ServiceAccountAuthenticationType, StringComparison.OrdinalIgnoreCase))
{
@@ -95,7 +93,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
}
}
- if (authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase))
+ if (!string.IsNullOrEmpty(authenticationType) && authenticationType.Equals(BigQueryConstants.UserAuthenticationType, StringComparison.OrdinalIgnoreCase))
{
if (!this.properties.TryGetValue(BigQueryParameters.ClientId, out clientId))
throw new ArgumentException($"The {BigQueryParameters.ClientId} parameter is not present");
@@ -106,21 +104,43 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
if (!this.properties.TryGetValue(BigQueryParameters.RefreshToken, out refreshToken))
throw new ArgumentException($"The {BigQueryParameters.RefreshToken} parameter is not present");
- this.credential = GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint));
+ this.credential = ApplyScopes(GoogleCredential.FromAccessToken(GetAccessToken(clientId, clientSecret, refreshToken, tokenEndpoint)));
}
else
{
- string json = string.Empty;
+ string? json = string.Empty;
if (!this.properties.TryGetValue(BigQueryParameters.JsonCredential, out json))
throw new ArgumentException($"The {BigQueryParameters.JsonCredential} parameter is not present");
- this.credential = GoogleCredential.FromJson(json);
+ this.credential = ApplyScopes(GoogleCredential.FromJson(json));
}
this.client = BigQueryClient.Create(projectId, this.credential);
}
+ /// <summary>
+ /// Apply any additional scopes to the credential.
+ /// </summary>
+ /// <param name="credential"><see cref="GoogleCredential"/></param>
+ /// <returns></returns>
+ private GoogleCredential ApplyScopes(GoogleCredential credential)
+ {
+ if (credential == null) throw new ArgumentNullException(nameof(credential));
+
+ if (this.properties.TryGetValue(BigQueryParameters.Scopes, out string? scopes))
+ {
+ if (!string.IsNullOrEmpty(scopes))
+ {
+ IEnumerable<string> parsedScopes = scopes.Split(',').Where(x => x.Length > 0);
+
+ return credential.CreateScoped(parsedScopes);
+ }
+ }
+
+ return credential;
+ }
+
public override IArrowArrayStream GetInfo(List<AdbcInfoCode> codes)
{
const int strValTypeID = 0;
@@ -261,23 +281,26 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
StringArray.Builder catalogNameBuilder = new StringArray.Builder();
List<IArrowArray?> catalogDbSchemasValues = new List<IArrowArray?>();
string catalogRegexp = PatternToRegEx(catalogPattern);
- PagedEnumerable<ProjectList, CloudProject> catalogs = this.client.ListProjects();
+ PagedEnumerable<ProjectList, CloudProject>? catalogs = this.client?.ListProjects();
- foreach (CloudProject catalog in catalogs)
+ if (catalogs != null)
{
- if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase))
+ foreach (CloudProject catalog in catalogs)
{
- catalogNameBuilder.Append(catalog.ProjectId);
-
- if (depth == GetObjectsDepth.Catalogs)
- {
- catalogDbSchemasValues.Add(null);
- }
- else
+ if (Regex.IsMatch(catalog.ProjectId, catalogRegexp, RegexOptions.IgnoreCase))
{
- catalogDbSchemasValues.Add(GetDbSchemas(
- depth, catalog.ProjectId, dbSchemaPattern,
- tableNamePattern, tableTypes, columnNamePattern));
+ catalogNameBuilder.Append(catalog.ProjectId);
+
+ if (depth == GetObjectsDepth.Catalogs)
+ {
+ catalogDbSchemasValues.Add(null);
+ }
+ else
+ {
+ catalogDbSchemasValues.Add(GetDbSchemas(
+ depth, catalog.ProjectId, dbSchemaPattern,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
}
}
}
@@ -306,25 +329,28 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
string dbSchemaRegexp = PatternToRegEx(dbSchemaPattern);
- PagedEnumerable<DatasetList, BigQueryDataset> schemas = this.client.ListDatasets(catalog);
+ PagedEnumerable<DatasetList, BigQueryDataset>? schemas = this.client?.ListDatasets(catalog);
- foreach (BigQueryDataset schema in schemas)
+ if (schemas != null)
{
- if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase))
+ foreach (BigQueryDataset schema in schemas)
{
- dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
- length++;
- nullBitmapBuffer.Append(true);
-
- if (depth == GetObjectsDepth.DbSchemas)
- {
- dbSchemaTablesValues.Add(null);
- }
- else
+ if (Regex.IsMatch(schema.Reference.DatasetId, dbSchemaRegexp, RegexOptions.IgnoreCase))
{
- dbSchemaTablesValues.Add(GetTableSchemas(
- depth, catalog, schema.Reference.DatasetId,
- tableNamePattern, tableTypes, columnNamePattern));
+ dbSchemaNameBuilder.Append(schema.Reference.DatasetId);
+ length++;
+ nullBitmapBuffer.Append(true);
+
+ if (depth == GetObjectsDepth.DbSchemas)
+ {
+ dbSchemaTablesValues.Add(null);
+ }
+ else
+ {
+ dbSchemaTablesValues.Add(GetTableSchemas(
+ depth, catalog, schema.Reference.DatasetId,
+ tableNamePattern, tableTypes, columnNamePattern));
+ }
}
}
}
@@ -379,25 +405,28 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
}
}
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- tableNameBuilder.Append(row["table_name"].ToString());
- tableTypeBuilder.Append(row["table_type"].ToString());
- nullBitmapBuffer.Append(true);
- length++;
+ foreach (BigQueryRow row in result)
+ {
+ tableNameBuilder.Append(GetValue(row["table_name"]));
+ tableTypeBuilder.Append(GetValue(row["table_type"]));
+ nullBitmapBuffer.Append(true);
+ length++;
- tableConstraintsValues.Add(GetConstraintSchema(
- depth, catalog, dbSchema, row["table_name"].ToString(), columnNamePattern));
+ tableConstraintsValues.Add(GetConstraintSchema(
+ depth, catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern));
- if (depth == GetObjectsDepth.Tables)
- {
- tableColumnsValues.Add(null);
- }
- else
- {
- tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, row["table_name"].ToString(), columnNamePattern));
+ if (depth == GetObjectsDepth.Tables)
+ {
+ tableColumnsValues.Add(null);
+ }
+ else
+ {
+ tableColumnsValues.Add(GetColumnSchema(catalog, dbSchema, GetValue(row["table_name"]), columnNamePattern));
+ }
}
}
@@ -452,46 +481,48 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
query = string.Concat(query, string.Format("AND column_name LIKE '{0}'", Sanitize(columnNamePattern)));
}
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- columnNameBuilder.Append(row["column_name"].ToString());
- ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]);
- remarksBuilder.Append("");
+ foreach (BigQueryRow row in result)
+ {
+ columnNameBuilder.Append(GetValue(row["column_name"]));
+ ordinalPositionBuilder.Append((int)(long)row["ordinal_position"]);
+ remarksBuilder.Append("");
- string dataType = ToTypeName(row["data_type"].ToString());
+ string dataType = ToTypeName(GetValue(row["data_type"]));
- if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL"))
- {
- ParsedDecimalValues values = ParsePrecisionAndScale(dataType);
- xdbcColumnSizeBuilder.Append(values.Precision);
- xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale));
- }
- else
- {
- xdbcColumnSizeBuilder.AppendNull();
- xdbcDecimalDigitsBuilder.AppendNull();
- }
+ if (dataType.StartsWith("NUMERIC") || dataType.StartsWith("DECIMAL") || dataType.StartsWith("BIGNUMERIC") || dataType.StartsWith("BIGDECIMAL"))
+ {
+ ParsedDecimalValues values = ParsePrecisionAndScale(dataType);
+ xdbcColumnSizeBuilder.Append(values.Precision);
+ xdbcDecimalDigitsBuilder.Append(Convert.ToInt16(values.Scale));
+ }
+ else
+ {
+ xdbcColumnSizeBuilder.AppendNull();
+ xdbcDecimalDigitsBuilder.AppendNull();
+ }
- xdbcDataTypeBuilder.AppendNull();
- xdbcTypeNameBuilder.Append(dataType);
- xdbcNumPrecRadixBuilder.AppendNull();
- xdbcNullableBuilder.AppendNull();
- xdbcColumnDefBuilder.AppendNull();
- xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType));
- xdbcDatetimeSubBuilder.AppendNull();
- xdbcCharOctetLengthBuilder.AppendNull();
- xdbcIsNullableBuilder.Append(row["is_nullable"].ToString());
- xdbcScopeCatalogBuilder.AppendNull();
- xdbcScopeSchemaBuilder.AppendNull();
- xdbcScopeTableBuilder.AppendNull();
- xdbcIsAutoincrementBuilder.AppendNull();
- xdbcIsGeneratedcolumnBuilder.Append(row["is_generated"].ToString().ToUpper() == "YES");
- nullBitmapBuffer.Append(true);
- length++;
+ xdbcDataTypeBuilder.AppendNull();
+ xdbcTypeNameBuilder.Append(dataType);
+ xdbcNumPrecRadixBuilder.AppendNull();
+ xdbcNullableBuilder.AppendNull();
+ xdbcColumnDefBuilder.AppendNull();
+ xdbcSqlDataTypeBuilder.Append((short)ToXdbcDataType(dataType));
+ xdbcDatetimeSubBuilder.AppendNull();
+ xdbcCharOctetLengthBuilder.AppendNull();
+ xdbcIsNullableBuilder.Append(row["is_nullable"].ToString());
+ xdbcScopeCatalogBuilder.AppendNull();
+ xdbcScopeSchemaBuilder.AppendNull();
+ xdbcScopeTableBuilder.AppendNull();
+ xdbcIsAutoincrementBuilder.AppendNull();
+ xdbcIsGeneratedcolumnBuilder.Append(GetValue(row["is_generated"]).ToUpper() == "YES");
+ nullBitmapBuffer.Append(true);
+ length++;
+ }
}
-
List<IArrowArray> dataArrays = new List<IArrowArray>
{
columnNameBuilder.Build(),
@@ -539,36 +570,39 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE table_name = '{2}'",
Sanitize(catalog), Sanitize(dbSchema), Sanitize(table));
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- string constraintName = row["constraint_name"].ToString();
- constraintNameBuilder.Append(constraintName);
- string constraintType = row["constraint_type"].ToString();
- constraintTypeBuilder.Append(constraintType);
- nullBitmapBuffer.Append(true);
- length++;
-
- if (depth == GetObjectsDepth.All || depth == GetObjectsDepth.Tables)
+ foreach (BigQueryRow row in result)
{
- constraintColumnNamesValues.Add(GetConstraintColumnNames(
- catalog, dbSchema, table, constraintName));
- if (constraintType.ToUpper() == "FOREIGN KEY")
+ string constraintName = GetValue(row["constraint_name"]);
+ constraintNameBuilder.Append(constraintName);
+ string constraintType = GetValue(row["constraint_type"]);
+ constraintTypeBuilder.Append(constraintType);
+ nullBitmapBuffer.Append(true);
+ length++;
+
+ if (depth == GetObjectsDepth.All || depth == GetObjectsDepth.Tables)
{
- constraintColumnUsageValues.Add(GetConstraintsUsage(
+ constraintColumnNamesValues.Add(GetConstraintColumnNames(
catalog, dbSchema, table, constraintName));
+ if (constraintType.ToUpper() == "FOREIGN KEY")
+ {
+ constraintColumnUsageValues.Add(GetConstraintsUsage(
+ catalog, dbSchema, table, constraintName));
+ }
+ else
+ {
+ constraintColumnUsageValues.Add(null);
+ }
}
else
{
+ constraintColumnNamesValues.Add(null);
constraintColumnUsageValues.Add(null);
}
}
- else
- {
- constraintColumnNamesValues.Add(null);
- constraintColumnUsageValues.Add(null);
- }
}
List<IArrowArray> dataArrays = new List<IArrowArray>
@@ -597,12 +631,15 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
StringArray.Builder constraintColumnNamesBuilder = new StringArray.Builder();
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- string column = row["column_name"].ToString();
- constraintColumnNamesBuilder.Append(column);
+ foreach (BigQueryRow row in result)
+ {
+ string column = GetValue(row["column_name"]);
+ constraintColumnNamesBuilder.Append(column);
+ }
}
return constraintColumnNamesBuilder.Build();
@@ -624,22 +661,25 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE WHERE constraint_name = '{2}'",
Sanitize(catalog), Sanitize(dbSchema), Sanitize(constraintName));
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- string constraint_catalog = row["constraint_catalog"].ToString();
- string constraint_schema = row["constraint_schema"].ToString();
- string table_name = row["table_name"].ToString();
- string column_name = row["column_name"].ToString();
-
- constraintFkCatalogBuilder.Append(constraint_catalog);
- constraintFkDbSchemaBuilder.Append(constraint_schema);
- constraintFkTableBuilder.Append(table_name);
- constraintFkColumnNameBuilder.Append(column_name);
-
- nullBitmapBuffer.Append(true);
- length++;
+ foreach (BigQueryRow row in result)
+ {
+ string constraint_catalog = GetValue(row["constraint_catalog"]);
+ string constraint_schema = GetValue(row["constraint_schema"]);
+ string table_name = GetValue(row["table_name"]);
+ string column_name = GetValue(row["column_name"]);
+
+ constraintFkCatalogBuilder.Append(constraint_catalog);
+ constraintFkDbSchemaBuilder.Append(constraint_schema);
+ constraintFkTableBuilder.Append(table_name);
+ constraintFkColumnNameBuilder.Append(column_name);
+
+ nullBitmapBuffer.Append(true);
+ length++;
+ }
}
List<IArrowArray> dataArrays = new List<IArrowArray>
@@ -748,13 +788,16 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'",
Sanitize(catalog), Sanitize(dbSchema), Sanitize(tableName));
- BigQueryResults result = this.client.ExecuteQuery(query, parameters: null);
+ BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null);
List<Field> fields = new List<Field>();
- foreach (BigQueryRow row in result)
+ if (result != null)
{
- fields.Add(DescToField(row));
+ foreach (BigQueryRow row in result)
+ {
+ fields.Add(DescToField(row));
+ }
}
return new Schema(fields, null);
@@ -764,22 +807,38 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
{
Dictionary<string, string> metaData = new Dictionary<string, string>();
metaData.Add("PRIMARY_KEY", "");
- metaData.Add("ORDINAL_POSITION", row["ordinal_position"].ToString());
- metaData.Add("DATA_TYPE", row["data_type"].ToString());
+ metaData.Add("ORDINAL_POSITION", GetValue(row["ordinal_position"]));
+ metaData.Add("DATA_TYPE", GetValue(row["data_type"]));
- Field.Builder fieldBuilder = SchemaFieldGenerator(row["column_name"].ToString().ToLower(), row["data_type"].ToString());
+ Field.Builder fieldBuilder = SchemaFieldGenerator(GetValue(row["column_name"]).ToLower(), GetValue(row["data_type"]));
fieldBuilder.Metadata(metaData);
- if (!row["is_nullable"].ToString().Equals("YES", StringComparison.OrdinalIgnoreCase))
+ if (!GetValue(row["is_nullable"]).Equals("YES", StringComparison.OrdinalIgnoreCase))
{
fieldBuilder.Nullable(false);
}
- fieldBuilder.Name(row["column_name"].ToString().ToLower());
+ fieldBuilder.Name(GetValue(row["column_name"]).ToLower());
return fieldBuilder.Build();
}
+ private string GetValue(object value)
+ {
+ switch (value)
+ {
+ case string sValue:
+ return sValue;
+ default:
+ if (value != null)
+ {
+ string? sValue = value.ToString();
+ return sValue ?? string.Empty;
+ }
+ throw new InvalidOperationException($"Cannot parse {value}");
+ }
+ }
+
private Field.Builder SchemaFieldGenerator(string name, string type)
{
int index = type.IndexOf("(");
@@ -874,7 +933,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
return new BigQueryInfoArrowStream(StandardSchemas.TableTypesSchema, dataArrays);
}
- private ListArray CreateNestedListArray(List<IArrowArray> arrayList, IArrowType dataType)
+ private ListArray CreateNestedListArray(List<IArrowArray?> arrayList, IArrowType dataType)
{
ArrowBuffer.Builder<int> valueOffsetsBufferBuilder = new ArrowBuffer.Builder<int>();
ArrowBuffer.BitmapBuilder validityBufferBuilder = new ArrowBuffer.BitmapBuilder();
@@ -882,7 +941,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
int length = 0;
int nullCount = 0;
- foreach (IArrowArray array in arrayList)
+ foreach (IArrowArray? array in arrayList)
{
if (array == null)
{
@@ -902,7 +961,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
ArrowBuffer validityBuffer = nullCount > 0
? validityBufferBuilder.Build() : ArrowBuffer.Empty;
- ArrayData data = ArrayDataConcatenator.Concatenate(arrayDataList);
+ ArrayData? data = ArrayDataConcatenator.Concatenate(arrayDataList);
if (data == null)
{
@@ -970,6 +1029,9 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
private string Sanitize(string input)
{
+ if (string.IsNullOrEmpty(input))
+ return string.Empty;
+
bool isValidInput = sanitizedInputRegex.IsMatch(input);
if (isValidInput)
@@ -990,7 +1052,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
/// <param name="refreshToken"></param>
/// <param name="tokenEndpoint"></param>
/// <returns></returns>
- private string GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint)
+ private string? GetAccessToken(string clientId, string clientSecret, string refreshToken, string tokenEndpoint)
{
string body = string.Format(
"grant_type=refresh_token&client_id={0}&client_secret={1}&refresh_token={2}",
@@ -1006,9 +1068,9 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
HttpResponseMessage response = httpClient.SendAsync(request).Result;
string responseBody = response.Content.ReadAsStringAsync().Result;
- BigQueryTokenResponse bigQueryTokenResponse = JsonSerializer.Deserialize<BigQueryTokenResponse>(responseBody);
+ BigQueryTokenResponse? bigQueryTokenResponse = JsonSerializer.Deserialize<BigQueryTokenResponse>(responseBody);
- return bigQueryTokenResponse.AccessToken;
+ return bigQueryTokenResponse?.AccessToken;
}
enum XdbcDataType
@@ -1049,7 +1111,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
IArrowTypeVisitor<StructType>,
IArrowTypeVisitor<MapType>
{
- public ArrayData Result { get; private set; }
+ public ArrayData? Result { get; private set; }
public void Visit(BooleanType type)
{
@@ -1074,7 +1136,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public void Visit(ListType type)
{
type.ValueDataType.Accept(this);
- ArrayData child = Result;
+ ArrayData? child = Result;
Result = new ArrayData(type, 0, 0, 0, new[] { ArrowBuffer.Empty, MakeInt0Buffer() }, new[] { child });
}
@@ -1082,14 +1144,14 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public void Visit(FixedSizeListType type)
{
type.ValueDataType.Accept(this);
- ArrayData child = Result;
+ ArrayData? child = Result;
Result = new ArrayData(type, 0, 0, 0, new[] { ArrowBuffer.Empty }, new[] { child });
}
public void Visit(StructType type)
{
- ArrayData[] children = new ArrayData[type.Fields.Count];
+ ArrayData?[] children = new ArrayData[type.Fields.Count];
for (int i = 0; i < type.Fields.Count; i++)
{
type.Fields[i].DataType.Accept(this);
diff --git a/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs b/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs
index 75a67de8..257798ce 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryInfoArrowStream.cs
@@ -28,7 +28,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
internal class BigQueryInfoArrowStream : IArrowArrayStream
{
private Schema schema;
- private RecordBatch batch;
+ private RecordBatch? batch;
public BigQueryInfoArrowStream(Schema schema, List<IArrowArray> data)
{
@@ -38,11 +38,11 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public Schema Schema { get { return this.schema; } }
- public ValueTask<RecordBatch> ReadNextRecordBatchAsync(CancellationToken cancellationToken = default)
+ public ValueTask<RecordBatch?> ReadNextRecordBatchAsync(CancellationToken cancellationToken = default)
{
- RecordBatch batch = this.batch;
+ RecordBatch? batch = this.batch;
this.batch = null;
- return new ValueTask<RecordBatch>(batch);
+ return new ValueTask<RecordBatch?>(batch);
}
public void Dispose()
diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
index 5a12388c..ff05036d 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
@@ -31,6 +31,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public const string AllowLargeResults = "adbc.bigquery.allow_large_results";
public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql";
public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string";
+ public const string Scopes = "adbc.bigquery.scopes";
}
/// <summary>
diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
index 009500b8..389e462e 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
@@ -16,7 +16,9 @@
*/
using System;
+using System.Collections;
using System.Collections.Generic;
+using System.Data.SqlTypes;
using System.IO;
using System.Linq;
using System.Text.Json;
@@ -87,16 +89,16 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
return new Field(field.Name, TranslateType(field), field.Mode == "NULLABLE");
}
- public override object GetValue(IArrowArray arrowArray, Field field, int index)
+ public override object GetValue(IArrowArray arrowArray, int index)
{
- switch(arrowArray)
+ switch (arrowArray)
{
case StructArray structArray:
return SerializeToJson(structArray, index);
case ListArray listArray:
return listArray.GetSlicedValues(index);
default:
- return base.GetValue(arrowArray, field, index);
+ return base.GetValue(arrowArray, index);
}
}
@@ -141,7 +143,10 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
return GetType(field, new Decimal128Type(38, 9));
case "BIGNUMERIC" or "BIGDECIMAL":
- return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38));
+ if (this.Options != null)
+ return bool.Parse(this.Options[BigQueryParameters.LargeDecimalsAsString]) ? GetType(field, StringType.Default) : GetType(field, new Decimal256Type(76, 38));
+ else
+ return GetType(field, StringType.Default);
default: throw new InvalidOperationException($"{field.Type} cannot be translated");
}
@@ -189,17 +194,135 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
}
private string SerializeToJson(StructArray structArray, int index)
+ {
+ Dictionary<String, object> jsonDictionary = ParseStructArray(structArray, index);
+
+ return JsonSerializer.Serialize(jsonDictionary);
+ }
+
+ private Dictionary<String, object> ParseStructArray(StructArray structArray, int index)
{
Dictionary<String, object> jsonDictionary = new Dictionary<String, object>();
StructType structType = (StructType)structArray.Data.DataType;
for (int i = 0; i < structArray.Data.Children.Length; i++)
{
- jsonDictionary.Add(structType.Fields[i].Name,
- GetValue(structArray.Fields[i], structType.GetFieldByName(structType.Fields[i].Name), index));
+ string name = structType.Fields[i].Name;
+ object value = GetValue(structArray.Fields[i], index);
+
+ if (value is StructArray structArray1)
+ {
+ List<Dictionary<string, object>> children = new List<Dictionary<string, object>>();
+
+ if (structArray1.Length > 1)
+ {
+ for (int j = 0; j < structArray1.Length; j++)
+ children.Add(ParseStructArray(structArray1, j));
+ }
+
+ if (children.Count > 0)
+ {
+ jsonDictionary.Add(name, children);
+ }
+ else
+ {
+ jsonDictionary.Add(name, ParseStructArray(structArray1, index));
+ }
+ }
+ else if (value is IArrowArray arrowArray)
+ {
+ IList? values = CreateList(arrowArray);
+
+ if (values != null)
+ {
+ for (int j = 0; j < arrowArray.Length; j++)
+ {
+ values.Add(base.GetValue(arrowArray, j));
+ }
+
+ jsonDictionary.Add(name, values);
+ }
+ else
+ {
+ jsonDictionary.Add(name, new List<object>());
+ }
+ }
+ else
+ {
+ jsonDictionary.Add(name, value);
+ }
+ }
+
+ return jsonDictionary;
+ }
+
+ private IList? CreateList(IArrowArray arrowArray)
+ {
+ if (arrowArray == null) throw new ArgumentNullException(nameof(arrowArray));
+
+ switch (arrowArray)
+ {
+ case BooleanArray booleanArray:
+ return new List<bool>();
+ case Date32Array date32Array:
+ case Date64Array date64Array:
+ return new List<DateTime>();
+ case Decimal128Array decimal128Array:
+ return new List<SqlDecimal>();
+ case Decimal256Array decimal256Array:
+ return new List<string>();
+ case DoubleArray doubleArray:
+ return new List<double>();
+ case FloatArray floatArray:
+ return new List<float>();
+#if NET5_0_OR_GREATER
+ case PrimitiveArray<Half> halfFloatArray:
+ return new List<Half>();
+#endif
+ case Int8Array int8Array:
+ return new List<sbyte>();
+ case Int16Array int16Array:
+ return new List<short>();
+ case Int32Array int32Array:
+ return new List<int>();
+ case Int64Array int64Array:
+ return new List<long>();
+ case StringArray stringArray:
+ return new List<string>();
+#if NET6_0_OR_GREATER
+ case Time32Array time32Array:
+ case Time64Array time64Array:
+ return new List<TimeOnly>();
+#else
+ case Time32Array time32Array:
+ case Time64Array time64Array:
+ return new List<TimeSpan>();
+#endif
+ case TimestampArray timestampArray:
+ return new List<DateTimeOffset>();
+ case UInt8Array uInt8Array:
+ return new List<byte>();
+ case UInt16Array uInt16Array:
+ return new List<ushort>();
+ case UInt32Array uInt32Array:
+ return new List<uint>();
+ case UInt64Array uInt64Array:
+ return new List<ulong>();
+
+ case BinaryArray binaryArray:
+ return new List<byte>();
+
+ // not covered:
+ // -- struct array
+ // -- dictionary array
+ // -- fixed size binary
+ // -- list array
+ // -- union array
}
- return JsonSerializer.Serialize(jsonDictionary);
+
+ return null;
}
+
class MultiArrowReader : IArrowArrayStream
{
readonly Schema schema;
diff --git a/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs b/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs
index ee6f9802..8df4b34c 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryTokenResponse.cs
@@ -25,15 +25,15 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
internal class BigQueryTokenResponse
{
[JsonPropertyName("access_token")]
- public string AccessToken { get; set; }
+ public string? AccessToken { get; set; }
[JsonPropertyName("expires_in")]
public int ExpiresIn { get; set; }
[JsonPropertyName("scope")]
- public string Scope { get; set; }
+ public string? Scope { get; set; }
[JsonPropertyName("token_type")]
- public string TokenType { get; set; }
+ public string? TokenType { get; set; }
}
}
diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md
index 310cf32d..611374c5 100644
--- a/csharp/src/Drivers/BigQuery/readme.md
+++ b/csharp/src/Drivers/BigQuery/readme.md
@@ -55,6 +55,9 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G
**adbc.bigquery.refresh_token**<br>
The refresh token used for when the generated OAuth token expires. Required for `user` authentication.
+**adbc.bigquery.scopes**<br>
+ Optional. Comma separated list of scopes to include for the credential.
+
**adbc.bigquery.use_legacy_sql**<br>
Sets the [UseLegacySql](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_UseLegacySql) value of the QueryOptions to `true` if configured; otherwise, the default is `false`.
diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs
index 9548b85e..423f2f81 100644
--- a/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs
+++ b/csharp/test/Apache.Arrow.Adbc.Tests/Client/ClientTests.cs
@@ -70,7 +70,7 @@ namespace Apache.Arrow.Adbc.Tests.Client
Mock<AdbcStatement> mockStatement = new Mock<AdbcStatement>();
mockStatement.Setup(x => x.ExecuteQuery()).Returns(queryResult); ;
- mockStatement.Setup(x => x.GetValue(It.IsAny<IArrowArray>(), It.IsAny<Field>(), It.IsAny<int>())).Returns(sqlDecimal);
+ mockStatement.Setup(x => x.GetValue(It.IsAny<IArrowArray>(), It.IsAny<int>())).Returns(sqlDecimal);
Adbc.Client.AdbcConnection mockConnection = new Adbc.Client.AdbcConnection();
mockConnection.DecimalBehavior = decimalBehavior;
diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
index cb55a94d..85eceb27 100644
--- a/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
+++ b/csharp/test/Apache.Arrow.Adbc.Tests/ClientTests.cs
@@ -137,8 +137,8 @@ namespace Apache.Arrow.Adbc.Tests
/// <param name="sampleDataBuilder">The <see cref="SampleDataBuilder"/> to use</param>
public static void VerifyTypesAndValues(Adbc.Client.AdbcConnection adbcConnection, SampleDataBuilder sampleDataBuilder)
{
- if(adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection));
- if(sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder));
+ if (adbcConnection == null) throw new ArgumentNullException(nameof(adbcConnection));
+ if (sampleDataBuilder == null) throw new ArgumentNullException(nameof(sampleDataBuilder));
adbcConnection.Open();
diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs
index 343719bc..1d695480 100644
--- a/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs
+++ b/csharp/test/Apache.Arrow.Adbc.Tests/Utils.cs
@@ -75,7 +75,7 @@ namespace Apache.Arrow.Adbc.Tests
public static T LoadTestConfiguration<T>(string environmentVariable)
where T : TestConfiguration
{
- if(CanExecuteTest(environmentVariable, out string environmentValue))
+ if (CanExecuteTest(environmentVariable, out string environmentValue))
return GetTestConfiguration<T>(environmentValue);
throw new InvalidOperationException($"Cannot execute test configuration from environment variable `{environmentVariable}`");
@@ -92,7 +92,7 @@ namespace Apache.Arrow.Adbc.Tests
public static T GetTestConfiguration<T>(string fileName)
where T : TestConfiguration
{
- if(!File.Exists(fileName))
+ if (!File.Exists(fileName))
throw new FileNotFoundException(fileName);
// use a JSON file for the various settings
diff --git a/csharp/test/Drivers/BigQuery/BigQueryData.cs b/csharp/test/Drivers/BigQuery/BigQueryData.cs
index f23e4ce2..010cbe0f 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryData.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryData.cs
@@ -146,6 +146,62 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
}
});
+ // complex struct
+ sampleDataBuilder.Samples.Add(
+ new SampleData()
+ {
+ Query = "SELECT " +
+ "STRUCT(" +
+ "\"Iron Man\" as name," +
+ "\"Avengers\" as team," +
+ "[\"Genius\", \"Billionaire\", \"Playboy\", \"Philanthropist\"] as powers," +
+ "[" +
+ " STRUCT(" +
+ " \"Captain America\" as name, " +
+ " \"Avengers\" as team, " +
+ " [\"Super Soldier Serum\", \"Vibranium Shield\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Thanos\" as name, " +
+ " \"Black Order\" as team, " +
+ " [\"Infinity Gauntlet\", \"Super Strength\", \"Teleportation\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Loki\" as name, " +
+ " \"Asgard\" as team, " +
+ " [\"Magic\", \"Shapeshifting\", \"Trickery\"] as powers " +
+ " )" +
+ " ] as allies" +
+ " )" +
+ " ] as enemies" +
+ " )," +
+ " STRUCT(" +
+ " \"Spider-Man\" as name, " +
+ " \"Avengers\" as team, " +
+ " [\"Spider-Sense\", \"Web-Shooting\", \"Wall-Crawling\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Green Goblin\" as name, " +
+ " \"Sinister Six\" as team, " +
+ " [\"Glider\", \"Pumpkin Bombs\", \"Super Strength\"] as powers, " +
+ " [" +
+ " STRUCT(" +
+ " \"Doctor Octopus\" as name, " +
+ " \"Sinister Six\" as team, " +
+ " [\"Mechanical Arms\", \"Genius\", \"Madness\"] as powers " +
+ " )" +
+ " ] as allies" +
+ " )" +
+ " ] as enemies" +
+ " )" +
+ " ] as friends" +
+ ") as iron_man",
+ ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
+ {
+ new ColumnNetTypeArrowTypeValue("iron_man", typeof(string), typeof(StringType), "{\"name\":\"Iron Man\",\"team\":\"Avengers\",\"powers\":[\"Genius\",\"Billionaire\",\"Playboy\",\"Philanthropist\"],\"friends\":[{\"name\":\"Captain America\",\"team\":\"Avengers\",\"powers\":[\"Super Soldier Serum\",\"Vibranium Shield\"],\"enemies\":{\"name\":\"Thanos\",\"team\":\"Black Order\",\"powers\":[\"Infinity Gauntlet\",\"Super Strength\",\"Teleportation\"],\"allies\":{\"name [...]
+ }
+ });
+
return sampleDataBuilder;
}
}
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
index ee97c801..a6b0ab6b 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
@@ -35,5 +35,12 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[JsonPropertyName("refreshToken")]
public string RefreshToken { get; set; }
+
+ [JsonPropertyName("scopes")]
+ public string Scopes { get; set; }
+
+ [JsonPropertyName("jsonCredential")]
+ public string JsonCredential { get; set; }
+
}
}
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
index 567532cb..874ccac7 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
@@ -54,11 +54,26 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
Dictionary<string, string> parameters = new Dictionary<string, string>
{
{ BigQueryParameters.ProjectId, testConfiguration.ProjectId },
- { BigQueryParameters.ClientId, testConfiguration.ClientId },
- { BigQueryParameters.ClientSecret, testConfiguration.ClientSecret},
- { BigQueryParameters.RefreshToken, testConfiguration.RefreshToken}
};
+ if (!string.IsNullOrEmpty(testConfiguration.JsonCredential))
+ {
+ parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.ServiceAccountAuthenticationType);
+ parameters.Add(BigQueryParameters.JsonCredential, testConfiguration.JsonCredential);
+ }
+ else
+ {
+ parameters.Add(BigQueryParameters.AuthenticationType, BigQueryConstants.UserAuthenticationType);
+ parameters.Add(BigQueryParameters.ClientId, testConfiguration.ClientId);
+ parameters.Add(BigQueryParameters.ClientSecret, testConfiguration.ClientSecret);
+ parameters.Add(BigQueryParameters.RefreshToken, testConfiguration.RefreshToken);
+ }
+
+ if (!string.IsNullOrEmpty(testConfiguration.Scopes))
+ {
+ parameters.Add(BigQueryParameters.Scopes, testConfiguration.Scopes);
+ }
+
return parameters;
}
diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs
index 089951d3..94b28833 100644
--- a/csharp/test/Drivers/BigQuery/ClientTests.cs
+++ b/csharp/test/Drivers/BigQuery/ClientTests.cs
@@ -26,16 +26,20 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
/// <summary>
/// Class for testing the ADBC Client using the BigQuery ADBC driver.
/// </summary>
- /// /// <remarks>
+ /// <remarks>
/// Tests are ordered to ensure data is created for the other
/// queries to run.
/// </remarks>
[TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")]
public class ClientTests
{
+ private BigQueryTestConfiguration _testConfiguration;
+
public ClientTests()
{
Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE));
+
+ _testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
}
/// <summary>
@@ -44,17 +48,15 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(1)]
public void CanClientExecuteUpdate()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration))
+ using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection())
{
adbcConnection.Open();
- string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration);
+ string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration);
List<int> expectedResults = new List<int>() { -1, 1, 1 };
- Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, testConfiguration, queries, expectedResults);
+ Tests.ClientTests.CanClientExecuteUpdate(adbcConnection, _testConfiguration, queries, expectedResults);
}
}
@@ -64,11 +66,9 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(2)]
public void CanClientGetSchema()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration))
+ using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection())
{
- Tests.ClientTests.CanClientGetSchema(adbcConnection, testConfiguration);
+ Tests.ClientTests.CanClientGetSchema(adbcConnection, _testConfiguration);
}
}
@@ -79,11 +79,9 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(3)]
public void CanClientExecuteQuery()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration))
+ using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection())
{
- Tests.ClientTests.CanClientExecuteQuery(adbcConnection, testConfiguration);
+ Tests.ClientTests.CanClientExecuteQuery(adbcConnection, _testConfiguration);
}
}
@@ -94,9 +92,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(4)]
public void VerifyTypesAndValues()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection(testConfiguration))
+ using(Adbc.Client.AdbcConnection dbConnection = GetAdbcConnection())
{
SampleDataBuilder sampleDataBuilder = BigQueryData.GetSampleData();
@@ -107,9 +103,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact]
public void VerifySchemaTables()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection(testConfiguration))
+ using (Adbc.Client.AdbcConnection adbcConnection = GetAdbcConnection())
{
adbcConnection.Open();
@@ -122,7 +116,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
Assert.Equal(3, restrictions.Columns.Count);
var catalogs = adbcConnection.GetSchema("Catalogs");
- Assert.Equal(1, catalogs.Columns.Count);
+ Assert.Single(catalogs.Columns);
var catalog = (string)catalogs.Rows[0].ItemArray[0];
catalogs = adbcConnection.GetSchema("Catalogs", new[] { catalog });
@@ -150,7 +144,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
Assert.Equal(0, schemas.Rows.Count);
var tableTypes = adbcConnection.GetSchema("TableTypes");
- Assert.Equal(1, tableTypes.Columns.Count);
+ Assert.Single(tableTypes.Columns);
var tables = adbcConnection.GetSchema("Tables", new[] { catalog, schema });
Assert.Equal(4, tables.Columns.Count);
@@ -172,11 +166,11 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
}
}
- private Adbc.Client.AdbcConnection GetAdbcConnection(BigQueryTestConfiguration testConfiguration)
+ private Adbc.Client.AdbcConnection GetAdbcConnection()
{
return new Adbc.Client.AdbcConnection(
new BigQueryDriver(),
- BigQueryTestingUtils.GetBigQueryParameters(testConfiguration),
+ BigQueryTestingUtils.GetBigQueryParameters(_testConfiguration),
new Dictionary<string,string>()
);
}
diff --git a/csharp/test/Drivers/BigQuery/DriverTests.cs b/csharp/test/Drivers/BigQuery/DriverTests.cs
index c94c737e..2af6de33 100644
--- a/csharp/test/Drivers/BigQuery/DriverTests.cs
+++ b/csharp/test/Drivers/BigQuery/DriverTests.cs
@@ -35,9 +35,13 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[TestCaseOrderer("Apache.Arrow.Adbc.Tests.Xunit.TestOrderer", "Apache.Arrow.Adbc.Tests")]
public class DriverTests
{
+ BigQueryTestConfiguration _testConfiguration;
+
public DriverTests()
{
Skip.IfNot(Utils.CanExecuteTestConfig(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE));
+ _testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
+
}
/// <summary>
@@ -47,11 +51,10 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(1)]
public void CanExecuteUpdate()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
- string[] queries = BigQueryTestingUtils.GetQueries(testConfiguration);
+ string[] queries = BigQueryTestingUtils.GetQueries(_testConfiguration);
List<int> expectedResults = new List<int>() { -1, 1, 1 };
@@ -73,9 +76,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(2)]
public void CanGetInfo()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
IArrowArrayStream stream = adbcConnection.GetInfo(new List<AdbcInfoCode>() { AdbcInfoCode.DriverName, AdbcInfoCode.DriverVersion, AdbcInfoCode.VendorName });
@@ -102,15 +103,13 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(3)]
public void CanGetObjects()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
// need to add the database
- string catalogName = testConfiguration.Metadata.Catalog;
- string schemaName = testConfiguration.Metadata.Schema;
- string tableName = testConfiguration.Metadata.Table;
+ string catalogName = _testConfiguration.Metadata.Catalog;
+ string schemaName = _testConfiguration.Metadata.Schema;
+ string tableName = _testConfiguration.Metadata.Table;
string columnName = null;
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
IArrowArrayStream stream = adbcConnection.GetObjects(
depth: AdbcConnection.GetObjectsDepth.All,
@@ -132,7 +131,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
.Select(c => c.Columns)
.FirstOrDefault();
- Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, columns.Count);
+ Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, columns.Count);
}
/// <summary>
@@ -141,19 +140,17 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(4)]
public void CanGetTableSchema()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
-
- string catalogName = testConfiguration.Metadata.Catalog;
- string schemaName = testConfiguration.Metadata.Schema;
- string tableName = testConfiguration.Metadata.Table;
+ string catalogName = _testConfiguration.Metadata.Catalog;
+ string schemaName = _testConfiguration.Metadata.Schema;
+ string tableName = _testConfiguration.Metadata.Table;
Schema schema = adbcConnection.GetTableSchema(catalogName, schemaName, tableName);
int numberOfFields = schema.FieldsList.Count;
- Assert.Equal(testConfiguration.Metadata.ExpectedColumnCount, numberOfFields);
+ Assert.Equal(_testConfiguration.Metadata.ExpectedColumnCount, numberOfFields);
}
/// <summary>
@@ -162,9 +159,7 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(5)]
public void CanGetTableTypes()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
IArrowArrayStream arrowArrayStream = adbcConnection.GetTableTypes();
@@ -199,16 +194,14 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[SkippableFact, Order(6)]
public void CanExecuteQuery()
{
- BigQueryTestConfiguration testConfiguration = Utils.LoadTestConfiguration<BigQueryTestConfiguration>(BigQueryTestingUtils.BIGQUERY_TEST_CONFIG_VARIABLE);
-
- AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(testConfiguration);
+ AdbcConnection adbcConnection = BigQueryTestingUtils.GetBigQueryAdbcConnection(_testConfiguration);
AdbcStatement statement = adbcConnection.CreateStatement();
- statement.SqlQuery = testConfiguration.Query;
+ statement.SqlQuery = _testConfiguration.Query;
QueryResult queryResult = statement.ExecuteQuery();
- Tests.DriverTests.CanExecuteQuery(queryResult, testConfiguration.ExpectedResultsCount);
+ Tests.DriverTests.CanExecuteQuery(queryResult, _testConfiguration.ExpectedResultsCount);
}
}
}
diff --git a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs
index d1e7b9dc..2f394a89 100644
--- a/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs
+++ b/csharp/test/Drivers/Interop/Snowflake/SnowflakeTestingUtils.cs
@@ -81,14 +81,14 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Interop.Snowflake
{ SnowflakeParameters.USE_HIGH_PRECISION, testConfiguration.UseHighPrecision.ToString().ToLowerInvariant() }
};
- if(testConfiguration.Authentication.Default is not null)
+ if (testConfiguration.Authentication.Default is not null)
{
parameters[SnowflakeParameters.AUTH_TYPE] = SnowflakeAuthentication.AuthSnowflake;
parameters[SnowflakeParameters.USERNAME] = testConfiguration.Authentication.Default.User;
parameters[SnowflakeParameters.PASSWORD] = testConfiguration.Authentication.Default.Password;
}
- if(!string.IsNullOrWhiteSpace(testConfiguration.Host))
+ if (!string.IsNullOrWhiteSpace(testConfiguration.Host))
{
parameters[SnowflakeParameters.HOST] = testConfiguration.Host;
}