You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by cu...@apache.org on 2024/10/30 17:08:31 UTC
(arrow-adbc) branch main updated: feat(csharp/src/Drivers/BigQuery): support max stream count setting when creating read session (#2289)
This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 8524b911a feat(csharp/src/Drivers/BigQuery): support max stream count setting when creating read session (#2289)
8524b911a is described below
commit 8524b911adee0ffa4d055c9b336cbebfd2dc7dbe
Author: qifanzhang-ms <15...@users.noreply.github.com>
AuthorDate: Thu Oct 31 01:08:26 2024 +0800
feat(csharp/src/Drivers/BigQuery): support max stream count setting when creating read session (#2289)
---
csharp/src/Drivers/BigQuery/BigQueryConnection.cs | 3 ++-
csharp/src/Drivers/BigQuery/BigQueryParameters.cs | 1 +
csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 13 ++++++++++++-
csharp/src/Drivers/BigQuery/readme.md | 3 +++
csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs | 3 +++
csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs | 5 +++++
csharp/test/Drivers/BigQuery/Resources/bigqueryconfig.json | 3 ++-
csharp/test/Drivers/BigQuery/readme.md | 1 +
8 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
index acba314eb..398365314 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs
@@ -1002,7 +1002,8 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
BigQueryParameters.UseLegacySQL,
BigQueryParameters.LargeDecimalsAsString,
BigQueryParameters.LargeResultsDestinationTable,
- BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes
+ BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes,
+ BigQueryParameters.MaxFetchConcurrency
};
foreach (string key in statementOptions)
diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
index 3f9aafb20..51272eb64 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs
@@ -35,6 +35,7 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
public const string Scopes = "adbc.bigquery.scopes";
public const string IncludeConstraintsWithGetObjects = "adbc.bigquery.include_constraints_getobjects";
public const string GetQueryResultsOptionsTimeoutMinutes = "adbc.bigquery.get_query_results_options.timeout";
+ public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency";
}
/// <summary>
diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
index d21f711c0..91e1f8356 100644
--- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
+++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs
@@ -103,8 +103,19 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
string table = $"projects/{results.TableReference.ProjectId}/datasets/{results.TableReference.DatasetId}/tables/{results.TableReference.TableId}";
+ int maxStreamCount = 1;
+ if (this.Options?.TryGetValue(BigQueryParameters.MaxFetchConcurrency, out string? maxStreamCountString) == true)
+ {
+ if (int.TryParse(maxStreamCountString, out int count))
+ {
+ if (count >= 0)
+ {
+ maxStreamCount = count;
+ }
+ }
+ }
ReadSession rs = new ReadSession { Table = table, DataFormat = DataFormat.Arrow };
- ReadSession rrs = readClient.CreateReadSession("projects/" + results.TableReference.ProjectId, rs, 1);
+ ReadSession rrs = readClient.CreateReadSession("projects/" + results.TableReference.ProjectId, rs, maxStreamCount);
long totalRows = results.TotalRows == null ? -1L : (long)results.TotalRows.Value;
IArrowArrayStream stream = new MultiArrowReader(TranslateSchema(results.Schema), rrs.Streams.Select(s => ReadChunk(readClient, s.Name)));
diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md
index eafaf2c86..d78ab4c89 100644
--- a/csharp/src/Drivers/BigQuery/readme.md
+++ b/csharp/src/Drivers/BigQuery/readme.md
@@ -54,6 +54,9 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G
**adbc.bigquery.get_query_results_options.timeout**<br>
Optional. Sets the timeout (in minutes) for the GetQueryResultsOptions value. If not set, defaults to 5 minutes.
+**adbc.bigquery.max_fetch_concurrency**<br>
+ Optional. Sets the [maxStreamCount](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.Storage.V1/latest/Google.Cloud.BigQuery.Storage.V1.BigQueryReadClient#Google_Cloud_BigQuery_Storage_V1_BigQueryReadClient_CreateReadSession_System_String_Google_Cloud_BigQuery_Storage_V1_ReadSession_System_Int32_Google_Api_Gax_Grpc_CallSettings_) for the CreateReadSession method. If not set, defaults to 1.
+
**adbc.bigquery.include_constraints_getobjects**<br>
Optional. Some callers do not need the constraint details when they get the table information and can improve the speed of obtaining the results. Setting this value to `"false"` will not include the constraint details. The default value is `"true"`.
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
index a1782910f..ffcae7cc0 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs
@@ -59,5 +59,8 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
[JsonPropertyName("timeoutMinutes")]
public int? TimeoutMinutes { get; set; }
+
+ [JsonPropertyName("maxStreamCount")]
+ public int? MaxStreamCount { get; set; }
}
}
diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
index bf80aa256..2bc6227ba 100644
--- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
+++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs
@@ -91,6 +91,11 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery
parameters.Add(BigQueryParameters.GetQueryResultsOptionsTimeoutMinutes, testConfiguration.TimeoutMinutes.Value.ToString());
}
+ if (testConfiguration.MaxStreamCount.HasValue)
+ {
+ parameters.Add(BigQueryParameters.MaxFetchConcurrency, testConfiguration.MaxStreamCount.Value.ToString());
+ }
+
return parameters;
}
diff --git a/csharp/test/Drivers/BigQuery/Resources/bigqueryconfig.json b/csharp/test/Drivers/BigQuery/Resources/bigqueryconfig.json
index 888ae7eb3..960bcb868 100644
--- a/csharp/test/Drivers/BigQuery/Resources/bigqueryconfig.json
+++ b/csharp/test/Drivers/BigQuery/Resources/bigqueryconfig.json
@@ -3,11 +3,12 @@
"clientId": "",
"clientSecret": "",
"refreshToken": "",
+ "maxStreamCount": 1,
"metadata": {
"catalog": "",
"schema": "",
"table": "",
- "expectedColumnCount": 0
+ "expectedColumnCount": 0
},
"query": "",
"expectedResults": 0
diff --git a/csharp/test/Drivers/BigQuery/readme.md b/csharp/test/Drivers/BigQuery/readme.md
index daffddc5f..26f177da4 100644
--- a/csharp/test/Drivers/BigQuery/readme.md
+++ b/csharp/test/Drivers/BigQuery/readme.md
@@ -39,6 +39,7 @@ The following values can be setup in the configuration
- **query** - The query to use.
- **expectedResults** - The expected number of results from the query.
- **timeoutMinutes** - The timeout (in minutes).
+- **maxStreamCount** - The max stream count.
- **includeTableConstraints** - Whether to include table constraints in the GetObjects query.
- **largeResultsDestinationTable** - Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76f [...]
- **allowLargeResults** - Whether to allow large results .