You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/08/16 21:48:52 UTC
(beam) branch master updated: Make BQ file load limit controls public (#32101)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 541e5c85031 Make BQ file load limit controls public (#32101)
541e5c85031 is described below
commit 541e5c85031a4c20a2c979e1b126ddaad2e0bd24
Author: Ahmed Abualsaud <65...@users.noreply.github.com>
AuthorDate: Fri Aug 16 14:48:45 2024 -0700
Make BQ file load limit controls public (#32101)
---
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 28 ++++++++++++++++++----
1 file changed, 24 insertions(+), 4 deletions(-)
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 2a16bf31a6c..aa094ace756 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -3199,14 +3199,34 @@ public class BigQueryIO {
return toBuilder().setMaxFilesPerBundle(maxFilesPerBundle).build();
}
- @VisibleForTesting
- Write<T> withMaxFileSize(long maxFileSize) {
+ /**
+ * Controls the maximum byte size per file to be loaded into BigQuery. If the amount of data
+ * written to one file reaches this threshold, we will close that file and continue writing in a
+ * new file.
+ *
+ * <p>The default value (4 TiB) respects BigQuery's maximum number of source URIs per job
+ * configuration.
+ *
+ * @see <a href="https://cloud.google.com/bigquery/quotas#load_jobs">BigQuery Load Job
+ * Limits</a>
+ */
+ public Write<T> withMaxFileSize(long maxFileSize) {
checkArgument(maxFileSize > 0, "maxFileSize must be > 0, but was: %s", maxFileSize);
return toBuilder().setMaxFileSize(maxFileSize).build();
}
- @VisibleForTesting
- Write<T> withMaxFilesPerPartition(int maxFilesPerPartition) {
+ /**
+ * Controls how many files will be assigned to a single BigQuery load job. If the number of
+ * files increases past this threshold, we will spill it over into multiple load jobs as
+ * necessary.
+ *
+ * <p>The default value (10,000 files) respects BigQuery's maximum number of source URIs per job
+ * configuration.
+ *
+ * @see <a href="https://cloud.google.com/bigquery/quotas#load_jobs">BigQuery Load Job
+ * Limits</a>
+ */
+ public Write<T> withMaxFilesPerPartition(int maxFilesPerPartition) {
checkArgument(
maxFilesPerPartition > 0,
"maxFilesPerPartition must be > 0, but was: %s",