You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ar...@apache.org on 2023/04/20 11:26:19 UTC
[beam] branch master updated: Make `withDesiredBundleSizeBytes` methods public (#25448)
This is an automated email from the ASF dual-hosted git repository.
aromanenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f1e95970ca4 Make `withDesiredBundleSizeBytes` methods public (#25448)
f1e95970ca4 is described below
commit f1e95970ca4cb9f4ed9099761ba6a757099d22d8
Author: Israel Herraiz <ih...@google.com>
AuthorDate: Thu Apr 20 13:26:08 2023 +0200
Make `withDesiredBundleSizeBytes` methods public (#25448)
* Make `withDesiredBundleSizeBytes` methods public
With these methods being private, the bundle size is 64 MB, which is too large for streaming applications using these classes.
This changes the method visibility to public so a smaller bundle size can be used for streaming.
Fixes #25447
* Add Javadoc to new public methods
* Make withDesiredBundleSizeBytes public, useful for streaming applications
---
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 28 +++++++++++++++-------
.../apache/beam/sdk/extensions/avro/io/AvroIO.java | 28 +++++++++++++++-------
2 files changed, 40 insertions(+), 16 deletions(-)
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index c8ccf753229..42fab212576 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -813,8 +813,11 @@ public class AvroIO {
abstract ReadFiles<T> build();
}
- @VisibleForTesting
- ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -927,8 +930,11 @@ public class AvroIO {
getMatchConfiguration().continuously(pollInterval, terminationCondition));
}
- @VisibleForTesting
- ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -1169,8 +1175,11 @@ public class AvroIO {
return toBuilder().setFileExceptionHandler(exceptionHandler).build();
}
- @VisibleForTesting
- ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -1278,8 +1287,11 @@ public class AvroIO {
return toBuilder().setCoder(coder).build();
}
- @VisibleForTesting
- ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
index b159d38c603..f35fc66e654 100644
--- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
+++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
@@ -816,8 +816,11 @@ public class AvroIO {
abstract ReadFiles<T> build();
}
- @VisibleForTesting
- ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -930,8 +933,11 @@ public class AvroIO {
getMatchConfiguration().continuously(pollInterval, terminationCondition));
}
- @VisibleForTesting
- ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -1172,8 +1178,11 @@ public class AvroIO {
return toBuilder().setFileExceptionHandler(exceptionHandler).build();
}
- @VisibleForTesting
- ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}
@@ -1281,8 +1290,11 @@ public class AvroIO {
return toBuilder().setCoder(coder).build();
}
- @VisibleForTesting
- ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
+ /**
+ * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a
+ * lower value (e.g. 1 MB) for streaming applications.
+ */
+ public ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) {
return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build();
}