You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/10/16 16:00:53 UTC
(beam) branch master updated: Fix StressTests Java BigQueryIO job flakiness (#32803)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 73112707381 Fix StressTests Java BigQueryIO job flakiness (#32803)
73112707381 is described below
commit 73112707381438f0ec09a6a8ef79d1122a2f0535
Author: Vitaly Terentyev <vi...@akvelon.com>
AuthorDate: Wed Oct 16 20:00:47 2024 +0400
Fix StressTests Java BigQueryIO job flakiness (#32803)
* Fix assert for at least once method
* Refactoring
---
.../java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
index 22ff94e293b..ddb300d74f6 100644
--- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
+++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
@@ -84,6 +84,7 @@ public final class BigQueryIOST extends IOStressTestBase {
private static final String READ_ELEMENT_METRIC_NAME = "read_count";
private static final String STORAGE_WRITE_API_METHOD = "STORAGE_WRITE_API";
private static final String STORAGE_API_AT_LEAST_ONCE_METHOD = "STORAGE_API_AT_LEAST_ONCE";
+ private static final double STORAGE_API_AT_LEAST_ONCE_MAX_ALLOWED_DIFFERENCE_FRACTION = 0.00001;
private static BigQueryResourceManager resourceManager;
private static String tableName;
@@ -334,11 +335,14 @@ public final class BigQueryIOST extends IOStressTestBase {
// Depending on writing method there might be duplicates on different sides (read or write).
if (configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)) {
+ long allowedDifference =
+ (long) (numRecords * STORAGE_API_AT_LEAST_ONCE_MAX_ALLOWED_DIFFERENCE_FRACTION);
+ long actualDifference = (long) numRecords - rowCount;
assertTrue(
String.format(
- "Number of rows in the table (%d) is less than the expected number (%d). Missing records: %d",
- rowCount, (long) numRecords, (long) numRecords - rowCount),
- rowCount >= numRecords);
+ "Row difference (%d) exceeds the limit of %d. Rows: %d, Expected: %d",
+ actualDifference, allowedDifference, rowCount, (long) numRecords),
+ actualDifference <= allowedDifference);
} else {
assertTrue(
String.format(