You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yi...@apache.org on 2022/05/10 23:00:38 UTC
[beam] 01/01: Revert "[BEAM-14429] Force java load test on dataflow runner v2 forceNumInitialBundles to 1 (#17576)"
This is an automated email from the ASF dual-hosted git repository.
yichi pushed a commit to branch revert-17576-pardo_split
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 812ac171d99505ef8728e2c06a0a91ba7e48441c
Author: Yichi Zhang <zy...@google.com>
AuthorDate: Tue May 10 16:00:30 2022 -0700
Revert "[BEAM-14429] Force java load test on dataflow runner v2 forceNumInitialBundles to 1 (#17576)"
This reverts commit 178441dcf0a1e03611c46bdd12ab27907289e310.
---
.../job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy | 10 ----------
.../job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy | 10 ----------
.../job_LoadTests_GBK_Dataflow_V2_Java11.groovy | 19 +++++--------------
.../job_LoadTests_GBK_Dataflow_V2_Java17.groovy | 19 +++++--------------
.../job_LoadTests_ParDo_Dataflow_V2_Java11.groovy | 14 ++++----------
.../job_LoadTests_ParDo_Dataflow_V2_Java17.groovy | 14 ++++----------
6 files changed, 18 insertions(+), 68 deletions(-)
diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy
index f96bd35b825..bd4b97c07e5 100644
--- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy
+++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def loadTestConfigurations = { mode, isStreaming ->
[
[
@@ -49,7 +47,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1
}
""".trim().replaceAll("\\s", ""),
@@ -58,7 +55,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -90,7 +86,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 5
}
""".trim().replaceAll("\\s", ""),
@@ -99,7 +94,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -132,7 +126,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 200000
}
""".trim().replaceAll("\\s", ""),
@@ -141,7 +134,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -174,7 +166,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -183,7 +174,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy
index 72646fab8be..0978753f40e 100644
--- a/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy
+++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def loadTestConfigurations = { mode, isStreaming ->
[
[
@@ -49,7 +47,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1
}
""".trim().replaceAll("\\s", ""),
@@ -58,7 +55,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -90,7 +86,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 5
}
""".trim().replaceAll("\\s", ""),
@@ -99,7 +94,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -132,7 +126,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 200000
}
""".trim().replaceAll("\\s", ""),
@@ -141,7 +134,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -174,7 +166,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
@@ -183,7 +174,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy
index 246ed4f0eaf..30c9dda3763 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def loadTestConfigurations = { mode, isStreaming ->
[
[
@@ -48,8 +46,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 200000000,
"keySizeBytes": 1,
- "valueSizeBytes": 9,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 9
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -80,8 +77,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -113,8 +109,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000,
"keySizeBytes": 10000,
- "valueSizeBytes": 90000,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90000
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -146,8 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 5000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 4,
@@ -178,8 +172,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 2500000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 8,
@@ -211,7 +204,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 200,
"hotKeyFraction": 1
}
@@ -245,7 +237,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 10,
"hotKeyFraction": 1
}
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy
index f015d828870..ca897e7eec1 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def loadTestConfigurations = { mode, isStreaming ->
[
[
@@ -48,8 +46,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 200000000,
"keySizeBytes": 1,
- "valueSizeBytes": 9,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 9
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -80,8 +77,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -113,8 +109,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000,
"keySizeBytes": 10000,
- "valueSizeBytes": 90000,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90000
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
@@ -146,8 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 5000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 4,
@@ -178,8 +172,7 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 2500000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
fanout : 8,
@@ -211,7 +204,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 200,
"hotKeyFraction": 1
}
@@ -245,7 +237,6 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
- "forceNumInitialBundles": 1,
"numHotKeys": 10,
"hotKeyFraction": 1
}
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy
index 86f5e6fe732..0a1cf8cfde5 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def commonLoadTestConfig = { jobType, isStreaming ->
[
[
@@ -48,8 +46,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 10,
@@ -81,8 +78,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 200,
@@ -115,8 +111,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
@@ -149,8 +144,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy
index ac40f39e62c..f09787225b9 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy
@@ -23,8 +23,6 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper
-// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.
-
def commonLoadTestConfig = { jobType, isStreaming ->
[
[
@@ -48,8 +46,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 10,
@@ -81,8 +78,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 200,
@@ -115,8 +111,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
@@ -149,8 +144,7 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
- "valueSizeBytes": 90,
- "forceNumInitialBundles": 1
+ "valueSizeBytes": 90
}
""".trim().replaceAll("\\s", ""),
iterations : 1,