You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by mw...@apache.org on 2020/06/19 13:09:04 UTC
[beam] 01/01: [BEAM-10258] Add tempRoot to jobs using
TestDataflowRunner
This is an automated email from the ASF dual-hosted git repository.
mwalenia pushed a commit to branch BEAM-10285-fix-dataflow
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 1332a161b5b9801370cddd161599d6d6916d578f
Author: Michal Walenia <mi...@polidea.com>
AuthorDate: Fri Jun 19 15:07:37 2020 +0200
[BEAM-10258] Add tempRoot to jobs using TestDataflowRunner
---
.test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy | 4 ++++
.test-infra/jenkins/job_LoadTests_Combine_Java.groovy | 3 +++
.test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy | 1 +
.test-infra/jenkins/job_LoadTests_GBK_Java.groovy | 7 +++++++
.test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy | 1 +
.test-infra/jenkins/job_LoadTests_ParDo_Java.groovy | 4 ++++
6 files changed, 20 insertions(+)
diff --git a/.test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy b/.test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy
index d235273..f40a960 100644
--- a/.test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy
+++ b/.test-infra/jenkins/job_LoadTests_CoGBK_Java.groovy
@@ -34,6 +34,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_CoGBK_1",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_CoGBK_1",
@@ -70,6 +71,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_CoGBK_2",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_CoGBK_2",
@@ -107,6 +109,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_CoGBK_3",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_CoGBK_3",
@@ -144,6 +147,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_CoGBK_4",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_CoGBK_4",
diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Java.groovy b/.test-infra/jenkins/job_LoadTests_Combine_Java.groovy
index 98c2967..231d977 100644
--- a/.test-infra/jenkins/job_LoadTests_Combine_Java.groovy
+++ b/.test-infra/jenkins/job_LoadTests_Combine_Java.groovy
@@ -35,6 +35,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_Combine_1",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_Combine_1",
@@ -65,6 +66,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_Combine_4",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_Combine_4",
@@ -95,6 +97,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_Combine_5",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_Combine_5",
diff --git a/.test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy b/.test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy
index e659ac5..48cd414 100644
--- a/.test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy
+++ b/.test-infra/jenkins/job_LoadTests_Combine_Java_Smoke.groovy
@@ -34,6 +34,7 @@ def smokeTestConfigurations = { datasetName ->
appName 'smoke-dsl-java'
project 'apache-beam-testing'
tempLocation 'gs://temp-storage-for-perf-tests/smoketests'
+ tempRoot 'gs://temp-storage-for-perf-tests/smoketests'
publishToBigQuery true
bigQueryDataset datasetName
bigQueryTable 'dataflow_combine'
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Java.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Java.groovy
index 030c67f..36d26cc 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Java.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Java.groovy
@@ -34,6 +34,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_1",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_1",
@@ -62,6 +63,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_2",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_2",
@@ -91,6 +93,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_3",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_3",
@@ -120,6 +123,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : 'load_tests_Java_Dataflow_${mode}_GBK_4',
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_4",
@@ -148,6 +152,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_5",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_5",
@@ -176,6 +181,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_6",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_6",
@@ -206,6 +212,7 @@ def loadTestConfigurations = { mode, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${mode}_GBK_7",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${mode}_GBK_7",
diff --git a/.test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy b/.test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy
index d8d9b1d..62b21f8 100644
--- a/.test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy
+++ b/.test-infra/jenkins/job_LoadTests_GBK_Java_Smoke.groovy
@@ -43,6 +43,7 @@ def smokeTestConfigurations = { datasetName -> [
project : 'apache-beam-testing',
region : 'us-central1',
tempLocation : 'gs://temp-storage-for-perf-tests/smoketests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/smoketests',
publishToBigQuery: true,
bigQueryDataset : datasetName,
bigQueryTable : 'dataflow_gbk',
diff --git a/.test-infra/jenkins/job_LoadTests_ParDo_Java.groovy b/.test-infra/jenkins/job_LoadTests_ParDo_Java.groovy
index bc49aa3..ea35dfd 100644
--- a/.test-infra/jenkins/job_LoadTests_ParDo_Java.groovy
+++ b/.test-infra/jenkins/job_LoadTests_ParDo_Java.groovy
@@ -34,6 +34,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_ParDo_1",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_ParDo_1",
@@ -63,6 +64,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_ParDo_2",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_ParDo_2",
@@ -93,6 +95,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_ParDo_3",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_ParDo_3",
@@ -123,6 +126,7 @@ def commonLoadTestConfig = { jobType, isStreaming, datasetName ->
region : 'us-central1',
appName : "load_tests_Java_Dataflow_${jobType}_ParDo_4",
tempLocation : 'gs://temp-storage-for-perf-tests/loadtests',
+ tempRoot : 'gs://temp-storage-for-perf-tests/loadtests',
publishToBigQuery : true,
bigQueryDataset : datasetName,
bigQueryTable : "java_dataflow_${jobType}_ParDo_4",