You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2022/12/30 17:18:51 UTC
[beam] branch master updated: Retry create database in InfluxDbIOIT (#24800)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3ee5b86a3f1 Retry create database in InfluxDbIOIT (#24800)
3ee5b86a3f1 is described below
commit 3ee5b86a3f16dc92b6825773cf134d1315610251
Author: Yi Hu <ya...@google.com>
AuthorDate: Fri Dec 30 12:18:39 2022 -0500
Retry create database in InfluxDbIOIT (#24800)
* Retry create database in InfluxDbIOIT
* Rename InfluxDb performance test to post commit
* This test is actually an integration test on direct runner
* Revert #23680 relax job node
* Update .test-infra/jenkins/README
---
.test-infra/jenkins/README.md | 7 +++++--
...vy => job_PostCommit_Java_InfluxDBIO_IT.groovy} | 15 +++++++-------
.../apache/beam/sdk/io/influxdb/InfluxDbIOIT.java | 24 ++++++++++++++++++++--
3 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md
index 8bc4b6f9e25..34e3f4d7b20 100644
--- a/.test-infra/jenkins/README.md
+++ b/.test-infra/jenkins/README.md
@@ -41,6 +41,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
| beam_PreCommit_Java_Kafka_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka_IO_Direct_Phrase/) | `Run Java_Kafka_IO_Direct PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kafka [...]
| beam_PreCommit_Java_Kinesis_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Kinesis_IO_Direct_Phrase/) | `Run Java_Kinesis_IO_Direct PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommi [...]
| beam_PreCommit_Java_Neo4j_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j_IO_Direct_Phrase/) | `Run Java_Neo4j_IO_Direct PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Java_Neo4j [...]
+| beam_PreCommit_Java_SingleStore_IO_Direct | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Java_SingleStore_IO_Direct_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Java_SingleStore_IO_Direct_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Java_SingleStore_IO_Direct_Phrase/) | `Run Java_SingleStore_IO_Direct PreCommit` | [](https://ci-beam.apac [...]
| beam_PreCommit_Portable_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Phrase/) | `Run Portable_Python PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Portable_Python_Cron) |
| beam_PreCommit_PythonLint | [commit](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Phrase/) | `Run PythonLint PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_PythonLint_Cron) |
| beam_PreCommit_Python | [commit](https://ci-beam.apache.org/job/beam_PreCommit_Python_Commit/), [cron](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron/), [phrase](https://ci-beam.apache.org/job/beam_PreCommit_Python_Phrase/) | `Run Python PreCommit` | [](https://ci-beam.apache.org/job/beam_PreCommit_Python_Cron) |
@@ -81,6 +82,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
| beam_PostCommit_Java | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_PR/) | `Run Java PostCommit` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java) |
| beam_PostCommit_Java_DataflowV1 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1_PR/) | `Run PostCommit_Java_Dataflow` | [](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV1/) |
| beam_PostCommit_Java_DataflowV2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2_PR/) | `Run PostCommit_Java_DataflowV2` | [](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_DataflowV2/) |
+| beam_PostCommit_Java_InfluxDbIO_IT | [cron](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PostCommit_Java_InfluxDbIO_IT/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_SingleStoreIO_IT_PR/) | `Run Java InfluxDbIO_IT` | [](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PostCommit_Java_InfluxDbIO_IT/) |
| beam_PostCommit_Java_Nexmark_Dataflow | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow_PR/) | `Dataflow Runner Nexmark Tests` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_Nexmark_Dataflow) |
| beam_PostCommit_Java_Nexmark_Dataflow_V2 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_DataflowV2_PR/) | `Run Dataflow Runner V2 Nexmark Tests` | [](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2/) |
| beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11 | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11/badge/icon) | `Dataflow Runner V2 Java 11 Nexmark Tests` | [](https://ci-beam.apache.org/view/PostCommit/job/ [...]
@@ -93,6 +95,7 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
| beam_PostCommit_Java_PVR_Spark_Batch | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch_PR/) | `Run Java Spark PortableValidatesRunner Batch` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_PVR_Spark_Batch) |
| beam_PostCommit_Java_PVR_Spark2_Streaming | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_PVR_Spark2_Streaming/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_PVR_Spark2_Streaming_PR/) | `Run Java Spark v2 PortableValidatesRunner Streaming` | [](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java [...]
| beam_PostCommit_Java_PVR_Spark3_Streaming | [cron](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_PVR_Spark3_Streaming/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_PVR_Spark3_Streaming_PR/) | `Run Java Spark v3 PortableValidatesRunner Streaming` | [](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java [...]
+| beam_PostCommit_Java_SingleStoreIO_IT | [cron](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PostCommit_Java_SingleStoreIO_IT/), [phrase](https://ci-beam.apache.org/view/PostCommit/job/beam_PostCommit_Java_SingleStoreIO_IT_PR/) | `Run Java SingleStoreIO_IT` | [](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PostCommit_Java_SingleStoreIO_IT/) |
| beam_PostCommit_Java_Dataflow_Examples_Java11 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_Java11/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_Java11_PR/) | `Run Java examples on Dataflow Java 11` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_Java11) |
| beam_PostCommit_Java_Examples_Dataflow_V2 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2/) | `Run Java Examples on Dataflow Runner V2` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2) |
| beam_PostCommit_Java_Examples_Dataflow_V2_Java11 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples__Dataflow_V2_Java11/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples__Dataflow_V2_Java11/) | `Run Java 11 Examples on Dataflow Runner V2` | [](https://ci-beam.apache.org/job/beam_PostCommit_Java_Examples_Dataflow_V2_Java11) |
@@ -151,16 +154,16 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
| beam_PerformanceTests_Cdap | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_Cdap/) | `Run Java CdapIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_Cdap) |
| beam_PerformanceTests_Compressed_TextIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT_HDFS/) | `Run Java CompressedTextIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_Compressed_TextIOIT) [ | `Run Java HadoopFormatIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_HadoopFormat) |
-| beam_PerformanceTests_InfluxDbIO_IT | [cron](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_InfluxDbIO_IT/) | `Run Java InfluxDbIO Performance Test` | [](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_InfluxDbIO_IT/) |
| beam_PerformanceTests_JDBC | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_JDBC/) | `Run Java JdbcIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_JDBC) |
| beam_PerformanceTests_KafkaIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_Kafka_IO/) | `Run Java KafkaIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_Kafka_IO) [](https://ci-beam.apache.org/job/beam_PerformanceTests_Kafka_IO) |
| beam_PerformanceTests_ManyFiles_TextIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS/) | `Run Java ManyFilesTextIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_ManyFiles_TextIOIT) [ | `Run Java MongoDBIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_MongoDBIO_IT) |
| beam_PerformanceTests_ParquetIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_ParquetIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_ParquetIOIT_HDFS/) | `Run Java ParquetIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_ParquetIOIT) [, [phrase](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming_PR/) | `Run PubsubIO Performance Test Python` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_PubsubIOIT_Python_Streaming) |
+| beam_PerformanceTests_SingleStoreIO | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SingleStoreIO/) | `Run Java SingleStoreIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_SingleStoreIO) |
| beam_PerformanceTests_SpannerIO_Read_2GB_Python | [cron](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_SpannerIO_Read_2GB_Python/), [phrase](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_SpannerIO_Read_2GB_Python_PR/) | `Run SpannerIO Read 2GB Performance Test Python Batch` | [](https://ci-beam.apache.o [...]
| beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch/), [phrase](https://ci-beam.apache.org/view/PerformanceTests/job/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch_PR/) | `Run SpannerIO Write 2GB Performance Test Python Batch` | [](https://ci-beam.apache.org/job/beam_Per [...]
-| beam_PerformanceTests_SparkReceiverIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiverIOIT/) | `Run Java SparkReceiverIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_Spa [...]
+| beam_PerformanceTests_SparkReceiverIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiverIOIT/) | `Run Java SparkReceiverIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_SparkReceiverIO) |
| beam_PerformanceTests_TFRecordIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TFRecordIOIT/) | `Run Java TFRecordIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_TFRecordIOIT) |
| beam_PerformanceTests_TextIOIT | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT/), [hdfs_cron](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT_HDFS/) | `Run Java TextIO Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_TextIOIT) [](https://ci-be [...]
| beam_PerformanceTests_WordCountIT_Py37 | [cron](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/) | `Run Python37 WordCountIT Performance Test` | [](https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37) |
diff --git a/.test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy
similarity index 83%
rename from .test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy
rename to .test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy
index 18a0f1e685f..cb74cbf3228 100644
--- a/.test-infra/jenkins/job_PerformanceTests_InfluxDBIO_IT.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_InfluxDBIO_IT.groovy
@@ -16,19 +16,18 @@
* limitations under the License.
*/
import CommonJobProperties as common
+import PostcommitJobBuilder
import Kubernetes
-String jobName = "beam_PerformanceTests_InfluxDbIO_IT"
+String jobName = "beam_PostCommit_Java_InfluxDbIO_IT"
-job(jobName) {
+PostcommitJobBuilder.postCommitJob(jobName, 'Run Java InfluxDbIO_IT', 'Java InfluxDbIO Integration Test', this) {
+ description('Runs the Java InfluxDbIO Integration Test.')
+ previousNames(/beam_PerformanceTests_InfluxDbIO_IT/)
// Set common parameters.
- common.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
- common.setAutoJob(delegate,'H H/6 * * *')
- common.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java InfluxDbIO Performance Test',
- 'Run Java InfluxDbIO Performance Test')
+ common.setTopLevelMainJobProperties(delegate)
+ // Deploy InfluxDb cluster
String namespace = common.getKubernetesNamespace(jobName)
String kubeconfigPath = common.getKubeconfigLocationForNamespace(namespace)
Kubernetes k8s = Kubernetes.create(delegate, kubeconfigPath, namespace)
diff --git a/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java b/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java
index 77042b882dc..da725ab541b 100644
--- a/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java
+++ b/sdks/java/io/influxdb/src/test/java/org/apache/beam/sdk/io/influxdb/InfluxDbIOIT.java
@@ -28,10 +28,15 @@ import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.BackOffUtils;
+import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.Sleeper;
import org.apache.beam.sdk.values.PCollection;
import org.influxdb.InfluxDB;
import org.influxdb.InfluxDB.ConsistencyLevel;
import org.influxdb.InfluxDBFactory;
+import org.influxdb.InfluxDBIOException;
import org.influxdb.dto.Query;
import org.junit.After;
import org.junit.Before;
@@ -90,13 +95,28 @@ public class InfluxDbIOIT {
}
@Before
- public void initTest() {
+ public void initTest() throws IOException, InterruptedException {
+ BackOff backOff = FluentBackoff.DEFAULT.withMaxRetries(4).backoff();
+ Query createQuery = new Query(String.format("CREATE DATABASE %s", options.getDatabaseName()));
try (InfluxDB connection =
InfluxDBFactory.connect(
options.getInfluxDBURL(),
options.getInfluxDBUserName(),
options.getInfluxDBPassword())) {
- connection.query(new Query(String.format("CREATE DATABASE %s", options.getDatabaseName())));
+ InfluxDBIOException lastException;
+ // retry create database
+ do {
+ try {
+ connection.query(createQuery);
+ lastException = null;
+ break;
+ } catch (InfluxDBIOException e) {
+ lastException = e;
+ }
+ } while (BackOffUtils.next(Sleeper.DEFAULT, backOff));
+ if (lastException != null) {
+ throw lastException;
+ }
}
}