You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jb...@apache.org on 2017/03/03 07:34:02 UTC
[1/2] beam git commit: [BEAM-1310] Add running integration tests in
JdbcIO on Spark and Dataflow runners
Repository: beam
Updated Branches:
refs/heads/master 0a6211b56 -> 178381992
[BEAM-1310] Add running integration tests in JdbcIO on Spark and Dataflow runners
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eb87ffc7
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eb87ffc7
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eb87ffc7
Branch: refs/heads/master
Commit: eb87ffc769df6ab410b90a599489034a83aede16
Parents: 0a6211b
Author: Stephen Sisk <si...@google.com>
Authored: Thu Feb 23 14:01:18 2017 -0800
Committer: Jean-Baptiste Onofr� <jb...@apache.org>
Committed: Fri Mar 3 08:05:16 2017 +0100
----------------------------------------------------------------------
sdks/java/io/jdbc/pom.xml | 50 ++++++++++++++++++++
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 27 ++++++-----
sdks/java/io/pom.xml | 37 +++++++++++++++
3 files changed, 102 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index afe428a..fd5c52b 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -30,6 +30,56 @@
<name>Apache Beam :: SDKs :: Java :: IO :: JDBC</name>
<description>IO to read and write on JDBC datasource.</description>
+ <!--
+ The dataflow-runner and spark-runner profiles support using those runners during an integration test. These are
+ not the long-term way we want to support using runners in ITs (e.g. it is annoying to add to all IO modules.)
+ We cannot create a dependency IO -> Runners since the runners depend on IO (e.g. kafka depends on spark.)
+ -->
+
+ <profiles>
+ <!-- Include the Apache Spark runner -P spark-runner -->
+ <profile>
+ <id>spark-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-spark</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <!-- Include the Google Cloud Dataflow runner -P dataflow-runner -->
+ <profile>
+ <id>dataflow-runner</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
index 15206c7..a09db48 100644
--- a/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
+++ b/sdks/java/io/jdbc/src/test/java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java
@@ -38,6 +38,7 @@ import org.apache.beam.sdk.values.PCollection;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -50,16 +51,19 @@ import org.postgresql.ds.PGSimpleDataSource;
* <p>This test requires a running instance of Postgres, and the test dataset must exist in the
* database. `JdbcTestDataSet` will create the read table.
*
- * <p>You can run just this test by doing the following:
+ * <p>You can run this test by doing the following:
* <pre>
- * mvn test-compile compile failsafe:integration-test -D beamTestPipelineOptions='[
- * "--postgresServerName=1.2.3.4",
- * "--postgresUsername=postgres",
- * "--postgresDatabaseName=myfancydb",
- * "--postgresPassword=yourpassword",
- * "--postgresSsl=false"
- * ]' -DskipITs=false -Dit.test=org.apache.beam.sdk.io.jdbc.JdbcIOIT -DfailIfNoTests=false
+ * mvn -e -Pio-it verify -pl sdks/java/io/jdbc -DintegrationTestPipelineOptions='[
+ * "--postgresServerName=1.2.3.4",
+ * "--postgresUsername=postgres",
+ * "--postgresDatabaseName=myfancydb",
+ * "--postgresPassword=mypass",
+ * "--postgresSsl=false" ]'
* </pre>
+ *
+ * <p>If you want to run this with a runner besides directrunner, there are profiles for dataflow
+ * and spark in the jdbc pom. You'll want to activate those in addition to the normal test runner
+ * invocation pipeline options.
*/
@RunWith(JUnit4.class)
public class JdbcIOIT {
@@ -103,6 +107,9 @@ public class JdbcIOIT {
}
}
+ @Rule
+ public TestPipeline pipeline = TestPipeline.create();
+
/**
* Does a test read of a few rows from a postgres database.
*
@@ -113,8 +120,6 @@ public class JdbcIOIT {
public void testRead() throws SQLException {
String tableName = JdbcTestDataSet.READ_TABLE_NAME;
- TestPipeline pipeline = TestPipeline.create();
-
PCollection<KV<String, Integer>> output = pipeline.apply(JdbcIO.<KV<String, Integer>>read()
.withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource))
.withQuery("select name,id from " + tableName)
@@ -148,8 +153,6 @@ public class JdbcIOIT {
public void testWrite() throws SQLException {
writeTableName = JdbcTestDataSet.createWriteDataTable(dataSource);
- TestPipeline pipeline = TestPipeline.create();
-
ArrayList<KV<Integer, String>> data = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
KV<Integer, String> kv = KV.of(i, "Test");
http://git-wip-us.apache.org/repos/asf/beam/blob/eb87ffc7/sdks/java/io/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/pom.xml b/sdks/java/io/pom.xml
index 082f26b..8d5b69b 100644
--- a/sdks/java/io/pom.xml
+++ b/sdks/java/io/pom.xml
@@ -77,4 +77,41 @@
<module>mqtt</module>
</modules>
+ <profiles>
+ <profile>
+ <id>io-it</id>
+ <activation>
+ <property><name>io-it</name></property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <useManifestOnlyJar>false</useManifestOnlyJar>
+ <skip>${skipDefaultIT}</skip>
+ <parallel>all</parallel>
+ <threadCount>4</threadCount>
+ <systemPropertyVariables>
+ <beamTestPipelineOptions>${integrationTestPipelineOptions}</beamTestPipelineOptions>
+ </systemPropertyVariables>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <properties>
+ <skipITs>false</skipITs>
+ </properties>
+ </profile>
+ </profiles>
+
</project>
[2/2] beam git commit: [BEAM-1310] This closes #2090
Posted by jb...@apache.org.
[BEAM-1310] This closes #2090
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/17838199
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/17838199
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/17838199
Branch: refs/heads/master
Commit: 1783819929bda5aaf34a7c176e11c2441d0a7346
Parents: 0a6211b eb87ffc
Author: Jean-Baptiste Onofr� <jb...@apache.org>
Authored: Fri Mar 3 08:33:52 2017 +0100
Committer: Jean-Baptiste Onofr� <jb...@apache.org>
Committed: Fri Mar 3 08:33:52 2017 +0100
----------------------------------------------------------------------
sdks/java/io/jdbc/pom.xml | 50 ++++++++++++++++++++
.../org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 27 ++++++-----
sdks/java/io/pom.xml | 37 +++++++++++++++
3 files changed, 102 insertions(+), 12 deletions(-)
----------------------------------------------------------------------