You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2016/08/19 23:47:20 UTC
[1/2] incubator-beam git commit: Modify example dependencies to only
add runners as optional dependencies. Also support excluding runners by
disabling the include-runners profile.
Repository: incubator-beam
Updated Branches:
refs/heads/master fe00b84dc -> c57643f52
Modify example dependencies to only add runners as optional dependencies.
Also support excluding runners by disabling the include-runners profile.
Tested ITs
BQ and WordCount pass on Dataflow
WordCount passes on Flink and Spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cb30ba31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cb30ba31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cb30ba31
Branch: refs/heads/master
Commit: cb30ba310929934fdf4cfdb324fc70f799108742
Parents: fe00b84
Author: Luke Cwik <lc...@google.com>
Authored: Thu Aug 18 16:35:33 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Aug 19 16:29:43 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 129 ++++++++++++-------
.../org/apache/beam/examples/WordCount.java | 3 +-
examples/java8/pom.xml | 40 +++++-
3 files changed, 125 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 6efbc54..21d7a3a 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -38,6 +38,78 @@
<spark.version>1.6.2</spark.version>
</properties>
+ <profiles>
+ <!--
+ A default profile that includes optional dependencies
+ on all of our runners. This is aimed at making it very
+ easy for users to run examples with any runner without
+ any configuration. It is not needed or desirable when
+ testing the examples with a particular runner.
+
+ This profile can be disabled on the command line
+ by specifying -P !include-runners.
+
+ This profile cannot be lifted to examples-parent because
+ it would be automatically deactivated when the Java 8
+ profile were activated.
+ -->
+ <profile>
+ <id>include-runners</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-flink_2.10</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-spark</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
@@ -270,52 +342,10 @@
</dependency>
<dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-direct-java</artifactId>
- <version>${project.version}</version>
- <scope>runtime</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
- <version>${project.version}</version>
- <scope>runtime</scope>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-flink_2.10</artifactId>
- <version>${project.version}</version>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.beam</groupId>
- <artifactId>beam-runners-spark</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
- <version>${spark.version}</version>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_2.10</artifactId>
- <version>${spark.version}</version>
- <scope>runtime</scope>
- </dependency>
-
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<scope>runtime</scope>
+ <optional>true</optional>
</dependency>
<!-- Hamcrest and JUnit are required dependencies of PAssert,
@@ -331,6 +361,19 @@
<artifactId>junit</artifactId>
</dependency>
+ <!-- Test dependencies -->
+
+ <!--
+ For testing the example itself, use the direct runner. This is separate from
+ the use of RunnableOnService tests for testing a particular runner.
+ -->
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-direct-java</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index 72ad35f..42d30bb 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.examples;
-import org.apache.beam.runners.spark.SparkRunner;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.Default;
@@ -211,7 +210,7 @@ public class WordCount {
public static class InputFactory implements DefaultValueFactory<String> {
@Override
public String create(PipelineOptions options) {
- if (options.getRunner().isAssignableFrom(SparkRunner.class)) {
+ if (options.getRunner().getName().contains("SparkRunner")) {
return Resources.getResource("LICENSE").getPath();
} else {
return "gs://apache-beam-samples/apache/LICENSE";
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cb30ba31/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 609dcfa..f3f2c3c 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -35,8 +35,11 @@
<packaging>jar</packaging>
- <profiles>
+ <properties>
+ <spark.version>1.6.2</spark.version>
+ </properties>
+ <profiles>
<!--
A default profile that includes optional dependencies
on all of our runners. This is aimed at making it very
@@ -53,7 +56,9 @@
-->
<profile>
<id>include-runners</id>
- <activation><activeByDefault>true</activeByDefault></activation>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<dependencies>
<dependency>
<groupId>org.apache.beam</groupId>
@@ -73,11 +78,35 @@
<dependency>
<groupId>org.apache.beam</groupId>
+ <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
<artifactId>beam-runners-spark</artifactId>
<version>${project.version}</version>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming_2.10</artifactId>
+ <version>${spark.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
</dependencies>
</profile>
</profiles>
@@ -165,6 +194,13 @@
</dependency>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
[2/2] incubator-beam git commit: [BEAM-124] Modify example
dependencies to only add runners as optional dependencies
Posted by lc...@apache.org.
[BEAM-124] Modify example dependencies to only add runners as optional dependencies
This closes #851
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c57643f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c57643f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c57643f5
Branch: refs/heads/master
Commit: c57643f5209f4a1982a45dcd4b3ca554dffd7f5f
Parents: fe00b84 cb30ba3
Author: Luke Cwik <lc...@google.com>
Authored: Fri Aug 19 16:31:27 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Aug 19 16:31:27 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 129 ++++++++++++-------
.../org/apache/beam/examples/WordCount.java | 3 +-
examples/java8/pom.xml | 40 +++++-
3 files changed, 125 insertions(+), 47 deletions(-)
----------------------------------------------------------------------