You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ie...@apache.org on 2017/04/18 14:14:00 UTC
[1/4] beam git commit: Remove flink-annotations dependency
Repository: beam
Updated Branches:
refs/heads/master e5568589c -> fac4f3e3c
Remove flink-annotations dependency
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/88513db3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/88513db3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/88513db3
Branch: refs/heads/master
Commit: 88513db31ee5301d96550b82b5f428b35966f2a7
Parents: 5fce8d2
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Tue Apr 18 14:42:37 2017 +0200
Committer: Isma�l Mej�a <ie...@apache.org>
Committed: Tue Apr 18 16:12:47 2017 +0200
----------------------------------------------------------------------
runners/flink/runner/pom.xml | 6 ------
.../translation/types/EncodedValueTypeInformation.java | 9 ---------
2 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/88513db3/runners/flink/runner/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/runner/pom.xml b/runners/flink/runner/pom.xml
index 30f376c..95880f4 100644
--- a/runners/flink/runner/pom.xml
+++ b/runners/flink/runner/pom.xml
@@ -160,12 +160,6 @@
<version>${flink.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-annotations</artifactId>
- <version>${flink.version}</version>
- </dependency>
-
<!-- For testing -->
<dependency>
<groupId>org.apache.flink</groupId>
http://git-wip-us.apache.org/repos/asf/beam/blob/88513db3/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
index 0315ae3..e24bf31 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/types/EncodedValueTypeInformation.java
@@ -18,7 +18,6 @@
package org.apache.beam.runners.flink.translation.types;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.AtomicType;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -36,43 +35,36 @@ public class EncodedValueTypeInformation
private static final long serialVersionUID = 1L;
@Override
- @PublicEvolving
public boolean isBasicType() {
return false;
}
@Override
- @PublicEvolving
public boolean isTupleType() {
return false;
}
@Override
- @PublicEvolving
public int getArity() {
return 0;
}
@Override
- @PublicEvolving
public int getTotalFields() {
return 0;
}
@Override
- @PublicEvolving
public Class<byte[]> getTypeClass() {
return byte[].class;
}
@Override
- @PublicEvolving
public boolean isKeyType() {
return true;
}
@Override
- @PublicEvolving
public TypeSerializer<byte[]> createSerializer(ExecutionConfig executionConfig) {
return new EncodedValueSerializer();
}
@@ -98,7 +90,6 @@ public class EncodedValueTypeInformation
}
@Override
- @PublicEvolving
public TypeComparator<byte[]> createComparator(
boolean sortOrderAscending,
ExecutionConfig executionConfig) {
[4/4] beam git commit: This closes #2549
Posted by ie...@apache.org.
This closes #2549
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/fac4f3e3
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/fac4f3e3
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/fac4f3e3
Branch: refs/heads/master
Commit: fac4f3e3c937a76d6ca92ca6f73f8df9928ae94c
Parents: e556858 88513db
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Tue Apr 18 16:12:58 2017 +0200
Committer: Isma�l Mej�a <ie...@apache.org>
Committed: Tue Apr 18 16:12:58 2017 +0200
----------------------------------------------------------------------
runners/flink/pom.xml | 45 +++++---
runners/flink/runner/pom.xml | 109 +++++++++++--------
.../flink/FlinkDetachedRunnerResult.java | 3 +-
.../types/EncodedValueTypeInformation.java | 9 --
.../streaming/SingletonKeyedWorkItem.java | 2 -
.../streaming/SingletonKeyedWorkItemCoder.java | 2 -
.../wrappers/streaming/WindowDoFnOperator.java | 3 -
.../beam/runners/flink/PipelineOptionsTest.java | 2 +-
8 files changed, 95 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
[3/4] beam git commit: Enable flink dependency enforcement and make
dependencies explicit
Posted by ie...@apache.org.
Enable flink dependency enforcement and make dependencies explicit
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f654ff47
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f654ff47
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f654ff47
Branch: refs/heads/master
Commit: f654ff471c52f6bba7fb34ac7b8b4f5768af17c0
Parents: e556858
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Sun Apr 16 18:02:12 2017 +0200
Committer: Isma�l Mej�a <ie...@apache.org>
Committed: Tue Apr 18 16:12:47 2017 +0200
----------------------------------------------------------------------
runners/flink/pom.xml | 45 +++++---
runners/flink/runner/pom.xml | 115 +++++++++++--------
.../flink/FlinkDetachedRunnerResult.java | 3 +-
.../beam/runners/flink/PipelineOptionsTest.java | 2 +-
4 files changed, 101 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/f654ff47/runners/flink/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index dcf5ff7..a5c5ea0 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -42,20 +42,6 @@
<flink.version>1.2.0</flink.version>
</properties>
- <repositories>
- <repository>
- <id>apache.snapshots</id>
- <name>Apache Development Snapshot Repository</name>
- <url>https://repository.apache.org/content/repositories/snapshots/</url>
- <releases>
- <enabled>false</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
-
<build>
<pluginManagement>
<plugins>
@@ -95,10 +81,37 @@
</execution>
</executions>
</plugin>
-
</plugins>
</pluginManagement>
-
</build>
+ <dependencies>
+ <!-- Flink dependencies -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_2.10</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-java-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/f654ff47/runners/flink/runner/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/runner/pom.xml b/runners/flink/runner/pom.xml
index 70cd246..30f376c 100644
--- a/runners/flink/runner/pom.xml
+++ b/runners/flink/runner/pom.xml
@@ -115,21 +115,54 @@
</profile>
</profiles>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+
+ <!-- Integration Tests -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+
+ <!-- Unit Tests -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
<dependencies>
<!-- Flink dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_2.10</artifactId>
+ <artifactId>flink-java</artifactId>
<version>${flink.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
+ <artifactId>flink-clients_2.10</artifactId>
<version>${flink.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_2.10</artifactId>
+ <artifactId>flink-runtime_2.10</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-annotations</artifactId>
<version>${flink.version}</version>
</dependency>
@@ -150,7 +183,6 @@
<scope>test</scope>
</dependency>
-
<!-- Beam -->
<dependency>
<groupId>org.apache.beam</groupId>
@@ -185,6 +217,26 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
<!--
Force an upgrade on the version of Apache Commons from Flink to support DEFLATE compression.
-->
@@ -192,20 +244,34 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>[1.9,)</version>
+ <scope>runtime</scope>
</dependency>
<!-- Test scoped -->
+ <dependency>
+ <groupId>com.google.apis</groupId>
+ <artifactId>google-api-services-bigquery</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
@@ -267,45 +333,4 @@
<scope>test</scope>
</dependency>
</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <goals><goal>analyze-only</goal></goals>
- <configuration>
- <!-- disable for now until dependencies are cleaned up -->
- <failOnWarning>false</failOnWarning>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- Integration Tests -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- </plugin>
-
- <!-- Unit Tests -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- </plugin>
-
- </plugins>
- </build>
-
</project>
http://git-wip-us.apache.org/repos/asf/beam/blob/f654ff47/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
index 6adcf07..bf4395f 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkDetachedRunnerResult.java
@@ -24,7 +24,6 @@ import org.apache.beam.sdk.AggregatorValues;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.metrics.MetricResults;
import org.apache.beam.sdk.transforms.Aggregator;
-import org.apache.commons.lang.NotImplementedException;
import org.joda.time.Duration;
@@ -46,7 +45,7 @@ public class FlinkDetachedRunnerResult implements PipelineResult {
throws AggregatorRetrievalException {
throw new AggregatorRetrievalException(
"Accumulators can't be retrieved for detached Job executions.",
- new NotImplementedException());
+ new UnsupportedOperationException());
}
@Override
http://git-wip-us.apache.org/repos/asf/beam/blob/f654ff47/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
index 2cb3dd3..06187f6 100644
--- a/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
+++ b/runners/flink/runner/src/test/java/org/apache/beam/runners/flink/PipelineOptionsTest.java
@@ -38,7 +38,7 @@ import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingStrategy;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
-import org.apache.commons.lang.SerializationUtils;
+import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
[2/4] beam git commit: Fix Javadoc warnings on Flink Runner
Posted by ie...@apache.org.
Fix Javadoc warnings on Flink Runner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5fce8d2a
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5fce8d2a
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5fce8d2a
Branch: refs/heads/master
Commit: 5fce8d2a27e4507012e8e61a1487f51d3293d9d9
Parents: f654ff4
Author: Isma�l Mej�a <ie...@apache.org>
Authored: Tue Apr 18 14:36:44 2017 +0200
Committer: Isma�l Mej�a <ie...@apache.org>
Committed: Tue Apr 18 16:12:47 2017 +0200
----------------------------------------------------------------------
.../translation/wrappers/streaming/SingletonKeyedWorkItem.java | 2 --
.../wrappers/streaming/SingletonKeyedWorkItemCoder.java | 2 --
.../flink/translation/wrappers/streaming/WindowDoFnOperator.java | 3 ---
3 files changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/5fce8d2a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItem.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItem.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItem.java
index b85efef..e843660 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItem.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItem.java
@@ -24,8 +24,6 @@ import org.apache.beam.sdk.util.WindowedValue;
/**
* Singleton keyed word item.
- * @param <K>
- * @param <ElemT>
*/
public class SingletonKeyedWorkItem<K, ElemT> implements KeyedWorkItem<K, ElemT> {
http://git-wip-us.apache.org/repos/asf/beam/blob/5fce8d2a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
index fe96eb1..9a52330 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/SingletonKeyedWorkItemCoder.java
@@ -37,8 +37,6 @@ import org.apache.beam.sdk.util.WindowedValue;
/**
* Singleton keyed work item coder.
- * @param <K>
- * @param <ElemT>
*/
public class SingletonKeyedWorkItemCoder<K, ElemT>
extends StandardCoder<SingletonKeyedWorkItem<K, ElemT>> {
http://git-wip-us.apache.org/repos/asf/beam/blob/5fce8d2a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
----------------------------------------------------------------------
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
index 8bbc6ef..7b899f4 100644
--- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
+++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/WindowDoFnOperator.java
@@ -44,9 +44,6 @@ import org.apache.flink.streaming.api.operators.InternalTimer;
/**
* Flink operator for executing window {@link DoFn DoFns}.
- *
- * @param <InputT>
- * @param <OutputT>
*/
public class WindowDoFnOperator<K, InputT, OutputT>
extends DoFnOperator<KeyedWorkItem<K, InputT>, KV<K, OutputT>, WindowedValue<KV<K, OutputT>>> {