You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2018/01/29 23:07:40 UTC
[beam] branch master updated (bc3bc68 -> ce740eb)
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from bc3bc68 [BEAM-3550] Add --awsServiceEndpoint option and use with S3 filesystem.
new a0ef2a5 Fixes for sdks/java/core for the eclipse compiler
new 27454ac Add some m2e lifecycle things to get the various maven plugins running as needed in m2e/eclipse
new 41cc515 Set the proper package for the Snippets example so eclipse won't raise an error
new 2970b53 Add some template args and direct casts to help the eclipse compiler determine the right casts
new ce740eb Incorporate reviews and rebase on latest master
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
build.gradle | 2 +-
build_rules.gradle | 9 +++++
.../{website_snippets => snippets}/Snippets.java | 2 +-
.../SnippetsTest.java | 2 +-
model/job-management/pom.xml | 18 +++++++++
pom.xml | 45 +++++++++++++++++++++-
.../apache/beam/runners/core/ReduceFnTester.java | 2 +-
.../core/triggers/TriggerStateMachineTester.java | 2 +-
.../runners/direct/WindowEvaluatorFactory.java | 2 +-
.../runners/direct/DirectGraphVisitorTest.java | 16 ++++----
.../SparkGroupAlsoByWindowViaWindowSet.java | 6 ++-
.../main/java/org/apache/beam/sdk/Pipeline.java | 2 +-
.../org/apache/beam/sdk/coders/CoderRegistry.java | 3 +-
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 7 ++--
sdks/java/extensions/sql/pom.xml | 7 ++--
.../operator/BeamSqlCaseExpression.java | 3 +-
sdks/java/maven-archetypes/examples/pom.xml | 27 +++++++++++++
18 files changed, 131 insertions(+), 25 deletions(-)
rename examples/java/src/main/java/org/apache/beam/examples/{website_snippets => snippets}/Snippets.java (98%)
rename examples/java/src/test/java/org/apache/beam/examples/{website_snippets => snippets}/SnippetsTest.java (99%)
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.
[beam] 05/05: Incorporate reviews and rebase on latest master
Posted by lc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit ce740ebf229d54986fdab39493c309fca6ffb87f
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Mon Jan 29 09:07:31 2018 -0500
Incorporate reviews and rebase on latest master
---
.gitignore | 1 +
build_rules.gradle | 12 +++---
pom.xml | 23 +++++------
.../SparkGroupAlsoByWindowViaWindowSet.java | 9 ++--
.../org/apache/beam/sdk/coders/CoderRegistry.java | 4 +-
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 8 ++--
.../operator/BeamSqlCaseExpression.java | 4 +-
sdks/java/maven-archetypes/examples/pom.xml | 48 +++++++++++-----------
8 files changed, 56 insertions(+), 53 deletions(-)
diff --git a/.gitignore b/.gitignore
index 00b57a3..ff2faad 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,6 +52,7 @@ sdks/python/apache_beam/portability/api/*pb2*.*
**/.fbExcludeFilterFile
**/.apt_generated/**/*
**/.settings/**/*
+**/.gitignore
# Ignore Visual Studio Code files.
**/.vscode/**/*
diff --git a/build_rules.gradle b/build_rules.gradle
index fde5668..c4c6746 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -222,7 +222,6 @@ ext.applyJavaNature = {
testApt auto_service
}
-
// Add the optional and provided configurations for dependencies
// TODO: Either remove these plugins and find another way to generate the Maven poms
// with the correct dependency scopes configured.
@@ -239,6 +238,13 @@ ext.applyJavaNature = {
maxErrors = 0
}
+ // Apply the eclipse and apt-eclipse plugins. This adds the "eclipse" task and
+ // connects the apt-eclipse plugin to update the eclipse project files
+ // with the instructions needed to run apt within eclipse to handle the AutoValue
+ // and additional annotations
+ apply plugin: 'eclipse'
+ apply plugin: "net.ltgt.apt-eclipse"
+
// Enables a plugin which can apply code formatting to source.
// TODO: Should this plugin be enabled for all projects?
apply plugin: "com.diffplug.gradle.spotless"
@@ -435,8 +441,4 @@ ext.applyAvroNature = {
apply plugin: "com.commercehub.gradle.plugin.avro"
}
-// Apply the apt-eclipse plugin so the eclipse import can work
-apply plugin: 'eclipse'
-apply plugin: "net.ltgt.apt-eclipse"
-
diff --git a/pom.xml b/pom.xml
index 9d083ab..55e852e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,7 +190,6 @@
<!-- Default skipping -->
<rat.skip>true</rat.skip>
-
</properties>
<packaging>pom</packaging>
@@ -1772,17 +1771,17 @@
</action>
</pluginExecution>
<pluginExecution>
- <pluginExecutionFilter>
- <groupId>net.revelc.code.formatter</groupId>
- <artifactId>formatter-maven-plugin</artifactId>
- <versionRange>[2.0.0,)</versionRange>
- <goals>
- <goal>format</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore />
- </action>
+ <pluginExecutionFilter>
+ <groupId>net.revelc.code.formatter</groupId>
+ <artifactId>formatter-maven-plugin</artifactId>
+ <versionRange>[2.0.0,)</versionRange>
+ <goals>
+ <goal>format</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
</pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 921d504..d03f176 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -74,6 +74,7 @@ import org.slf4j.LoggerFactory;
import scala.Option;
import scala.Tuple2;
import scala.Tuple3;
+import scala.collection.GenTraversable;
import scala.collection.Iterator;
import scala.collection.Seq;
import scala.runtime.AbstractFunction1;
@@ -272,12 +273,12 @@ public class SparkGroupAlsoByWindowViaWindowSet implements Serializable {
if (!encodedKeyedElements.isEmpty()) {
// new input for key.
try {
- //cast to GenTraversable to avoid a ambiguous call to head() which can come from
- //mulitple super interfacesof Seq<byte[]>
- byte[] b = ((scala.collection.GenTraversable<byte[]>) encodedKeyedElements).head();
+ // cast to GenTraversable to avoid a ambiguous call to head() which can come from
+ // multiple super interfacesof Seq<byte[]>
+ byte[] headBytes = ((GenTraversable<byte[]>) encodedKeyedElements).head();
final KV<Long, Iterable<WindowedValue<InputT>>> keyedElements =
CoderHelpers.fromByteArray(
- b, KvCoder.of(VarLongCoder.of(), itrWvCoder));
+ headBytes, KvCoder.of(VarLongCoder.of(), itrWvCoder));
final Long rddTimestamp = keyedElements.getKey();
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
index 7c84b35..946c9bc 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
@@ -564,9 +564,9 @@ public class CoderRegistry {
}
for (int i = 0; i < typeArgumentCoders.size(); i++) {
try {
- Coder<?> c2 = typeArgumentCoders.get(i);
+ Coder<?> typeArgumentCoder = typeArgumentCoders.get(i);
verifyCompatible(
- c2,
+ typeArgumentCoder,
candidateDescriptor.resolveType(typeArguments[i]).getType());
} catch (IncompatibleCoderException exn) {
throw new IncompatibleCoderException(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
index 532aa5a..9cb2a28 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
@@ -88,8 +88,8 @@ public class WindowFnTestUtils {
for (W window : assignedWindowsWithValue(windowFn, element)) {
windowSet.put(window, timestampValue(element.getTimestamp().getMillis()));
}
- TestMergeContext<T, W> tmc = new TestMergeContext<>(windowSet, windowFn);
- windowFn.mergeWindows(tmc);
+ TestMergeContext<T, W> mergeContext = new TestMergeContext<>(windowSet, windowFn);
+ windowFn.mergeWindows(mergeContext);
}
Map<W, Set<String>> actual = new HashMap<>();
for (W window : windowSet.windows()) {
@@ -113,8 +113,8 @@ public class WindowFnTestUtils {
*/
public static <T, W extends BoundedWindow> Collection<W> assignedWindowsWithValue(
WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
- TestAssignContext<T, W> tac = new TestAssignContext<>(timestampedValue, windowFn);
- return windowFn.assignWindows(tac);
+ TestAssignContext<T, W> assignContext = new TestAssignContext<>(timestampedValue, windowFn);
+ return windowFn.assignWindows(assignContext);
}
private static String timestampValue(long timestamp) {
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
index ec9ba40..c7eb156 100644
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
@@ -51,8 +51,8 @@ public class BeamSqlCaseExpression extends BeamSqlExpression {
@Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
for (int i = 0; i < operands.size() - 1; i += 2) {
- Boolean b = opValueEvaluated(i, inputRow, window);
- if (b != null && b) {
+ Boolean wasOpEvaluated = opValueEvaluated(i, inputRow, window);
+ if (wasOpEvaluated != null && wasOpEvaluated) {
return BeamSqlPrimitive.of(
outputType,
opValueEvaluated(i + 1, inputRow, window)
diff --git a/sdks/java/maven-archetypes/examples/pom.xml b/sdks/java/maven-archetypes/examples/pom.xml
index 69afe5b..8e25e73 100644
--- a/sdks/java/maven-archetypes/examples/pom.xml
+++ b/sdks/java/maven-archetypes/examples/pom.xml
@@ -73,30 +73,30 @@
</plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
<plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <versionRange>[1.5.0,)</versionRange>
- <goals>
- <goal>exec</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <execute>
- <runOnIncremental>false</runOnIncremental>
- </execute>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <versionRange>[1.5.0,)</versionRange>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ </execute>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
</plugin>
</plugins>
</pluginManagement>
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.
[beam] 02/05: Add some m2e lifecycle things to get the various
maven plugins running as needed in m2e/eclipse
Posted by lc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 27454acae3783322f36497e891806d8610ead97e
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Fri Jan 26 14:23:20 2018 -0500
Add some m2e lifecycle things to get the various maven plugins running as needed in m2e/eclipse
---
model/job-management/pom.xml | 18 +++++++++++
pom.xml | 46 ++++++++++++++++++++++++++++-
sdks/java/extensions/sql/pom.xml | 7 ++---
sdks/java/maven-archetypes/examples/pom.xml | 27 +++++++++++++++++
4 files changed, 93 insertions(+), 5 deletions(-)
diff --git a/model/job-management/pom.xml b/model/job-management/pom.xml
index 3e5803e..7210e9f 100644
--- a/model/job-management/pom.xml
+++ b/model/job-management/pom.xml
@@ -77,6 +77,24 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-grpc-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/protobuf/java/grpc-java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/pom.xml b/pom.xml
index 60f88b4..9d083ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,6 +190,7 @@
<!-- Default skipping -->
<rat.skip>true</rat.skip>
+
</properties>
<packaging>pom</packaging>
@@ -343,7 +344,7 @@
<dependency>
<groupId>org.eclipse.tycho</groupId>
<artifactId>tycho-compiler-jdt</artifactId>
- <version>0.26.0</version>
+ <version>1.0.0</version>
</dependency>
</dependencies>
</plugin>
@@ -1714,6 +1715,36 @@
</pluginExecution>
<pluginExecution>
<pluginExecutionFilter>
+ <groupId>com.googlecode.fmpp-maven-plugin</groupId>
+ <artifactId>fmpp-maven-plugin</artifactId>
+ <versionRange>[1.0,)</versionRange>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ </execute>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <versionRange>[2.4,)</versionRange>
+ <goals>
+ <goal>javacc</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ </execute>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<versionRange>[2.5,)</versionRange>
@@ -1740,6 +1771,19 @@
<ignore />
</action>
</pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>net.revelc.code.formatter</groupId>
+ <artifactId>formatter-maven-plugin</artifactId>
+ <versionRange>[2.0.0,)</versionRange>
+ <goals>
+ <goal>format</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
diff --git a/sdks/java/extensions/sql/pom.xml b/sdks/java/extensions/sql/pom.xml
index 3c626a4..81d9db9 100644
--- a/sdks/java/extensions/sql/pom.xml
+++ b/sdks/java/extensions/sql/pom.xml
@@ -166,7 +166,7 @@
</goals>
<configuration>
<cfgFile>${project.build.directory}/codegen/config.fmpp</cfgFile>
- <outputDirectory>target/generated-sources</outputDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/fmpp</outputDirectory>
<templateDirectory>${project.build.directory}/codegen/templates</templateDirectory>
</configuration>
</execution>
@@ -176,7 +176,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
- <version>1.9</version>
<executions>
<execution>
<id>add-generated-sources</id>
@@ -186,7 +185,7 @@
</goals>
<configuration>
<sources>
- <source>${project.build.directory}/generated-sources</source>
+ <source>${project.build.directory}/generated-sources/java</source>
</sources>
</configuration>
</execution>
@@ -211,7 +210,7 @@
</includes>
<lookAhead>2</lookAhead>
<isStatic>false</isStatic>
- <outputDirectory>${project.build.directory}/generated-sources/</outputDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/java</outputDirectory>
</configuration>
</execution>
</executions>
diff --git a/sdks/java/maven-archetypes/examples/pom.xml b/sdks/java/maven-archetypes/examples/pom.xml
index a994165..69afe5b 100644
--- a/sdks/java/maven-archetypes/examples/pom.xml
+++ b/sdks/java/maven-archetypes/examples/pom.xml
@@ -71,6 +71,33 @@
</execution>
</executions>
</plugin>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <versionRange>[1.5.0,)</versionRange>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute>
+ <runOnIncremental>false</runOnIncremental>
+ </execute>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.
[beam] 04/05: Add some template args and direct casts to help the
eclipse compiler determine the right casts
Posted by lc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 2970b53538cff9d97eb34cc3b4407ec0777b8a6d
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Fri Jan 26 14:26:06 2018 -0500
Add some template args and direct casts to help the eclipse compiler determine the right casts
---
.../org/apache/beam/runners/core/ReduceFnTester.java | 2 +-
.../runners/core/triggers/TriggerStateMachineTester.java | 2 +-
.../beam/runners/direct/WindowEvaluatorFactory.java | 2 +-
.../beam/runners/direct/DirectGraphVisitorTest.java | 16 +++++++++-------
.../stateful/SparkGroupAlsoByWindowViaWindowSet.java | 5 ++++-
.../impl/interpreter/operator/BeamSqlCaseExpression.java | 3 ++-
6 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
index ef2f39c..4dd0b42 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/ReduceFnTester.java
@@ -547,7 +547,7 @@ public class ReduceFnTester<InputT, OutputT, W extends BoundedWindow> {
Instant timestamp = input.getTimestamp();
Collection<W> windows =
windowFn.assignWindows(
- new TestAssignContext<>(
+ new TestAssignContext<W>(
windowFn, value, timestamp, GlobalWindow.INSTANCE));
return WindowedValue.of(value, timestamp, windows, PaneInfo.NO_FIRING);
} catch (Exception e) {
diff --git a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
index 99941ec..c8808fc 100644
--- a/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
+++ b/runners/core-java/src/test/java/org/apache/beam/runners/core/triggers/TriggerStateMachineTester.java
@@ -266,7 +266,7 @@ public class TriggerStateMachineTester<InputT, W extends BoundedWindow> {
Instant timestamp = input.getTimestamp();
Collection<W> assignedWindows =
windowFn.assignWindows(
- new TestAssignContext<>(windowFn, value, timestamp, GlobalWindow.INSTANCE));
+ new TestAssignContext<W>(windowFn, value, timestamp, GlobalWindow.INSTANCE));
for (W window : assignedWindows) {
activeWindows.addActiveForTesting(window);
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
index 03c1d50..339640b 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WindowEvaluatorFactory.java
@@ -99,7 +99,7 @@ class WindowEvaluatorFactory implements TransformEvaluatorFactory {
private <W extends BoundedWindow> Collection<? extends BoundedWindow> assignWindows(
WindowFn<InputT, W> windowFn, WindowedValue<InputT> element) throws Exception {
WindowFn<InputT, W>.AssignContext assignContext =
- new DirectAssignContext<>(windowFn, element);
+ new DirectAssignContext<InputT, W>(windowFn, element);
Collection<? extends BoundedWindow> windows = windowFn.assignWindows(assignContext);
return windows;
}
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
index 1ece894..5467c9a 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectGraphVisitorTest.java
@@ -95,8 +95,8 @@ public class DirectGraphVisitorTest implements Serializable {
assertThat(graph.getRootTransforms(), hasSize(3));
assertThat(
graph.getRootTransforms(),
- Matchers.containsInAnyOrder(
- graph.getProducer(created), graph.getProducer(counted), graph.getProducer(unCounted)));
+ Matchers.containsInAnyOrder(new Object[] {
+ graph.getProducer(created), graph.getProducer(counted), graph.getProducer(unCounted)}));
for (AppliedPTransform<?, ?, ?> root : graph.getRootTransforms()) {
// Root transforms will have no inputs
assertThat(root.getInputs().entrySet(), emptyIterable());
@@ -114,9 +114,10 @@ public class DirectGraphVisitorTest implements Serializable {
empty.setCoder(StringUtf8Coder.of());
p.traverseTopologically(visitor);
DirectGraph graph = visitor.getGraph();
- assertThat(graph.getRootTransforms(), Matchers.containsInAnyOrder(graph.getProducer(empty)));
+ assertThat(graph.getRootTransforms(),
+ Matchers.containsInAnyOrder(new Object[] {graph.getProducer(empty)}));
AppliedPTransform<?, ?, ?> onlyRoot = Iterables.getOnlyElement(graph.getRootTransforms());
- assertThat(onlyRoot.getTransform(), Matchers.equalTo(flatten));
+ assertThat((Object) onlyRoot.getTransform(), Matchers.equalTo(flatten));
assertThat(onlyRoot.getInputs().entrySet(), emptyIterable());
assertThat(onlyRoot.getOutputs(), equalTo(empty.expand()));
}
@@ -148,9 +149,10 @@ public class DirectGraphVisitorTest implements Serializable {
assertThat(
graph.getPerElementConsumers(created),
- Matchers.containsInAnyOrder(transformedProducer, flattenedProducer));
+ Matchers.containsInAnyOrder(new Object[] {transformedProducer, flattenedProducer}));
assertThat(
- graph.getPerElementConsumers(transformed), Matchers.containsInAnyOrder(flattenedProducer));
+ graph.getPerElementConsumers(transformed),
+ Matchers.containsInAnyOrder(new Object[] {flattenedProducer}));
assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
@@ -168,7 +170,7 @@ public class DirectGraphVisitorTest implements Serializable {
assertThat(
graph.getPerElementConsumers(created),
- Matchers.containsInAnyOrder(flattenedProducer, flattenedProducer));
+ Matchers.containsInAnyOrder(new Object[] {flattenedProducer, flattenedProducer}));
assertThat(graph.getPerElementConsumers(flattened), emptyIterable());
}
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
index 50957f2..921d504 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/stateful/SparkGroupAlsoByWindowViaWindowSet.java
@@ -272,9 +272,12 @@ public class SparkGroupAlsoByWindowViaWindowSet implements Serializable {
if (!encodedKeyedElements.isEmpty()) {
// new input for key.
try {
+ //cast to GenTraversable to avoid a ambiguous call to head() which can come from
+ //mulitple super interfacesof Seq<byte[]>
+ byte[] b = ((scala.collection.GenTraversable<byte[]>) encodedKeyedElements).head();
final KV<Long, Iterable<WindowedValue<InputT>>> keyedElements =
CoderHelpers.fromByteArray(
- encodedKeyedElements.head(), KvCoder.of(VarLongCoder.of(), itrWvCoder));
+ b, KvCoder.of(VarLongCoder.of(), itrWvCoder));
final Long rddTimestamp = keyedElements.getKey();
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
index 955444f..ec9ba40 100644
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/interpreter/operator/BeamSqlCaseExpression.java
@@ -51,7 +51,8 @@ public class BeamSqlCaseExpression extends BeamSqlExpression {
@Override public BeamSqlPrimitive evaluate(BeamRecord inputRow, BoundedWindow window) {
for (int i = 0; i < operands.size() - 1; i += 2) {
- if (opValueEvaluated(i, inputRow, window)) {
+ Boolean b = opValueEvaluated(i, inputRow, window);
+ if (b != null && b) {
return BeamSqlPrimitive.of(
outputType,
opValueEvaluated(i + 1, inputRow, window)
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.
[beam] 01/05: Fixes for sdks/java/core for the eclipse compiler
Posted by lc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit a0ef2a545e24644d5d8cf0fff1303b9bfc70adb3
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Fri Jan 26 09:04:39 2018 -0500
Fixes for sdks/java/core for the eclipse compiler
---
build.gradle | 2 +-
build_rules.gradle | 7 +++++++
sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java | 2 +-
.../src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java | 3 ++-
.../main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java | 7 ++++---
5 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/build.gradle b/build.gradle
index a86efa9..6dccd5c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -173,7 +173,7 @@ buildscript {
maven { url "http://repo.spring.io/plugins-release" }
}
dependencies {
- classpath "net.ltgt.gradle:gradle-apt-plugin:0.12" // Enable a Java annotation processor
+ classpath "net.ltgt.gradle:gradle-apt-plugin:0.13" // Enable a Java annotation processor
classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.1" // Enable proto code generation
classpath "io.spring.gradle:propdeps-plugin:0.0.9.RELEASE" // Enable provided and optional configurations
classpath "gradle.plugin.org.nosphere.apache:creadur-rat-gradle:0.3.1" // Enable Apache license enforcement
diff --git a/build_rules.gradle b/build_rules.gradle
index 2f9a5d0..fde5668 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -222,6 +222,7 @@ ext.applyJavaNature = {
testApt auto_service
}
+
// Add the optional and provided configurations for dependencies
// TODO: Either remove these plugins and find another way to generate the Maven poms
// with the correct dependency scopes configured.
@@ -433,3 +434,9 @@ ext.applyAvroNature = {
println "applyAvroNature with " + (it ? "$it" : "default configuration") + " for project $project.name"
apply plugin: "com.commercehub.gradle.plugin.avro"
}
+
+// Apply the apt-eclipse plugin so the eclipse import can work
+apply plugin: 'eclipse'
+apply plugin: "net.ltgt.apt-eclipse"
+
+
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index c6028ad..71bac5c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -302,7 +302,7 @@ public class Pipeline {
* by the options.
*/
public PipelineResult run(PipelineOptions options) {
- PipelineRunner runner = PipelineRunner.fromOptions(options);
+ PipelineRunner<? extends PipelineResult> runner = PipelineRunner.fromOptions(options);
// Ensure all of the nodes are fully specified before a PipelineRunner gets access to the
// pipeline.
LOG.debug("Running {} via {}", this, runner);
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
index b867d93..7c84b35 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/coders/CoderRegistry.java
@@ -564,8 +564,9 @@ public class CoderRegistry {
}
for (int i = 0; i < typeArgumentCoders.size(); i++) {
try {
+ Coder<?> c2 = typeArgumentCoders.get(i);
verifyCompatible(
- typeArgumentCoders.get(i),
+ c2,
candidateDescriptor.resolveType(typeArguments[i]).getType());
} catch (IncompatibleCoderException exn) {
throw new IncompatibleCoderException(
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
index c456e7b..532aa5a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/WindowFnTestUtils.java
@@ -88,7 +88,8 @@ public class WindowFnTestUtils {
for (W window : assignedWindowsWithValue(windowFn, element)) {
windowSet.put(window, timestampValue(element.getTimestamp().getMillis()));
}
- windowFn.mergeWindows(new TestMergeContext<>(windowSet, windowFn));
+ TestMergeContext<T, W> tmc = new TestMergeContext<>(windowSet, windowFn);
+ windowFn.mergeWindows(tmc);
}
Map<W, Set<String>> actual = new HashMap<>();
for (W window : windowSet.windows()) {
@@ -112,8 +113,8 @@ public class WindowFnTestUtils {
*/
public static <T, W extends BoundedWindow> Collection<W> assignedWindowsWithValue(
WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception {
- return windowFn.assignWindows(
- new TestAssignContext<>(timestampedValue, windowFn));
+ TestAssignContext<T, W> tac = new TestAssignContext<>(timestampedValue, windowFn);
+ return windowFn.assignWindows(tac);
}
private static String timestampValue(long timestamp) {
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.
[beam] 03/05: Set the proper package for the Snippets example so
eclipse won't raise an error
Posted by lc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 41cc51503f531853bddadda02ef167532ec16323
Author: Daniel Kulp <dk...@apache.org>
AuthorDate: Fri Jan 26 14:24:05 2018 -0500
Set the proper package for the Snippets example so eclipse won't raise an error
---
.../apache/beam/examples/{website_snippets => snippets}/Snippets.java | 2 +-
.../beam/examples/{website_snippets => snippets}/SnippetsTest.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/examples/java/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java b/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java
similarity index 98%
rename from examples/java/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java
rename to examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java
index 0630acd..e2853da 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/website_snippets/Snippets.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/snippets/Snippets.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.examples;
+package org.apache.beam.examples.snippets;
import java.util.ArrayList;
import java.util.Collections;
diff --git a/examples/java/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java b/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java
similarity index 99%
rename from examples/java/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java
rename to examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java
index 0f03063..7ebe162 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/website_snippets/SnippetsTest.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/snippets/SnippetsTest.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.beam.examples;
+package org.apache.beam.examples.snippets;
import java.io.IOException;
import java.io.Serializable;
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.