You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/11/08 03:41:00 UTC
[01/50] incubator-beam git commit: This closes #1252
Repository: incubator-beam
Updated Branches:
refs/heads/gearpump-runner 323ec1188 -> a14927f40
This closes #1252
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/faf55c78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/faf55c78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/faf55c78
Branch: refs/heads/gearpump-runner
Commit: faf55c78a30a6937a8b598e674a48f446fcb5424
Parents: 3419fef 8bf6d92
Author: Thomas Groh <tg...@google.com>
Authored: Fri Nov 4 09:57:44 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Nov 4 09:57:44 2016 -0700
----------------------------------------------------------------------
.../sdk/transforms/reflect/DoFnInvokers.java | 7 +-
.../sdk/transforms/reflect/DoFnSignature.java | 44 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 752 ++++++++++++-------
.../apache/beam/sdk/values/TypeDescriptor.java | 13 +
.../DoFnSignaturesProcessElementTest.java | 18 +-
.../DoFnSignaturesSplittableDoFnTest.java | 19 +-
.../transforms/reflect/DoFnSignaturesTest.java | 42 +-
.../reflect/DoFnSignaturesTestUtils.java | 13 +-
8 files changed, 578 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
[09/50] incubator-beam git commit: [BEAM-898] Fix Jenkins
BigQueryTornadoes IT Failure
Posted by ke...@apache.org.
[BEAM-898] Fix Jenkins BigQueryTornadoes IT Failure
This closes #1279
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/46fbfe06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/46fbfe06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/46fbfe06
Branch: refs/heads/gearpump-runner
Commit: 46fbfe06bc34f9c355d00f346f63767a861cb858
Parents: 99062d1 1eccd29
Author: Luke Cwik <lc...@google.com>
Authored: Fri Nov 4 14:20:14 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Nov 4 14:20:14 2016 -0700
----------------------------------------------------------------------
.../beam/examples/WindowedWordCountIT.java | 11 ++++++++---
.../org/apache/beam/examples/WordCountIT.java | 19 +++++++++++--------
.../examples/cookbook/BigQueryTornadoesIT.java | 11 ++++++++---
3 files changed, 27 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[49/50] incubator-beam git commit: [BEAM-79] update
GearpumpPipelineResult
Posted by ke...@apache.org.
[BEAM-79] update GearpumpPipelineResult
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2a96a17f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2a96a17f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2a96a17f
Branch: refs/heads/gearpump-runner
Commit: 2a96a17f2fe9a391ec5b2f0b4bea223530c1ba34
Parents: 0c36228
Author: manuzhang <ow...@gmail.com>
Authored: Tue Nov 8 11:25:27 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Nov 8 11:30:26 2016 +0800
----------------------------------------------------------------------
.../org/apache/beam/runners/gearpump/GearpumpPipelineResult.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2a96a17f/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
----------------------------------------------------------------------
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
index e7c621e..ed1201d 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/GearpumpPipelineResult.java
@@ -44,12 +44,12 @@ public class GearpumpPipelineResult implements PipelineResult {
}
@Override
- public State waitUntilFinish(Duration duration) throws IOException, InterruptedException {
+ public State waitUntilFinish(Duration duration) {
return null;
}
@Override
- public State waitUntilFinish() throws IOException, InterruptedException {
+ public State waitUntilFinish() {
return null;
}
[05/50] incubator-beam git commit: Closes #1284
Posted by ke...@apache.org.
Closes #1284
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6e1e57b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6e1e57b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6e1e57b0
Branch: refs/heads/gearpump-runner
Commit: 6e1e57b09f4fbab021eb0ebb0748accc8bd90d0a
Parents: f06deac f19a25d
Author: Dan Halperin <dh...@google.com>
Authored: Fri Nov 4 10:56:40 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Nov 4 10:56:40 2016 -0700
----------------------------------------------------------------------
pom.xml | 4 ----
sdks/java/pom.xml | 5 +----
2 files changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
[31/50] incubator-beam git commit: Fix IT Options Conflict
Posted by ke...@apache.org.
Fix IT Options Conflict
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/19279689
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/19279689
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/19279689
Branch: refs/heads/gearpump-runner
Commit: 1927968955dfe834b3e48482a606ffa7fc403749
Parents: 3e84a5f
Author: Mark Liu <ma...@markliu0.mtv.corp.google.com>
Authored: Mon Nov 7 11:27:15 2016 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 13:45:43 2016 -0800
----------------------------------------------------------------------
.../apache/beam/examples/WindowedWordCountIT.java | 16 +++++++---------
.../java/org/apache/beam/examples/WordCountIT.java | 17 +++++++----------
.../examples/cookbook/BigQueryTornadoesIT.java | 16 +++++++---------
3 files changed, 21 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/19279689/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index d545ad2..c78fad6 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.examples;
-import com.google.common.base.Strings;
import java.io.IOException;
import org.apache.beam.examples.WindowedWordCount.Options;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -26,6 +25,7 @@ import org.apache.beam.sdk.testing.BigqueryMatcher;
import org.apache.beam.sdk.testing.StreamingIT;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -44,8 +44,11 @@ public class WindowedWordCountIT {
*/
public interface WindowedWordCountITOptions
extends Options, TestPipelineOptions, StreamingOptions {
- String getChecksum();
- void setChecksum(String value);
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ PipelineOptionsFactory.register(TestPipelineOptions.class);
}
@Test
@@ -60,20 +63,15 @@ public class WindowedWordCountIT {
}
private void testWindowedWordCountPipeline(boolean isStreaming) throws IOException {
- PipelineOptionsFactory.register(WindowedWordCountITOptions.class);
WindowedWordCountITOptions options =
TestPipeline.testingPipelineOptions().as(WindowedWordCountITOptions.class);
options.setStreaming(isStreaming);
String query = String.format("SELECT word, SUM(count) FROM [%s:%s.%s] GROUP BY word",
options.getProject(), options.getBigQueryDataset(), options.getBigQueryTable());
- String outputChecksum =
- Strings.isNullOrEmpty(options.getChecksum())
- ? DEFAULT_OUTPUT_CHECKSUM
- : options.getChecksum();
options.setOnSuccessMatcher(
new BigqueryMatcher(
- options.getAppName(), options.getProject(), query, outputChecksum));
+ options.getAppName(), options.getProject(), query, DEFAULT_OUTPUT_CHECKSUM));
WindowedWordCount.main(TestPipeline.convertToArgs(options));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/19279689/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 8f170af..487f04b 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -18,7 +18,6 @@
package org.apache.beam.examples;
-import com.google.common.base.Strings;
import java.util.Date;
import org.apache.beam.examples.WordCount.WordCountOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -26,6 +25,7 @@ import org.apache.beam.sdk.testing.FileChecksumMatcher;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestPipelineOptions;
import org.apache.beam.sdk.util.IOChannelUtils;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -45,13 +45,15 @@ public class WordCountIT {
* with customized input.
*/
public interface WordCountITOptions extends TestPipelineOptions, WordCountOptions {
- String getChecksum();
- void setChecksum(String value);
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ PipelineOptionsFactory.register(TestPipelineOptions.class);
}
@Test
public void testE2EWordCount() throws Exception {
- PipelineOptionsFactory.register(WordCountITOptions.class);
WordCountITOptions options = TestPipeline.testingPipelineOptions().as(WordCountITOptions.class);
options.setOutput(IOChannelUtils.resolve(
@@ -59,13 +61,8 @@ public class WordCountIT {
String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
"output",
"results"));
-
- String outputChecksum =
- Strings.isNullOrEmpty(options.getChecksum())
- ? DEFAULT_OUTPUT_CHECKSUM
- : options.getChecksum();
options.setOnSuccessMatcher(
- new FileChecksumMatcher(outputChecksum, options.getOutput() + "*"));
+ new FileChecksumMatcher(DEFAULT_OUTPUT_CHECKSUM, options.getOutput() + "*"));
WordCount.main(TestPipeline.convertToArgs(options));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/19279689/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
index 209c713..15c261f 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
@@ -18,12 +18,12 @@
package org.apache.beam.examples.cookbook;
-import com.google.common.base.Strings;
import org.apache.beam.sdk.options.BigQueryOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.BigqueryMatcher;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -41,13 +41,15 @@ public class BigQueryTornadoesIT {
*/
public interface BigQueryTornadoesITOptions
extends TestPipelineOptions, BigQueryTornadoes.Options, BigQueryOptions {
- String getChecksum();
- void setChecksum(String value);
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ PipelineOptionsFactory.register(TestPipelineOptions.class);
}
@Test
public void testE2EBigQueryTornadoes() throws Exception {
- PipelineOptionsFactory.register(BigQueryTornadoesITOptions.class);
BigQueryTornadoesITOptions options =
TestPipeline.testingPipelineOptions().as(BigQueryTornadoesITOptions.class);
options.setOutput(String.format("%s.%s",
@@ -55,13 +57,9 @@ public class BigQueryTornadoesIT {
String query =
String.format("SELECT month, tornado_count FROM [%s]", options.getOutput());
- String outputChecksum =
- Strings.isNullOrEmpty(options.getChecksum())
- ? DEFAULT_OUTPUT_CHECKSUM
- : options.getChecksum();
options.setOnSuccessMatcher(
new BigqueryMatcher(
- options.getAppName(), options.getProject(), query, outputChecksum));
+ options.getAppName(), options.getProject(), query, DEFAULT_OUTPUT_CHECKSUM));
BigQueryTornadoes.main(TestPipeline.convertToArgs(options));
}
[25/50] incubator-beam git commit: [BEAM-725] Migrate to use the
generic credentials library compatible with Apiary and gRPC instead of the
Apiary only credentials library.
Posted by ke...@apache.org.
[BEAM-725] Migrate to use the generic credentials library compatible with Apiary and gRPC instead of the Apiary only credentials library.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bb260ecd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bb260ecd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bb260ecd
Branch: refs/heads/gearpump-runner
Commit: bb260ecd34e14a29e7939912a101b3733e379248
Parents: b5f8473
Author: Luke Cwik <lc...@google.com>
Authored: Mon Nov 7 09:40:38 2016 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 13:08:39 2016 -0800
----------------------------------------------------------------------
pom.xml | 21 ++
runners/google-cloud-dataflow-java/pom.xml | 15 +-
.../dataflow/util/DataflowTransport.java | 9 +-
sdks/java/core/pom.xml | 19 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 136 ++-----------
.../beam/sdk/testing/BigqueryMatcher.java | 15 +-
.../apache/beam/sdk/util/CredentialFactory.java | 4 +-
.../org/apache/beam/sdk/util/Credentials.java | 192 -------------------
.../beam/sdk/util/GcpCredentialFactory.java | 41 +++-
.../beam/sdk/util/NoopCredentialFactory.java | 9 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 9 +-
.../apache/beam/sdk/util/TestCredential.java | 44 +++--
.../org/apache/beam/sdk/util/Transport.java | 9 +-
.../beam/sdk/util/PubsubGrpcClientTest.java | 9 +-
sdks/java/io/google-cloud-platform/pom.xml | 9 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 25 ++-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 25 ++-
18 files changed, 186 insertions(+), 423 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c1bd5c8..bd6037e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,6 +112,7 @@
<datastore.proto.version>1.2.0</datastore.proto.version>
<google-auto-service.version>1.0-rc2</google-auto-service.version>
<google-auto-value.version>1.1</google-auto-value.version>
+ <google-auth.version>0.6.0</google-auth.version>
<google-clients.version>1.22.0</google-clients.version>
<google-cloud-bigdataoss.version>1.4.5</google-cloud-bigdataoss.version>
<google-cloud-dataflow-java-proto-library-all.version>0.5.160304</google-cloud-dataflow-java-proto-library-all.version>
@@ -530,6 +531,26 @@
</dependency>
<dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-credentials</artifactId>
+ <version>${google-auth.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-oauth2-http</artifactId>
+ <version>${google-auth.version}</version>
+ <exclusions>
+ <!-- Exclude an old version of guava that is being pulled in by a transitive
+ dependency of google-api-client -->
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-jdk5</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
<version>${bigquery.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 59b1465..6ed41d0 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -171,11 +171,6 @@
</dependency>
<dependency>
- <groupId>com.google.oauth-client</groupId>
- <artifactId>google-oauth-client</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
</dependency>
@@ -202,6 +197,16 @@
</dependency>
<dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-credentials</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-oauth2-http</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.cloud.bigdataoss</groupId>
<artifactId>util</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java
index 0391594..e0026de 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/DataflowTransport.java
@@ -20,10 +20,11 @@ package org.apache.beam.runners.dataflow.util;
import static org.apache.beam.sdk.util.Transport.getJsonFactory;
import static org.apache.beam.sdk.util.Transport.getTransport;
-import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.services.clouddebugger.v2.Clouddebugger;
import com.google.api.services.dataflow.Dataflow;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
import com.google.common.collect.ImmutableList;
import java.net.MalformedURLException;
@@ -91,11 +92,13 @@ public class DataflowTransport {
}
private static HttpRequestInitializer chainHttpRequestInitializer(
- Credential credential, HttpRequestInitializer httpRequestInitializer) {
+ Credentials credential, HttpRequestInitializer httpRequestInitializer) {
if (credential == null) {
return httpRequestInitializer;
} else {
- return new ChainingHttpRequestInitializer(credential, httpRequestInitializer);
+ return new ChainingHttpRequestInitializer(
+ new HttpCredentialsAdapter(credential),
+ httpRequestInitializer);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 77a3309..17ef193 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -239,16 +239,12 @@
<dependency>
<groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-credentials</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auth</groupId>
<artifactId>google-auth-library-oauth2-http</artifactId>
- <version>0.4.0</version>
- <exclusions>
- <!-- Exclude an old version of guava that is being pulled in by a transitive
- dependency of google-api-client -->
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava-jdk5</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -312,11 +308,6 @@
<dependency>
<groupId>com.google.oauth-client</groupId>
- <artifactId>google-oauth-client-java6</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java
index 0ea6faf..ffdab98 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/GcpOptions.java
@@ -18,8 +18,7 @@
package org.apache.beam.sdk.options;
import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleOAuthConstants;
+import com.google.auth.Credentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.io.Files;
@@ -40,31 +39,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Options used to configure Google Cloud Platform project and credentials.
+ * Options used to configure Google Cloud Platform specific options such as the project
+ * and credentials.
*
- * <p>These options configure which of the following three different mechanisms for obtaining a
- * credential are used:
- * <ol>
- * <li>
- * It can fetch the
- * <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- * application default credentials</a>.
- * </li>
- * <li>
- * The user can specify a client secrets file and go through the OAuth2
- * webflow. The credential will then be cached in the user's home
- * directory for reuse.
- * </li>
- * <li>
- * The user can specify a file containing a service account private key along
- * with the service account name.
- * </li>
- * </ol>
- *
- * <p>The default mechanism is to use the
+ * <p>These options defer to the
* <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- * application default credentials</a>. The other options can be
- * used by setting the corresponding properties.
+ * application default credentials</a> for authentication. See the
+ * <a href="https://github.com/google/google-auth-library-java">Google Auth Library</a> for
+ * alternative mechanisms for creating credentials.
*/
@Description("Options used to configure Google Cloud Platform project and credentials.")
public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
@@ -78,79 +60,6 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
void setProject(String value);
/**
- * This option controls which file to use when attempting to create the credentials using the
- * service account method.
- *
- * <p>This option if specified, needs be combined with the
- * {@link GcpOptions#getServiceAccountName() serviceAccountName}.
- */
- @JsonIgnore
- @Description("Controls which file to use when attempting to create the credentials "
- + "using the service account method. This option if specified, needs to be combined with "
- + "the serviceAccountName option.")
- String getServiceAccountKeyfile();
- void setServiceAccountKeyfile(String value);
-
- /**
- * This option controls which service account to use when attempting to create the credentials
- * using the service account method.
- *
- * <p>This option if specified, needs be combined with the
- * {@link GcpOptions#getServiceAccountKeyfile() serviceAccountKeyfile}.
- */
- @JsonIgnore
- @Description("Controls which service account to use when attempting to create the credentials "
- + "using the service account method. This option if specified, needs to be combined with "
- + "the serviceAccountKeyfile option.")
- String getServiceAccountName();
- void setServiceAccountName(String value);
-
- /**
- * This option controls which file to use when attempting to create the credentials
- * using the OAuth 2 webflow. After the OAuth2 webflow, the credentials will be stored
- * within credentialDir.
- */
- @JsonIgnore
- @Description("This option controls which file to use when attempting to create the credentials "
- + "using the OAuth 2 webflow. After the OAuth2 webflow, the credentials will be stored "
- + "within credentialDir.")
- String getSecretsFile();
- void setSecretsFile(String value);
-
- /**
- * This option controls which credential store to use when creating the credentials
- * using the OAuth 2 webflow.
- */
- @Description("This option controls which credential store to use when creating the credentials "
- + "using the OAuth 2 webflow.")
- @Default.String("cloud_dataflow")
- String getCredentialId();
- void setCredentialId(String value);
-
- /**
- * Directory for storing dataflow credentials after execution of the OAuth 2 webflow. Defaults
- * to using the $HOME/.store/data-flow directory.
- */
- @Description("Directory for storing dataflow credentials after execution of the OAuth 2 webflow. "
- + "Defaults to using the $HOME/.store/data-flow directory.")
- @Default.InstanceFactory(CredentialDirFactory.class)
- String getCredentialDir();
- void setCredentialDir(String value);
-
- /**
- * Returns the default credential directory of ${user.home}/.store/data-flow.
- */
- class CredentialDirFactory implements DefaultValueFactory<String> {
- @Override
- public String create(PipelineOptions options) {
- File home = new File(System.getProperty("user.home"));
- File store = new File(home, ".store");
- File dataflow = new File(store, "data-flow");
- return dataflow.getPath();
- }
- }
-
- /**
* The class of the credential factory that should be created and used to create
* credentials. If gcpCredential has not been set explicitly, an instance of this class will
* be constructed and used as a credential factory.
@@ -173,9 +82,8 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
+ "If no credential has been set explicitly, the default is to use the instance factory "
+ "that constructs a credential based upon the currently set credentialFactoryClass.")
@Default.InstanceFactory(GcpUserCredentialsFactory.class)
- @Hidden
- Credential getGcpCredential();
- void setGcpCredential(Credential value);
+ Credentials getGcpCredential();
+ void setGcpCredential(Credentials value);
/**
* Attempts to infer the default project based upon the environment this application
@@ -251,9 +159,9 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
* Attempts to load the GCP credentials. See
* {@link CredentialFactory#getCredential()} for more details.
*/
- class GcpUserCredentialsFactory implements DefaultValueFactory<Credential> {
+ class GcpUserCredentialsFactory implements DefaultValueFactory<Credentials> {
@Override
- public Credential create(PipelineOptions options) {
+ public Credentials create(PipelineOptions options) {
GcpOptions gcpOptions = options.as(GcpOptions.class);
try {
CredentialFactory factory = InstanceBuilder.ofType(CredentialFactory.class)
@@ -269,28 +177,6 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions {
}
/**
- * The token server URL to use for OAuth 2 authentication. Normally, the default is sufficient,
- * but some specialized use cases may want to override this value.
- */
- @Description("The token server URL to use for OAuth 2 authentication. Normally, the default "
- + "is sufficient, but some specialized use cases may want to override this value.")
- @Default.String(GoogleOAuthConstants.TOKEN_SERVER_URL)
- @Hidden
- String getTokenServerUrl();
- void setTokenServerUrl(String value);
-
- /**
- * The authorization server URL to use for OAuth 2 authentication. Normally, the default is
- * sufficient, but some specialized use cases may want to override this value.
- */
- @Description("The authorization server URL to use for OAuth 2 authentication. Normally, the "
- + "default is sufficient, but some specialized use cases may want to override this value.")
- @Default.String(GoogleOAuthConstants.AUTHORIZATION_SERVER_URL)
- @Hidden
- String getAuthorizationServerEncodedUrl();
- void setAuthorizationServerEncodedUrl(String value);
-
- /**
* A GCS path for storing temporary files in GCP.
*
* <p>Its default to {@link PipelineOptions#getTempLocation}.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java
index 95208ce..9b8589a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/BigqueryMatcher.java
@@ -19,8 +19,6 @@ package org.apache.beam.sdk.testing;
import static com.google.common.base.Preconditions.checkArgument;
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.util.BackOff;
@@ -32,6 +30,9 @@ import com.google.api.services.bigquery.model.QueryRequest;
import com.google.api.services.bigquery.model.QueryResponse;
import com.google.api.services.bigquery.model.TableCell;
import com.google.api.services.bigquery.model.TableRow;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -136,9 +137,9 @@ public class BigqueryMatcher extends TypeSafeMatcher<PipelineResult>
Bigquery newBigqueryClient(String applicationName) {
HttpTransport transport = Transport.getTransport();
JsonFactory jsonFactory = Transport.getJsonFactory();
- Credential credential = getDefaultCredential(transport, jsonFactory);
+ Credentials credential = getDefaultCredential();
- return new Bigquery.Builder(transport, jsonFactory, credential)
+ return new Bigquery.Builder(transport, jsonFactory, new HttpCredentialsAdapter(credential))
.setApplicationName(applicationName)
.build();
}
@@ -168,10 +169,10 @@ public class BigqueryMatcher extends TypeSafeMatcher<PipelineResult>
!Strings.isNullOrEmpty(value), "Expected valid %s, but was %s", name, value);
}
- private Credential getDefaultCredential(HttpTransport transport, JsonFactory jsonFactory) {
- GoogleCredential credential;
+ private Credentials getDefaultCredential() {
+ GoogleCredentials credential;
try {
- credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
+ credential = GoogleCredentials.getApplicationDefault();
} catch (IOException e) {
throw new RuntimeException("Failed to get application default credential.", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java
index 884a77a..6229650 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/CredentialFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.auth.Credentials;
import java.io.IOException;
import java.security.GeneralSecurityException;
@@ -25,5 +25,5 @@ import java.security.GeneralSecurityException;
* Construct an oauth credential to be used by the SDK and the SDK workers.
*/
public interface CredentialFactory {
- Credential getCredential() throws IOException, GeneralSecurityException;
+ Credentials getCredential() throws IOException, GeneralSecurityException;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Credentials.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Credentials.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Credentials.java
deleted file mode 100644
index 1e77f4d..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Credentials.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.util;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.extensions.java6.auth.oauth2.AbstractPromptReceiver;
-import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
-import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
-import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleOAuthConstants;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.http.GenericUrl;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.client.util.store.FileDataStoreFactory;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import org.apache.beam.sdk.options.GcpOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Provides support for loading credentials.
- */
-public class Credentials {
-
- private static final Logger LOG = LoggerFactory.getLogger(Credentials.class);
-
- /**
- * OAuth 2.0 scopes used by a local worker (not on GCE).
- * The scope cloud-platform provides access to all Cloud Platform resources.
- * cloud-platform isn't sufficient yet for talking to datastore so we request
- * those resources separately.
- *
- * <p>Note that trusted scope relationships don't apply to OAuth tokens, so for
- * services we access directly (GCS) as opposed to through the backend
- * (BigQuery, GCE), we need to explicitly request that scope.
- */
- private static final List<String> SCOPES = Arrays.asList(
- "https://www.googleapis.com/auth/cloud-platform",
- "https://www.googleapis.com/auth/devstorage.full_control",
- "https://www.googleapis.com/auth/userinfo.email",
- "https://www.googleapis.com/auth/datastore");
-
- private static class PromptReceiver extends AbstractPromptReceiver {
- @Override
- public String getRedirectUri() {
- return GoogleOAuthConstants.OOB_REDIRECT_URI;
- }
- }
-
- /**
- * Initializes OAuth2 credentials.
- *
- * <p>This can use 3 different mechanisms for obtaining a credential:
- * <ol>
- * <li>
- * It can fetch the
- * <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- * application default credentials</a>.
- * </li>
- * <li>
- * The user can specify a client secrets file and go through the OAuth2
- * webflow. The credential will then be cached in the user's home
- * directory for reuse. Provide the property "secrets_file" to use this
- * mechanism.
- * </li>
- * <li>
- * The user can specify a file containing a service account.
- * Provide the properties "service_account_keyfile" and
- * "service_account_name" to use this mechanism.
- * </li>
- * </ol>
- * The default mechanism is to use the
- * <a href="https://developers.google.com/accounts/docs/application-default-credentials">
- * application default credentials</a>. The other options can be used by providing the
- * corresponding properties.
- */
- public static Credential getCredential(GcpOptions options)
- throws IOException, GeneralSecurityException {
- String keyFile = options.getServiceAccountKeyfile();
- String accountName = options.getServiceAccountName();
-
- if (keyFile != null && accountName != null) {
- try {
- return getCredentialFromFile(keyFile, accountName, SCOPES);
- } catch (GeneralSecurityException e) {
- throw new IOException("Unable to obtain credentials from file", e);
- }
- }
-
- if (options.getSecretsFile() != null) {
- return getCredentialFromClientSecrets(options, SCOPES);
- }
-
- try {
- return GoogleCredential.getApplicationDefault().createScoped(SCOPES);
- } catch (IOException e) {
- throw new RuntimeException("Unable to get application default credentials. Please see "
- + "https://developers.google.com/accounts/docs/application-default-credentials "
- + "for details on how to specify credentials. This version of the SDK is "
- + "dependent on the gcloud core component version 2015.02.05 or newer to "
- + "be able to get credentials from the currently authorized user via gcloud auth.", e);
- }
- }
-
- /**
- * Loads OAuth2 credential from a local file.
- */
- private static Credential getCredentialFromFile(
- String keyFile, String accountId, Collection<String> scopes)
- throws IOException, GeneralSecurityException {
- GoogleCredential credential = new GoogleCredential.Builder()
- .setTransport(Transport.getTransport())
- .setJsonFactory(Transport.getJsonFactory())
- .setServiceAccountId(accountId)
- .setServiceAccountScopes(scopes)
- .setServiceAccountPrivateKeyFromP12File(new File(keyFile))
- .build();
-
- LOG.info("Created credential from file {}", keyFile);
- return credential;
- }
-
- /**
- * Loads OAuth2 credential from client secrets, which may require an
- * interactive authorization prompt.
- */
- private static Credential getCredentialFromClientSecrets(
- GcpOptions options, Collection<String> scopes)
- throws IOException, GeneralSecurityException {
- String clientSecretsFile = options.getSecretsFile();
-
- checkArgument(clientSecretsFile != null);
- HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
-
- JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
- GoogleClientSecrets clientSecrets;
-
- try {
- clientSecrets = GoogleClientSecrets.load(jsonFactory,
- new FileReader(clientSecretsFile));
- } catch (IOException e) {
- throw new RuntimeException(
- "Could not read the client secrets from file: " + clientSecretsFile,
- e);
- }
-
- FileDataStoreFactory dataStoreFactory =
- new FileDataStoreFactory(new java.io.File(options.getCredentialDir()));
-
- GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder(
- httpTransport, jsonFactory, clientSecrets, scopes)
- .setDataStoreFactory(dataStoreFactory)
- .setTokenServerUrl(new GenericUrl(options.getTokenServerUrl()))
- .setAuthorizationServerEncodedUrl(options.getAuthorizationServerEncodedUrl())
- .build();
-
- // The credentialId identifies the credential if we're using a persistent
- // credential store.
- Credential credential =
- new AuthorizationCodeInstalledApp(flow, new PromptReceiver())
- .authorize(options.getCredentialId());
-
- LOG.info("Got credential from client secret");
- return credential;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java
index 0497e75..feb93f7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcpCredentialFactory.java
@@ -17,10 +17,11 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.GoogleCredentials;
import java.io.IOException;
-import java.security.GeneralSecurityException;
-import org.apache.beam.sdk.options.GcpOptions;
+import java.util.Arrays;
+import java.util.List;
import org.apache.beam.sdk.options.PipelineOptions;
/**
@@ -28,18 +29,38 @@ import org.apache.beam.sdk.options.PipelineOptions;
* Returns a GCP credential.
*/
public class GcpCredentialFactory implements CredentialFactory {
- private GcpOptions options;
+ /**
+ * The scope cloud-platform provides access to all Cloud Platform resources.
+ * cloud-platform isn't sufficient yet for talking to datastore so we request
+ * those resources separately.
+ *
+ * <p>Note that trusted scope relationships don't apply to OAuth tokens, so for
+ * services we access directly (GCS) as opposed to through the backend
+ * (BigQuery, GCE), we need to explicitly request that scope.
+ */
+ private static final List<String> SCOPES = Arrays.asList(
+ "https://www.googleapis.com/auth/cloud-platform",
+ "https://www.googleapis.com/auth/devstorage.full_control",
+ "https://www.googleapis.com/auth/userinfo.email",
+ "https://www.googleapis.com/auth/datastore",
+ "https://www.googleapis.com/auth/pubsub");
- private GcpCredentialFactory(GcpOptions options) {
- this.options = options;
- }
+ private static final GcpCredentialFactory INSTANCE = new GcpCredentialFactory();
public static GcpCredentialFactory fromOptions(PipelineOptions options) {
- return new GcpCredentialFactory(options.as(GcpOptions.class));
+ return INSTANCE;
}
@Override
- public Credential getCredential() throws IOException, GeneralSecurityException {
- return Credentials.getCredential(options);
+ public Credentials getCredential() throws IOException {
+ try {
+ return GoogleCredentials.getApplicationDefault().createScoped(SCOPES);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to get application default credentials. Please see "
+ + "https://developers.google.com/accounts/docs/application-default-credentials "
+ + "for details on how to specify credentials. This version of the SDK is "
+ + "dependent on the gcloud core component version 2015.02.05 or newer to "
+ + "be able to get credentials from the currently authorized user via gcloud auth.", e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java
index 5d92552..29c3e72 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NoopCredentialFactory.java
@@ -17,9 +17,8 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.auth.Credentials;
import java.io.IOException;
-import java.security.GeneralSecurityException;
import org.apache.beam.sdk.options.PipelineOptions;
/**
@@ -27,12 +26,14 @@ import org.apache.beam.sdk.options.PipelineOptions;
* Always returns a null Credential object.
*/
public class NoopCredentialFactory implements CredentialFactory {
+ private static final NoopCredentialFactory INSTANCE = new NoopCredentialFactory();
+
public static NoopCredentialFactory fromOptions(PipelineOptions options) {
- return new NoopCredentialFactory();
+ return INSTANCE;
}
@Override
- public Credential getCredential() throws IOException, GeneralSecurityException {
+ public Credentials getCredential() throws IOException {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubGrpcClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubGrpcClient.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubGrpcClient.java
index 02152ba..201877c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubGrpcClient.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubGrpcClient.java
@@ -20,7 +20,7 @@ package org.apache.beam.sdk.util;
import static com.google.common.base.Preconditions.checkState;
-import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.Credentials;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
@@ -56,7 +56,6 @@ import io.grpc.netty.NegotiationType;
import io.grpc.netty.NettyChannelBuilder;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
@@ -74,10 +73,6 @@ import org.apache.beam.sdk.options.PubsubOptions;
public class PubsubGrpcClient extends PubsubClient {
private static final String PUBSUB_ADDRESS = "pubsub.googleapis.com";
private static final int PUBSUB_PORT = 443;
- // Will be needed when credentials are correctly constructed and scoped.
- @SuppressWarnings("unused")
- private static final List<String> PUBSUB_SCOPES =
- Collections.singletonList("https://www.googleapis.com/auth/pubsub");
private static final int LIST_BATCH_SIZE = 1000;
private static final int DEFAULT_TIMEOUT_S = 15;
@@ -92,15 +87,12 @@ public class PubsubGrpcClient extends PubsubClient {
.negotiationType(NegotiationType.TLS)
.sslContext(GrpcSslContexts.forClient().ciphers(null).build())
.build();
- // TODO: GcpOptions needs to support building com.google.auth.oauth2.Credentials from the
- // various command line options. It currently only supports the older
- // com.google.api.client.auth.oauth2.Credentials.
- GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
+
return new PubsubGrpcClient(timestampLabel,
idLabel,
DEFAULT_TIMEOUT_S,
channel,
- credentials);
+ options.getGcpCredential());
}
@Override
@@ -128,7 +120,7 @@ public class PubsubGrpcClient extends PubsubClient {
/**
* Credentials determined from options and environment.
*/
- private final GoogleCredentials credentials;
+ private final Credentials credentials;
/**
* Label to use for custom timestamps, or {@literal null} if should use Pubsub publish time
@@ -157,7 +149,7 @@ public class PubsubGrpcClient extends PubsubClient {
@Nullable String idLabel,
int timeoutSec,
ManagedChannel publisherChannel,
- GoogleCredentials credentials) {
+ Credentials credentials) {
this.timestampLabel = timestampLabel;
this.idLabel = idLabel;
this.timeoutSec = timeoutSec;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubJsonClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubJsonClient.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubJsonClient.java
index bdb5c04..215a136 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubJsonClient.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubJsonClient.java
@@ -20,7 +20,6 @@ package org.apache.beam.sdk.util;
import static com.google.common.base.Preconditions.checkState;
-import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.services.pubsub.Pubsub;
import com.google.api.services.pubsub.Pubsub.Builder;
@@ -36,6 +35,8 @@ import com.google.api.services.pubsub.model.PullResponse;
import com.google.api.services.pubsub.model.ReceivedMessage;
import com.google.api.services.pubsub.model.Subscription;
import com.google.api.services.pubsub.model.Topic;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
@@ -55,11 +56,13 @@ public class PubsubJsonClient extends PubsubClient {
private static class PubsubJsonClientFactory implements PubsubClientFactory {
private static HttpRequestInitializer chainHttpRequestInitializer(
- Credential credential, HttpRequestInitializer httpRequestInitializer) {
+ Credentials credential, HttpRequestInitializer httpRequestInitializer) {
if (credential == null) {
return httpRequestInitializer;
} else {
- return new ChainingHttpRequestInitializer(credential, httpRequestInitializer);
+ return new ChainingHttpRequestInitializer(
+ new HttpCredentialsAdapter(credential),
+ httpRequestInitializer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TestCredential.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TestCredential.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TestCredential.java
index 4b81a0e..f34527e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TestCredential.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TestCredential.java
@@ -17,35 +17,43 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.auth.oauth2.BearerToken;
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.auth.oauth2.TokenResponse;
-import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.auth.Credentials;
import java.io.IOException;
+import java.net.URI;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
/**
* Fake credential, for use in testing.
*/
-public class TestCredential extends Credential {
+public class TestCredential extends Credentials {
+ @Override
+ public String getAuthenticationType() {
+ return "Test";
+ }
- private final String token;
+ @Override
+ public Map<String, List<String>> getRequestMetadata() throws IOException {
+ return Collections.emptyMap();
+ }
- public TestCredential() {
- this("NULL");
+ @Override
+ public Map<String, List<String>> getRequestMetadata(URI uri) throws IOException {
+ return Collections.emptyMap();
}
- public TestCredential(String token) {
- super(new Builder(
- BearerToken.authorizationHeaderAccessMethod())
- .setTransport(new MockHttpTransport()));
- this.token = token;
+ @Override
+ public boolean hasRequestMetadata() {
+ return false;
+ }
+
+ @Override
+ public boolean hasRequestMetadataOnly() {
+ return true;
}
@Override
- protected TokenResponse executeRefreshToken() throws IOException {
- TokenResponse response = new TokenResponse();
- response.setExpiresInSeconds(5L * 60);
- response.setAccessToken(token);
- return response;
+ public void refresh() throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Transport.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Transport.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Transport.java
index 1f61299..38eecc2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Transport.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/Transport.java
@@ -17,7 +17,6 @@
*/
package org.apache.beam.sdk.util;
-import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
@@ -27,6 +26,8 @@ import com.google.api.services.bigquery.Bigquery;
import com.google.api.services.cloudresourcemanager.CloudResourceManager;
import com.google.api.services.pubsub.Pubsub;
import com.google.api.services.storage.Storage;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
@@ -161,11 +162,13 @@ public class Transport {
}
private static HttpRequestInitializer chainHttpRequestInitializer(
- Credential credential, HttpRequestInitializer httpRequestInitializer) {
+ Credentials credential, HttpRequestInitializer httpRequestInitializer) {
if (credential == null) {
return httpRequestInitializer;
} else {
- return new ChainingHttpRequestInitializer(credential, httpRequestInitializer);
+ return new ChainingHttpRequestInitializer(
+ new HttpCredentialsAdapter(credential),
+ httpRequestInitializer);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/core/src/test/java/org/apache/beam/sdk/util/PubsubGrpcClientTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/PubsubGrpcClientTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/PubsubGrpcClientTest.java
index cbdf5da..9767cde 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/PubsubGrpcClientTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/PubsubGrpcClientTest.java
@@ -20,7 +20,7 @@ package org.apache.beam.sdk.util;
import static org.junit.Assert.assertEquals;
-import com.google.auth.oauth2.GoogleCredentials;
+import com.google.auth.Credentials;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -52,7 +52,6 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-import org.mockito.Mockito;
/**
* Tests for PubsubGrpcClient.
@@ -60,7 +59,7 @@ import org.mockito.Mockito;
@RunWith(JUnit4.class)
public class PubsubGrpcClientTest {
private ManagedChannel inProcessChannel;
- private GoogleCredentials mockCredentials;
+ private Credentials testCredentials;
private PubsubClient client;
private String channelName;
@@ -83,8 +82,8 @@ public class PubsubGrpcClientTest {
channelName = String.format("%s-%s",
PubsubGrpcClientTest.class.getName(), ThreadLocalRandom.current().nextInt());
inProcessChannel = InProcessChannelBuilder.forName(channelName).directExecutor().build();
- mockCredentials = Mockito.mock(GoogleCredentials.class);
- client = new PubsubGrpcClient(TIMESTAMP_LABEL, ID_LABEL, 10, inProcessChannel, mockCredentials);
+ testCredentials = new TestCredential();
+ client = new PubsubGrpcClient(TIMESTAMP_LABEL, ID_LABEL, 10, inProcessChannel, testCredentials);
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/io/google-cloud-platform/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 9f7af52..31350a9 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -169,8 +169,13 @@
</dependency>
<dependency>
- <groupId>com.google.oauth-client</groupId>
- <artifactId>google-oauth-client</artifactId>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-credentials</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.auth</groupId>
+ <artifactId>google-auth-library-oauth2-http</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 1d1075c..d1a9a67 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -31,11 +31,14 @@ import static com.google.datastore.v1.client.DatastoreHelper.makeOrder;
import static com.google.datastore.v1.client.DatastoreHelper.makeUpsert;
import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.util.BackOff;
import com.google.api.client.util.BackOffUtils;
import com.google.api.client.util.Sleeper;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
import com.google.auto.value.AutoValue;
+import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
@@ -229,6 +232,7 @@ public class DatastoreV1 {
@Nullable public abstract String getNamespace();
public abstract int getNumQuerySplits();
+ @Override
public abstract String toString();
abstract Builder toBuilder();
@@ -1005,17 +1009,20 @@ public class DatastoreV1 {
/** Builds a Cloud Datastore client for the given pipeline options and project. */
public Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) {
+ Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
+ HttpRequestInitializer initializer;
+ if (credential != null) {
+ initializer = new ChainingHttpRequestInitializer(
+ new HttpCredentialsAdapter(credential),
+ new RetryHttpRequestInitializer());
+ } else {
+ initializer = new RetryHttpRequestInitializer();
+ }
+
DatastoreOptions.Builder builder =
new DatastoreOptions.Builder()
.projectId(projectId)
- .initializer(
- new RetryHttpRequestInitializer()
- );
-
- Credential credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
- if (credential != null) {
- builder.credential(credential);
- }
+ .initializer(initializer);
return DatastoreFactory.get().create(builder.build());
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bb260ecd/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
index b680a0e..76a1fc8 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
@@ -25,10 +25,13 @@ import static com.google.datastore.v1.client.DatastoreHelper.makeKey;
import static com.google.datastore.v1.client.DatastoreHelper.makeUpsert;
import static com.google.datastore.v1.client.DatastoreHelper.makeValue;
-import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.util.BackOff;
import com.google.api.client.util.BackOffUtils;
import com.google.api.client.util.Sleeper;
+import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
import com.google.datastore.v1.CommitRequest;
import com.google.datastore.v1.Entity;
import com.google.datastore.v1.EntityResult;
@@ -131,17 +134,21 @@ class V1TestUtil {
* Build a new datastore client.
*/
static Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) {
+ Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
+ HttpRequestInitializer initializer;
+ if (credential != null) {
+ initializer = new ChainingHttpRequestInitializer(
+ new HttpCredentialsAdapter(credential),
+ new RetryHttpRequestInitializer());
+ } else {
+ initializer = new RetryHttpRequestInitializer();
+ }
+
DatastoreOptions.Builder builder =
new DatastoreOptions.Builder()
.projectId(projectId)
- .initializer(
- new RetryHttpRequestInitializer()
- );
+ .initializer(initializer);
- Credential credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
- if (credential != null) {
- builder.credential(credential);
- }
return DatastoreFactory.get().create(builder.build());
}
@@ -209,6 +216,7 @@ class V1TestUtil {
*A MutationBuilder that performs upsert operation.
*/
static class UpsertMutationBuilder implements MutationBuilder {
+ @Override
public Mutation.Builder apply(Entity entity) {
return makeUpsert(entity);
}
@@ -218,6 +226,7 @@ class V1TestUtil {
* A MutationBuilder that performs delete operation.
*/
static class DeleteMutationBuilder implements MutationBuilder {
+ @Override
public Mutation.Builder apply(Entity entity) {
return makeDelete(entity.getKey());
}
[21/50] incubator-beam git commit: This closes #1296
Posted by ke...@apache.org.
This closes #1296
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/11024552
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/11024552
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/11024552
Branch: refs/heads/gearpump-runner
Commit: 110245526d204287ffb30d9b8e91aca4003542f6
Parents: f6a9733 79b0455
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Nov 7 10:28:21 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 10:28:21 2016 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/Regex.java | 454 +++++++++++++++++
.../beam/sdk/transforms/RegexTransform.java | 505 -------------------
.../apache/beam/sdk/transforms/RegexTest.java | 248 +++++++++
.../beam/sdk/transforms/RegexTransformTest.java | 262 ----------
4 files changed, 702 insertions(+), 767 deletions(-)
----------------------------------------------------------------------
[18/50] incubator-beam git commit: This closes #506
Posted by ke...@apache.org.
This closes #506
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f6a9733f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f6a9733f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f6a9733f
Branch: refs/heads/gearpump-runner
Commit: f6a9733f58f6af5248608b7c433fee5a01e3b8b8
Parents: 70255d2 bc28799
Author: Jesse Anderson <je...@smokinghand.com>
Authored: Mon Nov 7 09:40:32 2016 -0800
Committer: Jesse Anderson <je...@smokinghand.com>
Committed: Mon Nov 7 09:40:32 2016 -0800
----------------------------------------------------------------------
.../beam/sdk/transforms/RegexTransform.java | 505 +++++++++++++++++++
.../beam/sdk/transforms/RegexTransformTest.java | 262 ++++++++++
2 files changed, 767 insertions(+)
----------------------------------------------------------------------
[39/50] incubator-beam git commit: Allow BoundedWindow subclasses in
DoFn parameter list
Posted by ke...@apache.org.
Allow BoundedWindow subclasses in DoFn parameter list
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9c3e59fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9c3e59fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9c3e59fa
Branch: refs/heads/gearpump-runner
Commit: 9c3e59fab86e93477f14e0709ae8ecc37b84f3ef
Parents: 85b908b
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Nov 3 21:30:25 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:03 2016 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/ParDo.java | 43 +++++++++++++-
.../sdk/transforms/reflect/DoFnInvokers.java | 8 ++-
.../sdk/transforms/reflect/DoFnSignature.java | 40 +++++++++----
.../sdk/transforms/reflect/DoFnSignatures.java | 41 +++++++++++--
.../beam/sdk/transforms/windowing/WindowFn.java | 12 ++++
.../apache/beam/sdk/transforms/ParDoTest.java | 61 ++++++++++++++++++++
.../transforms/reflect/DoFnInvokersTest.java | 6 +-
7 files changed, 190 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index 0684a5c..26799c0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -32,7 +32,10 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.MethodWithExtraParameters;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.OnTimerMethod;
import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.StringUtils;
@@ -41,6 +44,7 @@ import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.TypedPValue;
/**
@@ -548,6 +552,42 @@ public class ParDo {
}
/**
+ * Perform common validations of the {@link DoFn} against the input {@link PCollection}, for
+ * example ensuring that the window type expected by the {@link DoFn} matches the window type of
+ * the {@link PCollection}.
+ */
+ private static <InputT, OutputT> void validateWindowType(
+ PCollection<? extends InputT> input, Serializable fn) {
+ // No validation for OldDoFn
+ if (!(fn instanceof DoFn)) {
+ return;
+ }
+
+ DoFnSignature signature = DoFnSignatures.INSTANCE.getSignature((Class) fn.getClass());
+
+ TypeDescriptor<? extends BoundedWindow> actualWindowT =
+ input.getWindowingStrategy().getWindowFn().getWindowTypeDescriptor();
+
+ validateWindowTypeForMethod(actualWindowT, signature.processElement());
+ for (OnTimerMethod method : signature.onTimerMethods().values()) {
+ validateWindowTypeForMethod(actualWindowT, method);
+ }
+ }
+
+ private static void validateWindowTypeForMethod(
+ TypeDescriptor<? extends BoundedWindow> actualWindowT,
+ MethodWithExtraParameters methodSignature) {
+ if (methodSignature.windowT() != null) {
+ checkArgument(
+ methodSignature.windowT().isSupertypeOf(actualWindowT),
+ "%s expects window type %s, which is not a supertype of actual window type %s",
+ methodSignature.targetMethod(),
+ methodSignature.windowT(),
+ actualWindowT);
+ }
+ }
+
+ /**
* Perform common validations of the {@link DoFn}, for example ensuring that state is used
* correctly and that its features can be supported.
*/
@@ -768,6 +808,7 @@ public class ParDo {
public PCollection<OutputT> apply(PCollection<? extends InputT> input) {
checkArgument(
!isSplittable(getOldFn()), "Splittable DoFn not supported by the current runner");
+ validateWindowType(input, fn);
return PCollection.<OutputT>createPrimitiveOutputInternal(
input.getPipeline(),
input.getWindowingStrategy(),
@@ -1024,7 +1065,7 @@ public class ParDo {
public PCollectionTuple apply(PCollection<? extends InputT> input) {
checkArgument(
!isSplittable(getOldFn()), "Splittable DoFn not supported by the current runner");
-
+ validateWindowType(input, fn);
PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal(
input.getPipeline(),
TupleTagList.of(mainOutputTag).and(sideOutputTags.getAll()),
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
index ba95f98..b975711 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
@@ -510,8 +510,8 @@ public class DoFnInvokers {
private static MethodDescription getExtraContextFactoryMethodDescription(
String methodName, Class<?>... parameterTypes) {
try {
- return new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod(methodName, parameterTypes));
+ return new MethodDescription.ForLoadedMethod(
+ DoFn.ExtraContextFactory.class.getMethod(methodName, parameterTypes));
} catch (Exception e) {
throw new IllegalStateException(
String.format(
@@ -538,7 +538,9 @@ public class DoFnInvokers {
@Override
public StackManipulation dispatch(WindowParameter p) {
- return simpleExtraContextParameter("window", pushExtraContextFactory);
+ return new StackManipulation.Compound(
+ simpleExtraContextParameter("window", pushExtraContextFactory),
+ TypeCasting.to(new TypeDescription.ForLoadedType(p.windowT().getRawType())));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index befc10b..4cbe219 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -35,6 +35,7 @@ import org.apache.beam.sdk.transforms.DoFn.StateId;
import org.apache.beam.sdk.transforms.DoFn.TimerId;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -144,6 +145,10 @@ public abstract class DoFnSignature {
* <p>Validation that these are allowed is external to this class.
*/
List<Parameter> extraParameters();
+
+ /** The type of window expected by this method, if any. */
+ @Nullable
+ TypeDescriptor<? extends BoundedWindow> windowT();
}
/** A descriptor for an optional parameter of the {@link DoFn.ProcessElement} method. */
@@ -229,18 +234,14 @@ public abstract class DoFnSignature {
}
// These parameter descriptors are constant
- private static final WindowParameter BOUNDED_WINDOW_PARAMETER =
- new AutoValue_DoFnSignature_Parameter_WindowParameter();
private static final InputProviderParameter INPUT_PROVIDER_PARAMETER =
new AutoValue_DoFnSignature_Parameter_InputProviderParameter();
private static final OutputReceiverParameter OUTPUT_RECEIVER_PARAMETER =
new AutoValue_DoFnSignature_Parameter_OutputReceiverParameter();
- /**
- * Returns a {@link WindowParameter}.
- */
- public static WindowParameter boundedWindow() {
- return BOUNDED_WINDOW_PARAMETER;
+ /** Returns a {@link WindowParameter}. */
+ public static WindowParameter boundedWindow(TypeDescriptor<? extends BoundedWindow> windowT) {
+ return new AutoValue_DoFnSignature_Parameter_WindowParameter(windowT);
}
/**
@@ -283,6 +284,7 @@ public abstract class DoFnSignature {
@AutoValue
public abstract static class WindowParameter extends Parameter {
WindowParameter() {}
+ public abstract TypeDescriptor<? extends BoundedWindow> windowT();
}
/**
@@ -357,6 +359,10 @@ public abstract class DoFnSignature {
@Nullable
public abstract TypeDescriptor<?> trackerT();
+ /** The window type used by this method, if any. */
+ @Nullable
+ public abstract TypeDescriptor<? extends BoundedWindow> windowT();
+
/** Whether this {@link DoFn} returns a {@link ProcessContinuation} or void. */
public abstract boolean hasReturnValue();
@@ -364,9 +370,14 @@ public abstract class DoFnSignature {
Method targetMethod,
List<Parameter> extraParameters,
TypeDescriptor<?> trackerT,
+ @Nullable TypeDescriptor<? extends BoundedWindow> windowT,
boolean hasReturnValue) {
return new AutoValue_DoFnSignature_ProcessElementMethod(
- targetMethod, Collections.unmodifiableList(extraParameters), trackerT, hasReturnValue);
+ targetMethod,
+ Collections.unmodifiableList(extraParameters),
+ trackerT,
+ windowT,
+ hasReturnValue);
}
/**
@@ -381,6 +392,7 @@ public abstract class DoFnSignature {
extraParameters(),
Predicates.or(
Predicates.instanceOf(WindowParameter.class),
+ Predicates.instanceOf(TimerParameter.class),
Predicates.instanceOf(StateParameter.class)));
}
@@ -404,13 +416,21 @@ public abstract class DoFnSignature {
@Override
public abstract Method targetMethod();
+ /** The window type used by this method, if any. */
+ @Nullable
+ public abstract TypeDescriptor<? extends BoundedWindow> windowT();
+
/** Types of optional parameters of the annotated method, in the order they appear. */
@Override
public abstract List<Parameter> extraParameters();
- static OnTimerMethod create(Method targetMethod, String id, List<Parameter> extraParameters) {
+ static OnTimerMethod create(
+ Method targetMethod,
+ String id,
+ TypeDescriptor<? extends BoundedWindow> windowT,
+ List<Parameter> extraParameters) {
return new AutoValue_DoFnSignature_OnTimerMethod(
- id, targetMethod, Collections.unmodifiableList(extraParameters));
+ id, targetMethod, windowT, Collections.unmodifiableList(extraParameters));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index 09c5f3d..e918182 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -50,6 +50,7 @@ import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerDeclaration;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
@@ -149,6 +150,9 @@ public class DoFnSignatures {
private final Map<String, TimerParameter> timerParameters = new HashMap<>();
private final List<Parameter> extraParameters = new ArrayList<>();
+ @Nullable
+ private TypeDescriptor<? extends BoundedWindow> windowT;
+
private MethodAnalysisContext() {}
/** Indicates whether a {@link RestrictionTrackerParameter} is known in this context. */
@@ -157,6 +161,18 @@ public class DoFnSignatures {
extraParameters, Predicates.instanceOf(RestrictionTrackerParameter.class));
}
+ /** Indicates whether a {@link WindowParameter} is known in this context. */
+ public boolean hasWindowParameter() {
+ return Iterables.any(
+ extraParameters, Predicates.instanceOf(WindowParameter.class));
+ }
+
+ /** The window type, if any, used by this method. */
+ @Nullable
+ public TypeDescriptor<? extends BoundedWindow> getWindowType() {
+ return windowT;
+ }
+
/** State parameters declared in this context, keyed by {@link StateId}. */
public Map<String, StateParameter> getStateParameters() {
return Collections.unmodifiableMap(stateParameters);
@@ -599,6 +615,8 @@ public class DoFnSignatures {
MethodAnalysisContext methodContext = MethodAnalysisContext.create();
+ @Nullable TypeDescriptor<? extends BoundedWindow> windowT = getWindowType(fnClass, m);
+
List<DoFnSignature.Parameter> extraParameters = new ArrayList<>();
TypeDescriptor<?> expectedOutputReceiverT = outputReceiverTypeOf(outputT);
ErrorReporter onTimerErrors = errors.forMethod(DoFn.OnTimer.class, m);
@@ -618,7 +636,7 @@ public class DoFnSignatures {
expectedOutputReceiverT));
}
- return DoFnSignature.OnTimerMethod.create(m, timerId, extraParameters);
+ return DoFnSignature.OnTimerMethod.create(m, timerId, windowT, extraParameters);
}
@VisibleForTesting
@@ -650,6 +668,7 @@ public class DoFnSignatures {
formatType(processContextT));
TypeDescriptor<?> trackerT = getTrackerType(fnClass, m);
+ TypeDescriptor<? extends BoundedWindow> windowT = getWindowType(fnClass, m);
TypeDescriptor<?> expectedInputProviderT = inputProviderTypeOf(inputT);
TypeDescriptor<?> expectedOutputReceiverT = outputReceiverTypeOf(outputT);
for (int i = 1; i < params.length; ++i) {
@@ -684,6 +703,7 @@ public class DoFnSignatures {
m,
methodContext.getExtraParameters(),
trackerT,
+ windowT,
DoFn.ProcessContinuation.class.equals(m.getReturnType()));
}
@@ -700,12 +720,12 @@ public class DoFnSignatures {
ErrorReporter paramErrors = methodErrors.forParameter(param);
- if (rawType.equals(BoundedWindow.class)) {
+ if (BoundedWindow.class.isAssignableFrom(rawType)) {
methodErrors.checkArgument(
- !methodContext.getExtraParameters().contains(Parameter.boundedWindow()),
+ !methodContext.hasWindowParameter(),
"Multiple %s parameters",
BoundedWindow.class.getSimpleName());
- return Parameter.boundedWindow();
+ return Parameter.boundedWindow((TypeDescriptor<? extends BoundedWindow>) paramT);
} else if (rawType.equals(DoFn.InputProvider.class)) {
methodErrors.checkArgument(
!methodContext.getExtraParameters().contains(Parameter.inputProvider()),
@@ -856,6 +876,19 @@ public class DoFnSignatures {
return null;
}
+ @Nullable
+ private static TypeDescriptor<? extends BoundedWindow> getWindowType(
+ TypeDescriptor<?> fnClass, Method method) {
+ Type[] params = method.getGenericParameterTypes();
+ for (int i = 0; i < params.length; i++) {
+ TypeDescriptor<?> paramT = fnClass.resolveType(params[i]);
+ if (BoundedWindow.class.isAssignableFrom(paramT.getRawType())) {
+ return (TypeDescriptor<? extends BoundedWindow>) paramT;
+ }
+ }
+ return null;
+ }
+
@VisibleForTesting
static DoFnSignature.BundleMethod analyzeBundleMethod(
ErrorReporter errors,
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
index 127fb4f..ea0bb79 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/WindowFn.java
@@ -24,6 +24,7 @@ import org.apache.beam.sdk.annotations.Experimental.Kind;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.values.TypeDescriptor;
import org.joda.time.Instant;
/**
@@ -159,6 +160,17 @@ public abstract class WindowFn<T, W extends BoundedWindow>
}
/**
+ * Returns a {@link TypeDescriptor} capturing what is known statically about the window type of
+ * this {@link WindowFn} instance's most-derived class.
+ *
+ * <p>In the normal case of a concrete {@link WindowFn} subclass with no generic type parameters
+ * of its own (including anonymous inner classes), this will be a complete non-generic type.
+ */
+ public TypeDescriptor<W> getWindowTypeDescriptor() {
+ return new TypeDescriptor<W>(this) {};
+ }
+
+ /**
* {@inheritDoc}
*
* <p>By default, does not register any display data. Implementors may override this method
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
index d3ea9fb..26f5570 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ParDoTest.java
@@ -51,6 +51,8 @@ import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.RunnableOnService;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.DoFn.OnTimer;
+import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
import org.apache.beam.sdk.transforms.ParDo.Bound;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayData.Builder;
@@ -58,7 +60,12 @@ import org.apache.beam.sdk.transforms.display.DisplayDataMatchers;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.TimerSpec;
+import org.apache.beam.sdk.util.TimerSpecs;
import org.apache.beam.sdk.util.common.ElementByteSizeObserver;
import org.apache.beam.sdk.util.state.StateSpec;
import org.apache.beam.sdk.util.state.StateSpecs;
@@ -72,6 +79,7 @@ import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -1513,6 +1521,59 @@ public class ParDoTest implements Serializable {
}
@Test
+ public void testRejectsWrongWindowType() {
+ Pipeline p = TestPipeline.create();
+
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage(GlobalWindow.class.getSimpleName());
+ thrown.expectMessage(IntervalWindow.class.getSimpleName());
+ thrown.expectMessage("window type");
+ thrown.expectMessage("not a supertype");
+
+ p.apply(Create.of(1, 2, 3))
+ .apply(
+ ParDo.of(
+ new DoFn<Integer, Integer>() {
+ @ProcessElement
+ public void process(ProcessContext c, IntervalWindow w) {}
+ }));
+ }
+
+ /**
+ * Tests that it is OK to use different window types in the parameter lists to different
+ * {@link DoFn} functions, as long as they are all subtypes of the actual window type
+ * of the input.
+ *
+ * <p>Today, the only method other than {@link ProcessElement @ProcessElement} that can accept
+ * extended parameters is {@link OnTimer @OnTimer}, which is rejected before it reaches window
+ * type validation. Rather than delay validation, this test is temporarily disabled.
+ */
+ @Ignore("ParDo rejects this on account of it using timers")
+ @Test
+ public void testMultipleWindowSubtypesOK() {
+ final String timerId = "gobbledegook";
+
+ Pipeline p = TestPipeline.create();
+
+ p.apply(Create.of(1, 2, 3))
+ .apply(Window.<Integer>into(FixedWindows.of(Duration.standardSeconds(10))))
+ .apply(
+ ParDo.of(
+ new DoFn<Integer, Integer>() {
+ @TimerId(timerId)
+ private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void process(ProcessContext c, IntervalWindow w) {}
+
+ @OnTimer(timerId)
+ public void onTimer(BoundedWindow w) {}
+ }));
+
+ // If it doesn't crash, we made it!
+ }
+
+ @Test
public void testRejectsSplittableDoFnByDefault() {
// ParDo with a splittable DoFn must be overridden by the runner.
// Without an override, applying it directly must fail.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9c3e59fa/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 60f82a8..7bdc007 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.reflect.testhelper.DoFnInvokersTestHelper;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.util.TimeDomain;
import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.TimerSpec;
@@ -71,7 +71,7 @@ public class DoFnInvokersTest {
@Rule public ExpectedException thrown = ExpectedException.none();
@Mock private DoFn<String, String>.ProcessContext mockContext;
- @Mock private BoundedWindow mockWindow;
+ @Mock private IntervalWindow mockWindow;
@Mock private DoFn.InputProvider<String> mockInputProvider;
@Mock private DoFn.OutputReceiver<String> mockOutputReceiver;
@Mock private WindowingInternals<String, String> mockWindowingInternals;
@@ -173,7 +173,7 @@ public class DoFnInvokersTest {
public void testDoFnWithWindow() throws Exception {
class MockFn extends DoFn<String, String> {
@DoFn.ProcessElement
- public void processElement(ProcessContext c, BoundedWindow w) throws Exception {}
+ public void processElement(ProcessContext c, IntervalWindow w) throws Exception {}
}
MockFn fn = mock(MockFn.class);
assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
[16/50] incubator-beam git commit: Closes #1292
Posted by ke...@apache.org.
Closes #1292
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/70255d26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/70255d26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/70255d26
Branch: refs/heads/gearpump-runner
Commit: 70255d26880ce6443b8a931a6a778040fdbc5339
Parents: 5864a38 68f2638
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 08:51:40 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 08:51:40 2016 -0800
----------------------------------------------------------------------
examples/java/pom.xml | 6 ------
examples/java8/pom.xml | 6 ------
pom.xml | 5 +++++
runners/core-java/pom.xml | 6 ------
runners/direct-java/pom.xml | 6 ------
runners/flink/examples/pom.xml | 4 ----
runners/flink/runner/pom.xml | 4 ----
runners/google-cloud-dataflow-java/pom.xml | 6 ------
runners/spark/pom.xml | 4 ----
sdks/java/core/pom.xml | 6 ------
sdks/java/extensions/join-library/pom.xml | 4 ----
sdks/java/extensions/sorter/pom.xml | 4 ----
sdks/java/io/google-cloud-platform/pom.xml | 4 ----
sdks/java/io/hdfs/pom.xml | 4 ----
sdks/java/io/jdbc/pom.xml | 4 ----
sdks/java/io/jms/pom.xml | 4 ----
sdks/java/io/kafka/pom.xml | 4 ----
sdks/java/io/kinesis/pom.xml | 4 ----
sdks/java/io/mongodb/pom.xml | 4 ----
sdks/java/java8tests/pom.xml | 6 ------
sdks/java/microbenchmarks/pom.xml | 5 -----
21 files changed, 5 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
[17/50] incubator-beam git commit: Added Regex Transform and test.
Posted by ke...@apache.org.
Added Regex Transform and test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bc28799d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bc28799d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bc28799d
Branch: refs/heads/gearpump-runner
Commit: bc28799d575341d4ab359c971a10514f518249a0
Parents: 70255d2
Author: Jesse Anderson <je...@smokinghand.com>
Authored: Mon Jun 20 16:44:43 2016 -0700
Committer: Jesse Anderson <je...@smokinghand.com>
Committed: Mon Nov 7 09:39:29 2016 -0800
----------------------------------------------------------------------
.../beam/sdk/transforms/RegexTransform.java | 505 +++++++++++++++++++
.../beam/sdk/transforms/RegexTransformTest.java | 262 ++++++++++
2 files changed, 767 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc28799d/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
new file mode 100644
index 0000000..bd7848a
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * {@code PTransorm}s to use Regular Expressions to process elements in a
+ * {@link PCollection}.
+ *
+ * <p>
+ * {@link RegexTransform#matches(String, int)} can be used to see if an entire line matches
+ * a Regex. {@link RegexTransform#matchesKV(String, int, int)} can be used to see if an entire
+ * line matches a Regex and output certain groups as a {@link KV}.
+ * </p>
+ * <p>
+ * {@link RegexTransform#find(String, int)} can be used to see if a portion of a line
+ * matches a Regex. {@link RegexTransform#matchesKV(String, int, int)} can be used to see if a
+ * portion of a line matches a Regex and output certain groups as a {@link KV}.
+ * </p>
+ * <p>
+ * Lines that do not match the Regex will not be output.
+ * </p>
+ */
+public class RegexTransform {
+ private RegexTransform() {
+ // do not instantiate
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Matches} {@link PTransform} that checks if
+ * the entire line matches the Regex. Returns the entire line (group 0) as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Matches matches(String regex) {
+ return matches(regex, 0);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Matches} {@link PTransform} that checks if
+ * the entire line matches the Regex. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param group
+ * The Regex group to return as a PCollection
+ */
+ public static Matches matches(String regex, int group) {
+ return new Matches(regex, group);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.MatchesKV} {@link PTransform} that checks
+ * if the entire line matches the Regex. Returns the specified groups as the
+ * key and value as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param keyGroup
+ * The Regex group to use as the key
+ * @param valueGroup
+ * The Regex group to use the value
+ */
+ public static MatchesKV matchesKV(String regex, int keyGroup,
+ int valueGroup) {
+ return new MatchesKV(regex, keyGroup, valueGroup);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Find} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the entire line (group 0) as
+ * a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Find find(String regex) {
+ return find(regex, 0);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Find} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param group
+ * The Regex group to return as a PCollection
+ */
+ public static Find find(String regex, int group) {
+ return new Find(regex, group);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.FindKV} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the specified groups as the
+ * key and value as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param keyGroup
+ * The Regex group to use as the key
+ * @param valueGroup
+ * The Regex group to use the value
+ */
+ public static FindKV findKV(String regex, int keyGroup, int valueGroup) {
+ return new FindKV(regex, keyGroup, valueGroup);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.ReplaceAll} {@link PTransform} that checks if a
+ * portion of the line matches the Regex and replaces all matches with the replacement
+ * String. Returns the group as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param replacement
+ * The string to be substituted for each match
+ */
+ public static ReplaceAll replaceAll(String regex, String replacement) {
+ return new ReplaceAll(regex, replacement);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.ReplaceAll} {@link PTransform} that checks if a
+ * portion of the line matches the Regex and replaces the first match with the replacement
+ * String. Returns the group as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param replacement
+ * The string to be substituted for each match
+ */
+ public static ReplaceFirst replaceFirst(String regex, String replacement) {
+ return new ReplaceFirst(regex, replacement);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Split} {@link PTransform} that splits a string
+ * on the regular expression and then outputs each item. It will not output empty
+ * items. Returns the group as a {@link PCollection}.
+ * a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Split split(String regex) {
+ return split(regex, false);
+ }
+
+ /**
+ * Returns a {@link RegexTransform.Split} {@link PTransform} that splits a string
+ * on the regular expression and then outputs each item. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param outputEmpty
+ * Should empty be output. True to output empties and false if not.
+ */
+ public static Split split(String regex, boolean outputEmpty) {
+ return new Split(regex, outputEmpty);
+ }
+
+ /**
+ * {@code RegexTransform.Matches<String>} takes a {@code PCollection<String>}
+ * and returns a {@code PCollection<String>} representing the value
+ * extracted from the Regex groups of the input {@code PCollection}
+ * to the number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If the entire line
+ * does not match the Regex, the line will not be output. If it does match the
+ * entire line, the group in the Regex will be used. The output will be the
+ * Regex group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(RegexTransform.matches("myregex (mygroup)", 1));
+ * }
+ * </pre>
+ */
+ public static class Matches
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ int group;
+
+ public Matches(String regex, int group) {
+ this.pattern = Pattern.compile(regex);
+ this.group = group;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in
+ .apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.matches()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.MatchesKV<KV<String, String>>} takes a
+ * {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value
+ * extracted from the Regex groups of the input {@code PCollection} to the
+ * number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If the entire line
+ * does not match the Regex, the line will not be output. If it does match the
+ * entire line, the groups in the Regex will be used. The key will be the
+ * key's group and the value will be the value's group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<KV<String, String>> keysAndValues =
+ * words.apply(RegexTransform.matchesKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
+ * }
+ * </pre>
+ */
+ public static class MatchesKV
+ extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
+ Pattern pattern;
+ int keyGroup, valueGroup;
+
+ public MatchesKV(String regex, int keyGroup, int valueGroup) {
+ this.pattern = Pattern.compile(regex);
+ this.keyGroup = keyGroup;
+ this.valueGroup = valueGroup;
+ }
+
+ public PCollection<KV<String, String>> apply(PCollection<String> in) {
+ return in.apply(ParDo
+ .of(new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.Find<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} representing the value extracted
+ * from the Regex groups of the input {@code PCollection} to
+ * the number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will not be output. If it does
+ * match a portion of the line, the group in the Regex will be used. The
+ * output will be the Regex group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(RegexTransform.find("myregex (mygroup)", 1));
+ * }
+ * </pre>
+ */
+ public static class Find
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ int group;
+
+ public Find(String regex, int group) {
+ this.pattern = Pattern.compile(regex);
+ this.group = group;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.MatchesKV<KV<String, String>>} takes a
+ * {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value
+ * extracted from the Regex groups of the input {@code PCollection} to the
+ * number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will not be output. If it does
+ * match a portion of the line, the groups in the Regex will be used. The key
+ * will be the key's group and the value will be the value's group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<KV<String, String>> keysAndValues =
+ * words.apply(RegexTransform.findKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
+ * }
+ * </pre>
+ */
+ public static class FindKV
+ extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
+ Pattern pattern;
+ int keyGroup, valueGroup;
+
+ public FindKV(String regex, int keyGroup, int valueGroup) {
+ this.pattern = Pattern.compile(regex);
+ this.keyGroup = keyGroup;
+ this.valueGroup = valueGroup;
+ }
+
+ public PCollection<KV<String, String>> apply(PCollection<String> in) {
+ return in.apply(
+ ParDo.of(new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.ReplaceAll<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with all Strings that matched the
+ * Regex being replaced with the replacement string.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will be output without changes. If it does
+ * match a portion of the line, all portions matching the Regex will be replaced
+ * with the replacement String.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(RegexTransform.replaceAll("myregex", "myreplacement"));
+ * }
+ * </pre>
+ */
+ public static class ReplaceAll
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ String replacement;
+
+ public ReplaceAll(String regex, String replacement) {
+ this.pattern = Pattern.compile(regex);
+ this.replacement = replacement;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceAll(replacement));
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.ReplaceFirst<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with the first Strings that matched the
+ * Regex being replaced with the replacement string.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will be output without changes. If it does
+ * match a portion of the line, the first portion matching the Regex will be replaced
+ * with the replacement String.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(RegexTransform.replaceFirst("myregex", "myreplacement"));
+ * }
+ * </pre>
+ */
+ public static class ReplaceFirst
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ String replacement;
+
+ public ReplaceFirst(String regex, String replacement) {
+ this.pattern = Pattern.compile(regex);
+ this.replacement = replacement;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceFirst(replacement));
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code RegexTransform.Split<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with the input string split into
+ * individual items in a list. Each item is then output as a separate string.
+ *
+ * <p>
+ * This transform runs a Regex as part of a splint the entire input line. The split
+ * gives back an array of items. Each item is output as a separate item in the
+ * {@code PCollection<String>}.
+ * </p>
+ *
+ * <p>
+ * Depending on the Regex, a split can be an empty or
+ * "" string. You can pass in a parameter if you want empty strings or not.
+ * </p>
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(RegexTransform.split("\W*"));
+ * }
+ * </pre>
+ */
+ public static class Split
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ boolean outputEmpty;
+
+ public Split(String regex, boolean outputEmpty) {
+ this.pattern = Pattern.compile(regex);
+ this.outputEmpty = outputEmpty;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ String[] items = pattern.split(c.element());
+
+ for (String item : items) {
+ if (outputEmpty || !item.isEmpty()) {
+ c.output(item);
+ }
+ }
+ }
+ }));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bc28799d/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
new file mode 100644
index 0000000..63d02d7
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import java.io.Serializable;
+
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link RegexTransform}.
+ */
+@RunWith(JUnit4.class)
+public class RegexTransformTest implements Serializable {
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFind() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("aj", "xj", "yj", "zj"))
+ .apply(RegexTransform.find("[xyz]"));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFindGroup() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("aj", "xj", "yj", "zj"))
+ .apply(RegexTransform.find("([xyz])", 1));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFindNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "b", "c", "d"))
+ .apply(RegexTransform.find("[xyz]"));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVFind() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("a b c"))
+ .apply(RegexTransform.findKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVFindNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("x y z"))
+ .apply(RegexTransform.findKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatches() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "x", "y", "z"))
+ .apply(RegexTransform.matches("[xyz]"));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchesNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "b", "c", "d"))
+ .apply(RegexTransform.matches("[xyz]"));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchesGroup() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "x xxx", "x yyy", "x zzz"))
+ .apply(RegexTransform.matches("x ([xyz]*)", 1));
+
+ PAssert.that(output).containsInAnyOrder("xxx", "yyy", "zzz");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVMatches() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("a b c"))
+ .apply(RegexTransform.matchesKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVMatchesNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("x y z"))
+ .apply(RegexTransform.matchesKV("a (b) (c)", 1, 2));
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceAll() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("xj", "yj", "zj"))
+ .apply(RegexTransform.replaceAll("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("newj", "newj", "newj");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceAllMixed() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("abc", "xj", "yj", "zj", "def"))
+ .apply(RegexTransform.replaceAll("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("abc", "newj", "newj", "newj", "def");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceFirst() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("xjx", "yjy", "zjz"))
+ .apply(RegexTransform.replaceFirst("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("newjx", "newjy", "newjz");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceFirstMixed() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("abc", "xjx", "yjy", "zjz", "def"))
+ .apply(RegexTransform.replaceFirst("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("abc", "newjx", "newjy", "newjz", "def");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplits() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(RegexTransform.split("\\W+"));
+
+ PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
+ "fox", "jumps", "over", "the", "lazy", "dog");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplitsWithEmpty() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(RegexTransform.split("\\s", true));
+
+ String[] outputStr = "The quick brown fox jumps over the lazy dog".split("\\s");
+
+ PAssert.that(output).containsInAnyOrder("The", "", "quick", "brown", "", "",
+ "fox", "jumps", "over", "", "", "", "the", "lazy", "dog");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplitsWithoutEmpty() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(RegexTransform.split("\\s", false));
+
+ PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
+ "fox", "jumps", "over", "the", "lazy", "dog");
+ p.run();
+ }
+}
[14/50] incubator-beam git commit: Closes #1290
Posted by ke...@apache.org.
Closes #1290
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5864a38b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5864a38b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5864a38b
Branch: refs/heads/gearpump-runner
Commit: 5864a38ba595cdf90d2d5559e4312ef6144f60c7
Parents: 14e093a 717b431
Author: Dan Halperin <dh...@google.com>
Authored: Sat Nov 5 01:30:41 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Sat Nov 5 01:30:41 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 68 -------------------
examples/java8/pom.xml | 8 ---
.../beam/examples/complete/game/GameStats.java | 3 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
pom.xml | 11 ++-
runners/core-java/pom.xml | 23 -------
runners/direct-java/pom.xml | 61 -----------------
runners/flink/examples/pom.xml | 8 ---
.../flink/examples/streaming/AutoComplete.java | 8 +--
.../flink/examples/streaming/JoinExamples.java | 12 ++--
.../examples/streaming/KafkaIOExamples.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 8 +--
runners/flink/runner/pom.xml | 8 ---
runners/google-cloud-dataflow-java/pom.xml | 70 --------------------
.../runners/dataflow/DataflowPipelineJob.java | 1 -
.../runners/dataflow/internal/IsmFormat.java | 5 +-
.../runners/dataflow/util/MonitoringUtil.java | 1 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
runners/spark/pom.xml | 4 --
sdks/java/core/pom.xml | 70 --------------------
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 11 ++-
.../apache/beam/sdk/io/CompressedSource.java | 2 -
.../org/apache/beam/sdk/io/FileBasedSink.java | 9 +--
.../java/org/apache/beam/sdk/io/XmlSource.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../apache/beam/sdk/options/ValueProvider.java | 2 +-
.../apache/beam/sdk/transforms/Aggregator.java | 12 ++--
.../apache/beam/sdk/transforms/CombineFns.java | 4 +-
.../org/apache/beam/sdk/transforms/DoFn.java | 22 +++---
.../org/apache/beam/sdk/transforms/ParDo.java | 2 +-
.../org/apache/beam/sdk/transforms/View.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 5 +-
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../sdk/transforms/reflect/DoFnSignature.java | 2 +-
.../splittabledofn/RestrictionTracker.java | 2 +-
.../transforms/splittabledofn/package-info.java | 4 +-
.../beam/sdk/util/BaseExecutionContext.java | 2 +-
.../BufferedElementCountingOutputStream.java | 4 +-
.../apache/beam/sdk/util/ExecutionContext.java | 8 +--
.../sdk/util/ExposedByteArrayInputStream.java | 5 +-
.../sdk/util/ExposedByteArrayOutputStream.java | 2 -
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 -
.../apache/beam/sdk/util/MovingFunction.java | 4 +-
.../org/apache/beam/sdk/util/PubsubClient.java | 32 ++-------
.../org/apache/beam/sdk/util/StringUtils.java | 8 +--
.../apache/beam/sdk/util/TimerInternals.java | 8 ++-
.../beam/sdk/util/state/StateNamespace.java | 16 ++---
.../java/org/apache/beam/sdk/values/PDone.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
sdks/java/io/jdbc/pom.xml | 4 --
sdks/java/io/kafka/pom.xml | 7 --
sdks/java/io/kinesis/pom.xml | 7 --
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +++++-----
sdks/java/io/mongodb/pom.xml | 4 --
56 files changed, 119 insertions(+), 503 deletions(-)
----------------------------------------------------------------------
[26/50] incubator-beam git commit: [BEAM-725] Migrate to use the
generic Google credentials library
Posted by ke...@apache.org.
[BEAM-725] Migrate to use the generic Google credentials library
This closes #1294
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/baa7fb03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/baa7fb03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/baa7fb03
Branch: refs/heads/gearpump-runner
Commit: baa7fb0317afa4a18462ed66ab0a7ba1e8eb2c89
Parents: b5f8473 bb260ec
Author: Luke Cwik <lc...@google.com>
Authored: Mon Nov 7 13:09:19 2016 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 13:09:19 2016 -0800
----------------------------------------------------------------------
pom.xml | 21 ++
runners/google-cloud-dataflow-java/pom.xml | 15 +-
.../dataflow/util/DataflowTransport.java | 9 +-
sdks/java/core/pom.xml | 19 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 136 ++-----------
.../beam/sdk/testing/BigqueryMatcher.java | 15 +-
.../apache/beam/sdk/util/CredentialFactory.java | 4 +-
.../org/apache/beam/sdk/util/Credentials.java | 192 -------------------
.../beam/sdk/util/GcpCredentialFactory.java | 41 +++-
.../beam/sdk/util/NoopCredentialFactory.java | 9 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 9 +-
.../apache/beam/sdk/util/TestCredential.java | 44 +++--
.../org/apache/beam/sdk/util/Transport.java | 9 +-
.../beam/sdk/util/PubsubGrpcClientTest.java | 9 +-
sdks/java/io/google-cloud-platform/pom.xml | 9 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 25 ++-
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 25 ++-
18 files changed, 186 insertions(+), 423 deletions(-)
----------------------------------------------------------------------
[11/50] incubator-beam git commit: This closes #1272
Posted by ke...@apache.org.
This closes #1272
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/14e093a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/14e093a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/14e093a0
Branch: refs/heads/gearpump-runner
Commit: 14e093a0a574c8c3920a83c38e411a06b29bf44b
Parents: 46fbfe0 90a75d1
Author: Sela <an...@paypal.com>
Authored: Sat Nov 5 00:02:22 2016 +0200
Committer: Sela <an...@paypal.com>
Committed: Sat Nov 5 00:02:22 2016 +0200
----------------------------------------------------------------------
.../runners/spark/SparkPipelineOptions.java | 3 +--
.../SparkRunnerStreamingContextFactory.java | 23 +++++---------------
.../streaming/EmptyStreamAssertionTest.java | 3 +--
.../streaming/FlattenStreamingTest.java | 6 ++---
.../streaming/KafkaStreamingTest.java | 6 ++---
.../ResumeFromCheckpointStreamingTest.java | 3 +--
.../streaming/SimpleStreamingWordCountTest.java | 3 +--
.../utils/TestOptionsForStreaming.java | 12 +++++-----
8 files changed, 19 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-beam git commit: Generalize extraction of DoFn
parameters from context
Posted by ke...@apache.org.
Generalize extraction of DoFn parameters from context
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/40ff9d40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/40ff9d40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/40ff9d40
Branch: refs/heads/gearpump-runner
Commit: 40ff9d401f0ba3f85d1bab848d1c6a662b03bc99
Parents: ac252a7
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Nov 3 18:42:25 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:03 2016 -0800
----------------------------------------------------------------------
.../sdk/transforms/reflect/DoFnInvokers.java | 115 ++++++++++++-------
.../sdk/transforms/reflect/DoFnSignature.java | 11 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 15 ++-
.../DoFnSignaturesSplittableDoFnTest.java | 5 +-
4 files changed, 94 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/40ff9d40/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
index c5a23dc..ad2b766 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
@@ -69,6 +69,13 @@ import org.apache.beam.sdk.transforms.DoFn.ExtraContextFactory;
import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
import org.apache.beam.sdk.transforms.DoFnAdapters;
import org.apache.beam.sdk.transforms.OldDoFn;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BoundedWindowParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.InputProviderParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.OutputReceiverParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -496,40 +503,76 @@ public class DoFnInvokers {
}
}
+ private static StackManipulation simpleExtraContextParameter(
+ String methodName,
+ StackManipulation pushExtraContextFactory) {
+ try {
+ return new StackManipulation.Compound(
+ pushExtraContextFactory,
+ MethodInvocation.invoke(
+ new MethodDescription.ForLoadedMethod(
+ DoFn.ExtraContextFactory.class.getMethod(methodName))));
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ String.format(
+ "Failed to locate required method %s.%s",
+ ExtraContextFactory.class.getSimpleName(), methodName),
+ e);
+ }
+ }
+
+ private static StackManipulation getExtraContextParameter(
+ DoFnSignature.Parameter parameter,
+ final StackManipulation pushExtraContextFactory) {
+
+ return parameter.match(new Cases<StackManipulation>() {
+
+ @Override
+ public StackManipulation dispatch(BoundedWindowParameter p) {
+ return simpleExtraContextParameter("window", pushExtraContextFactory);
+ }
+
+ @Override
+ public StackManipulation dispatch(InputProviderParameter p) {
+ return simpleExtraContextParameter("inputProvider", pushExtraContextFactory);
+ }
+
+ @Override
+ public StackManipulation dispatch(OutputReceiverParameter p) {
+ return simpleExtraContextParameter("outputReceiver", pushExtraContextFactory);
+ }
+
+ @Override
+ public StackManipulation dispatch(RestrictionTrackerParameter p) {
+ // ExtraContextFactory.restrictionTracker() returns a RestrictionTracker,
+ // but the @ProcessElement method expects a concrete subtype of it.
+ // Insert a downcast.
+ return new StackManipulation.Compound(
+ simpleExtraContextParameter("restrictionTracker", pushExtraContextFactory),
+ TypeCasting.to(new TypeDescription.ForLoadedType(p.trackerT().getRawType())));
+ }
+
+ @Override
+ public StackManipulation dispatch(StateParameter p) {
+ throw new UnsupportedOperationException("State parameters are not yet supported.");
+ }
+
+ @Override
+ public StackManipulation dispatch(TimerParameter p) {
+ throw new UnsupportedOperationException("Timer parameters are not yet supported.");
+ }
+ });
+ }
+
/**
* Implements the invoker's {@link DoFnInvoker#invokeProcessElement} method by delegating to the
* {@link DoFn.ProcessElement} method.
*/
private static final class ProcessElementDelegation extends DoFnMethodDelegation {
- private static final Map<DoFnSignature.Parameter, MethodDescription>
- EXTRA_CONTEXT_FACTORY_METHODS;
private static final MethodDescription PROCESS_CONTINUATION_STOP_METHOD;
static {
try {
- Map<DoFnSignature.Parameter, MethodDescription> methods = new HashMap<>();
- methods.put(
- DoFnSignature.Parameter.boundedWindow(),
- new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod("window")));
- methods.put(
- DoFnSignature.Parameter.inputProvider(),
- new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod("inputProvider")));
- methods.put(
- DoFnSignature.Parameter.outputReceiver(),
- new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod("outputReceiver")));
- methods.put(
- DoFnSignature.Parameter.restrictionTracker(),
- new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod("restrictionTracker")));
- EXTRA_CONTEXT_FACTORY_METHODS = Collections.unmodifiableMap(methods);
- } catch (Exception e) {
- throw new RuntimeException(
- "Failed to locate an ExtraContextFactory method that was expected to exist", e);
- }
- try {
PROCESS_CONTINUATION_STOP_METHOD =
new MethodDescription.ForLoadedMethod(DoFn.ProcessContinuation.class.getMethod("stop"));
} catch (NoSuchMethodException e) {
@@ -537,10 +580,6 @@ public class DoFnInvokers {
}
}
- private static MethodDescription getExtraContextFactoryMethod(DoFnSignature.Parameter param) {
- return EXTRA_CONTEXT_FACTORY_METHODS.get(param);
- }
-
private final DoFnSignature.ProcessElementMethod signature;
/** Implementation of {@link MethodDelegation} for the {@link ProcessElement} method. */
@@ -555,25 +594,15 @@ public class DoFnInvokers {
// DoFn.ProcessContext, ExtraContextFactory.
// Parameters of the wrapped DoFn method:
// DoFn.ProcessContext, [BoundedWindow, InputProvider, OutputReceiver] in any order
- ArrayList<StackManipulation> parameters = new ArrayList<>();
+ ArrayList<StackManipulation> pushParameters = new ArrayList<>();
// Push the ProcessContext argument.
- parameters.add(MethodVariableAccess.REFERENCE.loadOffset(1));
+ pushParameters.add(MethodVariableAccess.REFERENCE.loadOffset(1));
// Push the extra arguments in their actual order.
StackManipulation pushExtraContextFactory = MethodVariableAccess.REFERENCE.loadOffset(2);
for (DoFnSignature.Parameter param : signature.extraParameters()) {
- parameters.add(
- new StackManipulation.Compound(
- pushExtraContextFactory,
- MethodInvocation.invoke(getExtraContextFactoryMethod(param)),
- // ExtraContextFactory.restrictionTracker() returns a RestrictionTracker,
- // but the @ProcessElement method expects a concrete subtype of it.
- // Insert a downcast.
- DoFnSignature.Parameter.restrictionTracker().equals(param)
- ? TypeCasting.to(
- new TypeDescription.ForLoadedType(signature.trackerT().getRawType()))
- : StackManipulation.Trivial.INSTANCE));
+ pushParameters.add(getExtraContextParameter(param, pushExtraContextFactory));
}
- return new StackManipulation.Compound(parameters);
+ return new StackManipulation.Compound(pushParameters);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/40ff9d40/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 11f6aa7..a189bd5 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -34,6 +34,7 @@ import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.DoFn.StateId;
import org.apache.beam.sdk.transforms.DoFn.TimerId;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BoundedWindowParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
@@ -230,8 +231,6 @@ public abstract class DoFnSignature {
// These parameter descriptors are constant
private static final BoundedWindowParameter BOUNDED_WINDOW_PARAMETER =
new AutoValue_DoFnSignature_Parameter_BoundedWindowParameter();
- private static final RestrictionTrackerParameter RESTRICTION_TRACKER_PARAMETER =
- new AutoValue_DoFnSignature_Parameter_RestrictionTrackerParameter();
private static final InputProviderParameter INPUT_PROVIDER_PARAMETER =
new AutoValue_DoFnSignature_Parameter_InputProviderParameter();
private static final OutputReceiverParameter OUTPUT_RECEIVER_PARAMETER =
@@ -261,8 +260,8 @@ public abstract class DoFnSignature {
/**
* Returns a {@link RestrictionTrackerParameter}.
*/
- public static RestrictionTrackerParameter restrictionTracker() {
- return RESTRICTION_TRACKER_PARAMETER;
+ public static RestrictionTrackerParameter restrictionTracker(TypeDescriptor<?> trackerT) {
+ return new AutoValue_DoFnSignature_Parameter_RestrictionTrackerParameter(trackerT);
}
/**
@@ -315,6 +314,7 @@ public abstract class DoFnSignature {
public abstract static class RestrictionTrackerParameter extends Parameter {
// Package visible for AutoValue
RestrictionTrackerParameter() {}
+ public abstract TypeDescriptor<?> trackerT();
}
/**
@@ -388,7 +388,8 @@ public abstract class DoFnSignature {
* Whether this {@link DoFn} is <a href="https://s.apache.org/splittable-do-fn">splittable</a>.
*/
public boolean isSplittable() {
- return extraParameters().contains(Parameter.restrictionTracker());
+ return Iterables.any(
+ extraParameters(), Predicates.instanceOf(RestrictionTrackerParameter.class));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/40ff9d40/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
index 0475404..09c5f3d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignatures.java
@@ -21,7 +21,9 @@ import static com.google.common.base.Preconditions.checkState;
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import java.lang.annotation.Annotation;
import java.lang.reflect.AnnotatedElement;
@@ -45,6 +47,7 @@ import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFn.StateId;
import org.apache.beam.sdk.transforms.DoFn.TimerId;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.StateDeclaration;
@@ -148,6 +151,12 @@ public class DoFnSignatures {
private MethodAnalysisContext() {}
+ /** Indicates whether a {@link RestrictionTrackerParameter} is known in this context. */
+ public boolean hasRestrictionTrackerParameter() {
+ return Iterables.any(
+ extraParameters, Predicates.instanceOf(RestrictionTrackerParameter.class));
+ }
+
/** State parameters declared in this context, keyed by {@link StateId}. */
public Map<String, StateParameter> getStateParameters() {
return Collections.unmodifiableMap(stateParameters);
@@ -663,7 +672,7 @@ public class DoFnSignatures {
}
// A splittable DoFn can not have any other extra context parameters.
- if (methodContext.getExtraParameters().contains(DoFnSignature.Parameter.restrictionTracker())) {
+ if (methodContext.hasRestrictionTrackerParameter()) {
errors.checkArgument(
methodContext.getExtraParameters().size() == 1,
"Splittable DoFn must not have any extra arguments, but has: %s",
@@ -724,10 +733,10 @@ public class DoFnSignatures {
} else if (RestrictionTracker.class.isAssignableFrom(rawType)) {
methodErrors.checkArgument(
- !methodContext.getExtraParameters().contains(Parameter.restrictionTracker()),
+ !methodContext.hasRestrictionTrackerParameter(),
"Multiple %s parameters",
RestrictionTracker.class.getSimpleName());
- return Parameter.restrictionTracker();
+ return Parameter.restrictionTracker(paramT);
} else if (rawType.equals(Timer.class)) {
// m.getParameters() is not available until Java 8
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/40ff9d40/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
index 0751b59..7b93eb9 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesSplittableDoFnTest.java
@@ -22,6 +22,8 @@ import static org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.err
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
import java.util.List;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.KvCoder;
@@ -80,7 +82,8 @@ public class DoFnSignaturesSplittableDoFnTest {
});
assertTrue(signature.isSplittable());
- assertTrue(signature.extraParameters().contains(DoFnSignature.Parameter.restrictionTracker()));
+ assertTrue(Iterables.any(signature.extraParameters(),
+ Predicates.instanceOf(DoFnSignature.Parameter.RestrictionTrackerParameter.class)));
assertEquals(SomeRestrictionTracker.class, signature.trackerT().getRawType());
}
[44/50] incubator-beam git commit: [BEAM-917] ExpectedLogs: clear
saved records after each test.
Posted by ke...@apache.org.
[BEAM-917] ExpectedLogs: clear saved records after each test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6bf729e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6bf729e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6bf729e9
Branch: refs/heads/gearpump-runner
Commit: 6bf729e90c38e910138d332c994325223c220abd
Parents: c6d9bf2
Author: Pei He <pe...@google.com>
Authored: Fri Nov 4 18:45:47 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 17:36:00 2016 -0800
----------------------------------------------------------------------
.../apache/beam/sdk/testing/ExpectedLogs.java | 15 ++++++----
.../beam/sdk/testing/ExpectedLogsTest.java | 30 +++++++++++++++++++-
2 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6bf729e9/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogs.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogs.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogs.java
index a8e3f94..3e51f34 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogs.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogs.java
@@ -268,6 +268,7 @@ public class ExpectedLogs extends ExternalResource {
protected void after() {
log.removeHandler(logSaver);
log.setLevel(previousLevel);
+ logSaver.reset();
}
private final Logger log;
@@ -285,11 +286,7 @@ public class ExpectedLogs extends ExternalResource {
*/
@ThreadSafe
private static class LogSaver extends Handler {
- Collection<LogRecord> logRecords = new ConcurrentLinkedDeque<>();
-
- public Collection<LogRecord> getLogs() {
- return logRecords;
- }
+ private final Collection<LogRecord> logRecords = new ConcurrentLinkedDeque<>();
@Override
public void publish(LogRecord record) {
@@ -301,5 +298,13 @@ public class ExpectedLogs extends ExternalResource {
@Override
public void close() throws SecurityException {}
+
+ private Collection<LogRecord> getLogs() {
+ return logRecords;
+ }
+
+ private void reset() {
+ logRecords.clear();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6bf729e9/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
index 84d5584..1762d0d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/testing/ExpectedLogsTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.testing;
import static org.apache.beam.sdk.testing.SystemNanoTimeSleeper.sleepMillis;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
@@ -30,8 +31,10 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.Description;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.junit.runners.model.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +42,6 @@ import org.slf4j.LoggerFactory;
@RunWith(JUnit4.class)
public class ExpectedLogsTest {
private static final Logger LOG = LoggerFactory.getLogger(ExpectedLogsTest.class);
-
private Random random = new Random();
@Rule public ExpectedLogs expectedLogs = ExpectedLogs.none(ExpectedLogsTest.class);
@@ -146,6 +148,32 @@ public class ExpectedLogsTest {
}
}
+ @Test
+ public void testLogsCleared() throws Throwable {
+ final String messageUnexpected = "Message prior to ExpectedLogs.";
+ final String messageExpected = "Message expected.";
+ LOG.info(messageUnexpected);
+
+ expectedLogs = ExpectedLogs.none(ExpectedLogsTest.class);
+ final boolean[] evaluateRan = new boolean[1];
+
+ expectedLogs.apply(
+ new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ evaluateRan[0] = true;
+ expectedLogs.verifyNotLogged(messageUnexpected);
+ LOG.info(messageExpected);
+ expectedLogs.verifyInfo(messageExpected);
+ }
+ },
+ Description.EMPTY).evaluate();
+ assertTrue(evaluateRan[0]);
+ // Verify expectedLogs is cleared.
+ expectedLogs.verifyNotLogged(messageExpected);
+ expectedLogs.verifyNotLogged(messageUnexpected);
+ }
+
// Generates a random fake error message.
private String generateRandomString() {
return "Fake error message: " + random.nextInt();
[07/50] incubator-beam git commit: Flatten FiredTimers and
ExtractFiredTimers
Posted by ke...@apache.org.
Flatten FiredTimers and ExtractFiredTimers
Pass a single collection of fired timers, and have those objects contain
the associated transform and key that they fired for. Timers already
contain the domain they are in.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5dca2674
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5dca2674
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5dca2674
Branch: refs/heads/gearpump-runner
Commit: 5dca2674a8d145c6e619005c2282c6064cd7aab7
Parents: 6e1e57b
Author: Thomas Groh <tg...@google.com>
Authored: Thu Nov 3 14:10:37 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Nov 4 13:05:21 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/EvaluationContext.java | 6 +-
.../direct/ExecutorServiceParallelExecutor.java | 41 ++++----
.../beam/runners/direct/WatermarkManager.java | 79 ++++++++------
.../runners/direct/EvaluationContextTest.java | 23 ++---
.../runners/direct/WatermarkManagerTest.java | 102 ++++++-------------
5 files changed, 109 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5dca2674/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
index 965e77d..b814def 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/EvaluationContext.java
@@ -391,11 +391,9 @@ class EvaluationContext {
* <p>This is a destructive operation. Timers will only appear in the result of this method once
* for each time they are set.
*/
- public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
+ public Collection<FiredTimers> extractFiredTimers() {
forceRefresh();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
- watermarkManager.extractFiredTimers();
- return fired;
+ return watermarkManager.extractFiredTimers();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5dca2674/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
index e32f671..d1ffea1 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ExecutorServiceParallelExecutor.java
@@ -47,7 +47,6 @@ import org.apache.beam.sdk.transforms.AppliedPTransform;
import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.util.KeyedWorkItem;
import org.apache.beam.sdk.util.KeyedWorkItems;
-import org.apache.beam.sdk.util.TimeDomain;
import org.apache.beam.sdk.util.TimerInternals.TimerData;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
@@ -440,29 +439,23 @@ final class ExecutorServiceParallelExecutor implements PipelineExecutor {
*/
private void fireTimers() throws Exception {
try {
- for (Map.Entry<
- AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> transformTimers :
- evaluationContext.extractFiredTimers().entrySet()) {
- AppliedPTransform<?, ?, ?> transform = transformTimers.getKey();
- for (Map.Entry<StructuralKey<?>, FiredTimers> keyTimers :
- transformTimers.getValue().entrySet()) {
- for (TimeDomain domain : TimeDomain.values()) {
- Collection<TimerData> delivery = keyTimers.getValue().getTimers(domain);
- if (delivery.isEmpty()) {
- continue;
- }
- KeyedWorkItem<?, Object> work =
- KeyedWorkItems.timersWorkItem(keyTimers.getKey().getKey(), delivery);
- @SuppressWarnings({"unchecked", "rawtypes"})
- CommittedBundle<?> bundle =
- evaluationContext
- .createKeyedBundle(keyTimers.getKey(), (PCollection) transform.getInput())
- .add(WindowedValue.valueInGlobalWindow(work))
- .commit(evaluationContext.now());
- scheduleConsumption(transform, bundle, new TimerIterableCompletionCallback(delivery));
- state.set(ExecutorState.ACTIVE);
- }
- }
+ for (FiredTimers transformTimers : evaluationContext.extractFiredTimers()) {
+ Collection<TimerData> delivery = transformTimers.getTimers();
+ KeyedWorkItem<?, Object> work =
+ KeyedWorkItems.timersWorkItem(transformTimers.getKey().getKey(), delivery);
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ CommittedBundle<?> bundle =
+ evaluationContext
+ .createKeyedBundle(
+ transformTimers.getKey(),
+ (PCollection) transformTimers.getTransform().getInput())
+ .add(WindowedValue.valueInGlobalWindow(work))
+ .commit(evaluationContext.now());
+ scheduleConsumption(
+ transformTimers.getTransform(),
+ bundle,
+ new TimerIterableCompletionCallback(delivery));
+ state.set(ExecutorState.ACTIVE);
}
} catch (Exception e) {
LOG.error("Internal Error while delivering timers", e);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5dca2674/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 31b8091..f01c13c 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -740,14 +740,17 @@ public class WatermarkManager {
wms =
new TransformWatermarks(
- inputWatermark, outputWatermark, inputProcessingWatermark, outputProcessingWatermark);
+ transform,
+ inputWatermark,
+ outputWatermark,
+ inputProcessingWatermark,
+ outputProcessingWatermark);
transformToWatermarks.put(transform, wms);
}
return wms;
}
- private Collection<Watermark> getInputProcessingWatermarks(
- AppliedPTransform<?, ?, ?> transform) {
+ private Collection<Watermark> getInputProcessingWatermarks(AppliedPTransform<?, ?, ?> transform) {
ImmutableList.Builder<Watermark> inputWmsBuilder = ImmutableList.builder();
Collection<? extends PValue> inputs = transform.getInput().expand();
if (inputs.isEmpty()) {
@@ -924,15 +927,12 @@ public class WatermarkManager {
* Returns a map of each {@link PTransform} that has pending timers to those timers. All of the
* pending timers will be removed from this {@link WatermarkManager}.
*/
- public Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> extractFiredTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> allTimers = new HashMap<>();
+ public Collection<FiredTimers> extractFiredTimers() {
+ Collection<FiredTimers> allTimers = new ArrayList<>();
for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> watermarksEntry :
transformToWatermarks.entrySet()) {
- Map<StructuralKey<?>, FiredTimers> keyFiredTimers =
- watermarksEntry.getValue().extractFiredTimers();
- if (!keyFiredTimers.isEmpty()) {
- allTimers.put(watermarksEntry.getKey(), keyFiredTimers);
- }
+ Collection<FiredTimers> firedTimers = watermarksEntry.getValue().extractFiredTimers();
+ allTimers.addAll(firedTimers);
}
return allTimers;
}
@@ -1043,6 +1043,8 @@ public class WatermarkManager {
* A reference to the input and output watermarks of an {@link AppliedPTransform}.
*/
public class TransformWatermarks {
+ private final AppliedPTransform<?, ?, ?> transform;
+
private final AppliedPTransformInputWatermark inputWatermark;
private final AppliedPTransformOutputWatermark outputWatermark;
@@ -1053,10 +1055,12 @@ public class WatermarkManager {
private Instant latestSynchronizedOutputWm;
private TransformWatermarks(
+ AppliedPTransform<?, ?, ?> transform,
AppliedPTransformInputWatermark inputWatermark,
AppliedPTransformOutputWatermark outputWatermark,
SynchronizedProcessingTimeInputWatermark inputSynchProcessingWatermark,
SynchronizedProcessingTimeOutputWatermark outputSynchProcessingWatermark) {
+ this.transform = transform;
this.inputWatermark = inputWatermark;
this.outputWatermark = outputWatermark;
@@ -1128,7 +1132,7 @@ public class WatermarkManager {
synchronizedProcessingInputWatermark.addPending(bundle);
}
- private Map<StructuralKey<?>, FiredTimers> extractFiredTimers() {
+ private Collection<FiredTimers> extractFiredTimers() {
Map<StructuralKey<?>, List<TimerData>> eventTimeTimers =
inputWatermark.extractFiredEventTimeTimers();
Map<StructuralKey<?>, List<TimerData>> processingTimers;
@@ -1137,31 +1141,33 @@ public class WatermarkManager {
TimeDomain.PROCESSING_TIME, clock.now());
synchronizedTimers = synchronizedProcessingInputWatermark.extractFiredDomainTimers(
TimeDomain.SYNCHRONIZED_PROCESSING_TIME, getSynchronizedProcessingInputTime());
- Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedTimers = new HashMap<>();
- groupFiredTimers(groupedTimers, eventTimeTimers, processingTimers, synchronizedTimers);
- Map<StructuralKey<?>, FiredTimers> keyFiredTimers = new HashMap<>();
- for (Map.Entry<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> firedTimers :
- groupedTimers.entrySet()) {
- keyFiredTimers.put(firedTimers.getKey(), new FiredTimers(firedTimers.getValue()));
+ Map<StructuralKey<?>, List<TimerData>> timersPerKey =
+ groupFiredTimers(eventTimeTimers, processingTimers, synchronizedTimers);
+ Collection<FiredTimers> keyFiredTimers = new ArrayList<>(timersPerKey.size());
+ for (Map.Entry<StructuralKey<?>, List<TimerData>> firedTimers :
+ timersPerKey.entrySet()) {
+ keyFiredTimers.add(
+ new FiredTimers(transform, firedTimers.getKey(), firedTimers.getValue()));
}
return keyFiredTimers;
}
@SafeVarargs
- private final void groupFiredTimers(
- Map<StructuralKey<?>, Map<TimeDomain, List<TimerData>>> groupedToMutate,
+ private final Map<StructuralKey<?>, List<TimerData>> groupFiredTimers(
Map<StructuralKey<?>, List<TimerData>>... timersToGroup) {
+ Map<StructuralKey<?>, List<TimerData>> groupedTimers = new HashMap<>();
for (Map<StructuralKey<?>, List<TimerData>> subGroup : timersToGroup) {
for (Map.Entry<StructuralKey<?>, List<TimerData>> newTimers : subGroup.entrySet()) {
- Map<TimeDomain, List<TimerData>> grouped = groupedToMutate.get(newTimers.getKey());
+ List<TimerData> grouped = groupedTimers.get(newTimers.getKey());
if (grouped == null) {
- grouped = new HashMap<>();
- groupedToMutate.put(newTimers.getKey(), grouped);
+ grouped = new ArrayList<>();
+ groupedTimers.put(newTimers.getKey(), grouped);
}
- grouped.put(newTimers.getValue().get(0).getDomain(), newTimers.getValue());
+ grouped.addAll(newTimers.getValue());
}
}
+ return groupedTimers;
}
private void updateTimers(TimerUpdate update) {
@@ -1334,24 +1340,35 @@ public class WatermarkManager {
* {@link WatermarkManager}.
*/
public static class FiredTimers {
- private final Map<TimeDomain, ? extends Collection<TimerData>> timers;
+ /** The transform the timers were set at and will be delivered to. */
+ private final AppliedPTransform<?, ?, ?> transform;
+ /** The key the timers were set for and will be delivered to. */
+ private final StructuralKey<?> key;
+ private final Collection<TimerData> timers;
- private FiredTimers(Map<TimeDomain, ? extends Collection<TimerData>> timers) {
+ private FiredTimers(
+ AppliedPTransform<?, ?, ?> transform, StructuralKey<?> key, Collection<TimerData> timers) {
+ this.transform = transform;
+ this.key = key;
this.timers = timers;
}
+ public AppliedPTransform<?, ?, ?> getTransform() {
+ return transform;
+ }
+
+ public StructuralKey<?> getKey() {
+ return key;
+ }
+
/**
* Gets all of the timers that have fired within the provided {@link TimeDomain}. If no timers
* fired within the provided domain, return an empty collection.
*
* <p>Timers within a {@link TimeDomain} are guaranteed to be in order of increasing timestamp.
*/
- public Collection<TimerData> getTimers(TimeDomain domain) {
- Collection<TimerData> domainTimers = timers.get(domain);
- if (domainTimers == null) {
- return Collections.emptyList();
- }
- return domainTimers;
+ public Collection<TimerData> getTimers() {
+ return timers;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5dca2674/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
index bc53570..e1277ac 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/EvaluationContextTest.java
@@ -373,36 +373,31 @@ public class EvaluationContextTest {
.build();
// haven't added any timers, must be empty
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+ assertThat(context.extractFiredTimers(), emptyIterable());
context.handleResult(
context.createKeyedBundle(key, created).commit(Instant.now()),
ImmutableList.<TimerData>of(),
timerResult);
// timer hasn't fired
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+ assertThat(context.extractFiredTimers(), emptyIterable());
TransformResult advanceResult =
StepTransformResult.withoutHold(created.getProducingTransformInternal()).build();
// Should cause the downstream timer to fire
context.handleResult(null, ImmutableList.<TimerData>of(), advanceResult);
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> fired =
- context.extractFiredTimers();
+ Collection<FiredTimers> fired = context.extractFiredTimers();
assertThat(
- fired,
- Matchers.<AppliedPTransform<?, ?, ?>>hasKey(downstream.getProducingTransformInternal()));
- Map<StructuralKey<?>, FiredTimers> downstreamFired =
- fired.get(downstream.getProducingTransformInternal());
- assertThat(downstreamFired, Matchers.<Object>hasKey(key));
+ Iterables.getOnlyElement(fired).getKey(),
+ Matchers.<StructuralKey<?>>equalTo(key));
- FiredTimers firedForKey = downstreamFired.get(key);
- assertThat(firedForKey.getTimers(TimeDomain.PROCESSING_TIME), emptyIterable());
- assertThat(firedForKey.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), emptyIterable());
- assertThat(firedForKey.getTimers(TimeDomain.EVENT_TIME), contains(toFire));
+ FiredTimers firedForKey = Iterables.getOnlyElement(fired);
+ // Contains exclusively the fired timer
+ assertThat(firedForKey.getTimers(), contains(toFire));
// Don't reextract timers
- assertThat(context.extractFiredTimers().entrySet(), emptyIterable());
+ assertThat(context.extractFiredTimers(), emptyIterable());
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5dca2674/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
index 1954005..6bde462 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/WatermarkManagerTest.java
@@ -22,7 +22,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
@@ -68,6 +67,7 @@ import org.apache.beam.sdk.values.TimestampedValue;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
+import org.hamcrest.Matchers;
import org.joda.time.Instant;
import org.joda.time.ReadableInstant;
import org.junit.Before;
@@ -915,12 +915,9 @@ public class WatermarkManagerTest implements Serializable {
filteredDoubledWms.getSynchronizedProcessingOutputTime(),
not(earlierThan(initialFilteredDoubledWm)));
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firedTimers =
- manager.extractFiredTimers();
+ Collection<FiredTimers> firedTimers = manager.extractFiredTimers();
assertThat(
- firedTimers.get(filtered.getProducingTransformInternal())
- .get(key)
- .getTimers(TimeDomain.PROCESSING_TIME),
+ Iterables.getOnlyElement(firedTimers).getTimers(),
contains(pastTimer));
// Our timer has fired, but has not been completed, so it holds our synchronized processing WM
assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
@@ -1099,10 +1096,9 @@ public class WatermarkManagerTest implements Serializable {
@Test
public void extractFiredTimersReturnsFiredEventTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
+ Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
// Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
+ assertThat(initialTimers, emptyIterable());
// Advance WM of keyed past the first timer, but ahead of the second and third
CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
@@ -1136,15 +1132,11 @@ public class WatermarkManagerTest implements Serializable {
new Instant(1000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
+ Collection<FiredTimers> firstFiredTimers =
manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(firstFired.getTimers(TimeDomain.EVENT_TIME), contains(earliestTimer));
+ assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
+ assertThat(firstFired.getTimers(), contains(earliestTimer));
manager.updateWatermarks(null,
TimerUpdate.empty(),
@@ -1153,24 +1145,18 @@ public class WatermarkManagerTest implements Serializable {
Collections.<CommittedBundle<?>>emptyList()),
new Instant(50_000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
+ Collection<FiredTimers> secondFiredTimers = manager.extractFiredTimers();
+ assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
// Contains, in order, middleTimer and then lastTimer
- assertThat(secondFired.getTimers(TimeDomain.EVENT_TIME), contains(middleTimer, lastTimer));
+ assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}
@Test
public void extractFiredTimersReturnsFiredProcessingTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
+ Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
// Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
+ assertThat(initialTimers, emptyIterable());
// Advance WM of keyed past the first timer, but ahead of the second and third
CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
@@ -1204,15 +1190,10 @@ public class WatermarkManagerTest implements Serializable {
new Instant(1000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(firstFired.getTimers(TimeDomain.PROCESSING_TIME), contains(earliestTimer));
+ Collection<FiredTimers> firstFiredTimers = manager.extractFiredTimers();
+ assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
+ assertThat(firstFired.getTimers(), contains(earliestTimer));
clock.set(new Instant(50_000L));
manager.updateWatermarks(null,
@@ -1222,24 +1203,19 @@ public class WatermarkManagerTest implements Serializable {
Collections.<CommittedBundle<?>>emptyList()),
new Instant(50_000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
+ Collection<FiredTimers> secondFiredTimers =
manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
+ assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
// Contains, in order, middleTimer and then lastTimer
- assertThat(secondFired.getTimers(TimeDomain.PROCESSING_TIME), contains(middleTimer, lastTimer));
+ assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}
@Test
public void extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers() {
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> initialTimers =
- manager.extractFiredTimers();
+ Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
// Watermarks haven't advanced
- assertThat(initialTimers.entrySet(), emptyIterable());
+ assertThat(initialTimers, emptyIterable());
// Advance WM of keyed past the first timer, but ahead of the second and third
CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
@@ -1273,16 +1249,11 @@ public class WatermarkManagerTest implements Serializable {
new Instant(1000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> firstTransformFiredTimers =
+ Collection<FiredTimers> firstFiredTimers =
manager.extractFiredTimers();
- assertThat(
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> firstFilteredTimers =
- firstTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(firstFilteredTimers.get(key), not(nullValue()));
- FiredTimers firstFired = firstFilteredTimers.get(key);
- assertThat(
- firstFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME), contains(earliestTimer));
+ assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
+ assertThat(firstFired.getTimers(), contains(earliestTimer));
clock.set(new Instant(50_000L));
manager.updateWatermarks(null,
@@ -1292,18 +1263,11 @@ public class WatermarkManagerTest implements Serializable {
Collections.<CommittedBundle<?>>emptyList()),
new Instant(50_000L));
manager.refreshAll();
- Map<AppliedPTransform<?, ?, ?>, Map<StructuralKey<?>, FiredTimers>> secondTransformFiredTimers =
- manager.extractFiredTimers();
- assertThat(
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal()), not(nullValue()));
- Map<StructuralKey<?>, FiredTimers> secondFilteredTimers =
- secondTransformFiredTimers.get(filtered.getProducingTransformInternal());
- assertThat(secondFilteredTimers.get(key), not(nullValue()));
- FiredTimers secondFired = secondFilteredTimers.get(key);
+ Collection<FiredTimers> secondFiredTimers = manager.extractFiredTimers();
+ assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
+ FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
// Contains, in order, middleTimer and then lastTimer
- assertThat(
- secondFired.getTimers(TimeDomain.SYNCHRONIZED_PROCESSING_TIME),
- contains(middleTimer, lastTimer));
+ assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}
@Test
[33/50] incubator-beam git commit: Update Watermarks even if a Reader
is empty
Posted by ke...@apache.org.
Update Watermarks even if a Reader is empty
This ensures that the pipeline will make progress even if a reader stops
producing elements.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ff7fe07b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ff7fe07b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ff7fe07b
Branch: refs/heads/gearpump-runner
Commit: ff7fe07be96de393b763e7b3d213734040aa3795
Parents: 912500f
Author: Thomas Groh <tg...@google.com>
Authored: Mon Nov 7 12:59:06 2016 -0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 15:08:43 2016 -0800
----------------------------------------------------------------------
.../direct/UnboundedReadEvaluatorFactory.java | 6 ++++--
.../UnboundedReadEvaluatorFactoryTest.java | 19 +++++++++++++------
2 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ff7fe07b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index e529088..fb09b3e 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -35,6 +35,7 @@ import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
import org.apache.beam.sdk.transforms.AppliedPTransform;
+import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
@@ -143,12 +144,13 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
// If the reader had no elements available, but the shard is not done, reuse it later
resultBuilder.addUnprocessedElements(
Collections.<WindowedValue<?>>singleton(
- element.withValue(
+ WindowedValue.timestampedValueInGlobalWindow(
UnboundedSourceShard.of(
shard.getSource(),
shard.getDeduplicator(),
reader,
- shard.getCheckpoint()))));
+ shard.getCheckpoint()),
+ reader.getWatermark())));
}
} catch (IOException e) {
if (reader != null) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ff7fe07b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index 9a7fec3..18c7cec 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -260,6 +260,7 @@ public class UnboundedReadEvaluatorFactoryTest {
(WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>>)
Iterables.getOnlyElement(result.getUnprocessedElements());
secondEvaluator.processElement(residual);
+
TransformResult secondResult = secondEvaluator.finishBundle();
// Sanity check that nothing was output (The test would have to run for more than a day to do
@@ -268,11 +269,14 @@ public class UnboundedReadEvaluatorFactoryTest {
secondOutput.commit(Instant.now()).getElements(),
Matchers.<WindowedValue<Long>>emptyIterable());
- // Test that even though the reader produced no outputs, there is still a residual shard.
- UnboundedSourceShard<Long, TestCheckpointMark> residualShard =
- (UnboundedSourceShard<Long, TestCheckpointMark>)
- Iterables.getOnlyElement(secondResult.getUnprocessedElements()).getValue();
- assertThat(residualShard.getExistingReader(), not(nullValue()));
+ // Test that even though the reader produced no outputs, there is still a residual shard with
+ // the updated watermark.
+ WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>> unprocessed =
+ (WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>>)
+ Iterables.getOnlyElement(secondResult.getUnprocessedElements());
+ assertThat(
+ unprocessed.getTimestamp(), Matchers.<ReadableInstant>greaterThan(residual.getTimestamp()));
+ assertThat(unprocessed.getValue().getExistingReader(), not(nullValue()));
}
@Test
@@ -377,6 +381,8 @@ public class UnboundedReadEvaluatorFactoryTest {
}
private static class TestUnboundedSource<T> extends UnboundedSource<T, TestCheckpointMark> {
+ private static int getWatermarkCalls = 0;
+
static int readerClosedCount;
static int readerAdvancedCount;
private final Coder<T> coder;
@@ -447,7 +453,8 @@ public class UnboundedReadEvaluatorFactoryTest {
@Override
public Instant getWatermark() {
- return Instant.now();
+ getWatermarkCalls++;
+ return new Instant(index + getWatermarkCalls);
}
@Override
[27/50] incubator-beam git commit: Sorter: fix README to support
syntax highlighting
Posted by ke...@apache.org.
Sorter: fix README to support syntax highlighting
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0604d2bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0604d2bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0604d2bb
Branch: refs/heads/gearpump-runner
Commit: 0604d2bb0a666787545b75b1a70ca2fe6496cddc
Parents: baa7fb0
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 12:18:11 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 13:31:47 2016 -0800
----------------------------------------------------------------------
sdks/java/extensions/sorter/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0604d2bb/sdks/java/extensions/sorter/README.md
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/README.md b/sdks/java/extensions/sorter/README.md
index 80d2a40..18bd0d2 100644
--- a/sdks/java/extensions/sorter/README.md
+++ b/sdks/java/extensions/sorter/README.md
@@ -28,7 +28,7 @@ This module provides the SortValues transform, which takes a `PCollection<KV<K,
* The user can customize the temporary location used if sorting requires spilling to disk and the maximum amount of memory to use by creating a custom instance of `BufferedExternalSorter.Options` to pass into `SortValues.create`.
##Using `SortValues`
-~~~~
+```java
PCollection<KV<String, KV<String, Integer>>> input = ...
// Group by primary key, bringing <SecondaryKey, Value> pairs for the same key together.
@@ -39,4 +39,4 @@ PCollection<KV<String, Iterable<KV<String, Integer>>>> grouped =
PCollection<KV<String, Iterable<KV<String, Integer>>>> groupedAndSorted =
grouped.apply(
SortValues.<String, String, Integer>create(new BufferedExternalSorter.Options()));
-~~~~
\ No newline at end of file
+```
[13/50] incubator-beam git commit: Fix javadoc throughout Beam
Posted by ke...@apache.org.
Fix javadoc throughout Beam
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/717b431c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/717b431c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/717b431c
Branch: refs/heads/gearpump-runner
Commit: 717b431c1aeeeec4bad279944c2a1a406b3f08ee
Parents: aae65db
Author: Dan Halperin <dh...@google.com>
Authored: Fri Nov 4 18:28:53 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Sat Nov 5 01:30:40 2016 -0700
----------------------------------------------------------------------
.../beam/examples/complete/game/GameStats.java | 3 +-
.../examples/complete/game/LeaderBoard.java | 2 +-
.../beam/examples/complete/game/UserScore.java | 2 +-
.../flink/examples/streaming/AutoComplete.java | 8 ++---
.../flink/examples/streaming/JoinExamples.java | 12 +++----
.../examples/streaming/KafkaIOExamples.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 8 ++---
.../runners/dataflow/DataflowPipelineJob.java | 1 -
.../runners/dataflow/internal/IsmFormat.java | 5 +--
.../runners/dataflow/util/MonitoringUtil.java | 1 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 4 +--
.../org/apache/beam/sdk/io/BoundedSource.java | 11 +++---
.../apache/beam/sdk/io/CompressedSource.java | 2 --
.../org/apache/beam/sdk/io/FileBasedSink.java | 9 ++---
.../java/org/apache/beam/sdk/io/XmlSource.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../apache/beam/sdk/options/ValueProvider.java | 2 +-
.../apache/beam/sdk/transforms/Aggregator.java | 12 +++----
.../apache/beam/sdk/transforms/CombineFns.java | 4 +--
.../org/apache/beam/sdk/transforms/DoFn.java | 22 ++++++------
.../org/apache/beam/sdk/transforms/ParDo.java | 2 +-
.../org/apache/beam/sdk/transforms/View.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 5 ++-
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 ++--
.../sdk/transforms/reflect/DoFnSignature.java | 2 +-
.../splittabledofn/RestrictionTracker.java | 2 +-
.../transforms/splittabledofn/package-info.java | 4 +--
.../beam/sdk/util/BaseExecutionContext.java | 2 +-
.../BufferedElementCountingOutputStream.java | 4 +--
.../apache/beam/sdk/util/ExecutionContext.java | 8 ++---
.../sdk/util/ExposedByteArrayInputStream.java | 5 +--
.../sdk/util/ExposedByteArrayOutputStream.java | 2 --
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 --
.../apache/beam/sdk/util/MovingFunction.java | 4 +--
.../org/apache/beam/sdk/util/PubsubClient.java | 32 +++---------------
.../org/apache/beam/sdk/util/StringUtils.java | 8 ++---
.../apache/beam/sdk/util/TimerInternals.java | 8 +++--
.../beam/sdk/util/state/StateNamespace.java | 16 ++++-----
.../java/org/apache/beam/sdk/values/PDone.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 2 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +++++++++-----------
42 files changed, 109 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
index 5ebf892..2b5255f 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
@@ -100,7 +100,8 @@ public class GameStats extends LeaderBoard {
/**
* Filter out all but those users with a high clickrate, which we will consider as 'spammy' uesrs.
* We do this by finding the mean total score per user, then using that information as a side
- * input to filter out all but those user scores that are > (mean * SCORE_WEIGHT)
+ * input to filter out all but those user scores that are larger than
+ * {@code (mean * SCORE_WEIGHT)}.
*/
// [START DocInclude_AbuseDetect]
public static class CalculateSpammyUsers
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
index d5e3345..0a3dfb3 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java
@@ -73,7 +73,7 @@ import org.joda.time.format.DateTimeFormatter;
* results, e.g. for 'team prizes'. We're now outputting window results as they're
* calculated, giving us much lower latency than with the previous batch examples.
*
- * <p>Run {@link injector.Injector} to generate pubsub data for this pipeline. The Injector
+ * <p>Run {@code injector.Injector} to generate pubsub data for this pipeline. The Injector
* documentation provides more detail on how to do this.
*
* <p>To execute this pipeline using the Dataflow service, specify the pipeline configuration
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 32c939f..f4dca32 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -68,7 +68,7 @@ import org.slf4j.LoggerFactory;
* where the BigQuery dataset you specify must already exist.
*
* <p>Optionally include the --input argument to specify a batch input file.
- * See the --input default value for example batch data file, or use {@link injector.Injector} to
+ * See the --input default value for example batch data file, or use {@code injector.Injector} to
* generate your own batch data.
*/
public class UserScore {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
index 97ba232..1e0c3ac 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/AutoComplete.java
@@ -53,11 +53,9 @@ import org.joda.time.Duration;
/**
* To run the example, first open a socket on a terminal by executing the command:
- * <li>
- * <li>
- * <code>nc -lk 9999</code>
- * </li>
- * </li>
+ * <ul>
+ * <li><code>nc -lk 9999</code>
+ * </ul>
* and then launch the example. Now whatever you type in the terminal is going to be
* the input to the program.
* */
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
index 96638aa..8fefc9f 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/JoinExamples.java
@@ -39,14 +39,10 @@ import org.joda.time.Duration;
/**
* To run the example, first open two sockets on two terminals by executing the commands:
- * <li>
- * <li>
- * <code>nc -lk 9999</code>, and
- * </li>
- * <li>
- * <code>nc -lk 9998</code>
- * </li>
- * </li>
+ * <ul>
+ * <li><code>nc -lk 9999</code>, and
+ * <li><code>nc -lk 9998</code>
+ * </ul>
* and then launch the example. Now whatever you type in the terminal is going to be
* the input to the program.
* */
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
index f0bf188..3c8a89b 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/KafkaIOExamples.java
@@ -170,7 +170,7 @@ public class KafkaIOExamples {
/**
* Serialiation/Deserialiation schema for Avro types.
- * @param <T>
+ * @param <T> the type being encoded
*/
static class AvroSerializationDeserializationSchema<T>
implements SerializationSchema<T>, DeserializationSchema<T> {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
index 0e250b8..2246bdd 100644
--- a/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
+++ b/runners/flink/examples/src/main/java/org/apache/beam/runners/flink/examples/streaming/WindowedWordCount.java
@@ -42,11 +42,9 @@ import org.slf4j.LoggerFactory;
/**
* To run the example, first open a socket on a terminal by executing the command:
- * <li>
- * <li>
- * <code>nc -lk 9999</code>
- * </li>
- * </li>
+ * <ul>
+ * <li><code>nc -lk 9999</code>
+ * </ul>
* and then launch the example. Now whatever you type in the terminal is going to be
* the input to the program.
* */
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
index 27006a4..a2b632f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java
@@ -199,7 +199,6 @@ public class DataflowPipelineJob implements PipelineResult {
* thread is interrupted.
* @throws IOException If there is a persistent problem getting job
* information.
- * @throws InterruptedException
*/
@Nullable
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
index 903e7b4..2f83ffd 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/IsmFormat.java
@@ -167,11 +167,12 @@ public class IsmFormat {
}
}
- /** A {@link Coder} for {@link IsmRecord}s.
+ /**
+ * A {@link Coder} for {@link IsmRecord}s.
*
* <p>Note that this coder standalone will not produce an Ism file. This coder can be used
* to materialize a {@link PCollection} of {@link IsmRecord}s. Only when this coder
- * is combined with an {@link IsmSink} will one produce an Ism file.
+ * is combined with an {@code IsmSink} will one produce an Ism file.
*
* <p>The {@link IsmRecord} encoded format is:
* <ul>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
index a6afccd..efb6d2b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/MonitoringUtil.java
@@ -149,7 +149,6 @@ public final class MonitoringUtil {
* @param startTimestampMs Return only those messages with a
* timestamp greater than this value.
* @return collection of messages
- * @throws IOException
*/
public ArrayList<JobMessage> getJobMessages(
String jobId, long startTimestampMs) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
index 84c53ea..11eec19 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/util/RandomAccessData.java
@@ -121,7 +121,7 @@ public class RandomAccessData {
* A {@link Comparator} that compares two byte arrays lexicographically. It compares
* values as a list of unsigned bytes. The first pair of values that follow any common prefix,
* or when one array is a prefix of the other, treats the shorter array as the lesser.
- * For example, [] < [0x01] < [0x01, 0x7F] < [0x01, 0x80] < [0x02] < POSITIVE INFINITY.
+ * For example, {@code [] < [0x01] < [0x01, 0x7F] < [0x01, 0x80] < [0x02] < POSITIVE INFINITY}.
*
* <p>Note that a token type of positive infinity is supported and is greater than
* all other {@link RandomAccessData}.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
index 6deca7f..bd0c655 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java
@@ -117,10 +117,10 @@ import org.apache.beam.sdk.values.PDone;
* } </pre>
*
* <p>By default, {@link AvroIO.Write} produces output files that are compressed using the
- * {@link org.apache.avro.file.DeflateCodec CodecFactory.deflateCodec(6)}. This default can
+ * {@link org.apache.avro.file.Codec CodecFactory.deflateCodec(6)}. This default can
* be changed or overridden using {@link AvroIO.Write#withCodec}.
*
- * <p><h3>Permissions</h3>
+ * <h3>Permissions</h3>
* Permission requirements depend on the {@link PipelineRunner} that is used to execute the
* Dataflow job. Please refer to the documentation of corresponding {@link PipelineRunner}s for
* more details.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java
index cd4d7db..b3dd138 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java
@@ -50,9 +50,6 @@ import org.joda.time.Instant;
* </li>
* </ul>
*
- * <p>To use this class for supporting your custom input type, derive your class
- * class from it, and override the abstract methods. For an example, see {@link DatastoreIO}.
- *
* @param <T> Type of records read by the source.
*/
public abstract class BoundedSource<T> extends Source<T> {
@@ -135,7 +132,7 @@ public abstract class BoundedSource<T> extends Source<T> {
*
* <p>By default, returns null to indicate that this cannot be estimated.
*
- * <h5>Thread safety</h5>
+ * <h3>Thread safety</h3>
* If {@link #splitAtFraction} is implemented, this method can be called concurrently to other
* methods (including itself), and it is therefore critical for it to be implemented
* in a thread-safe way.
@@ -345,7 +342,7 @@ public abstract class BoundedSource<T> extends Source<T> {
*
* <p>Returns a {@code BoundedSource} representing the remainder.
*
- * <h5>Detailed description</h5>
+ * <h3>Detailed description</h3>
* Assuming the following sequence of calls:
* <pre>{@code
* BoundedSource<T> initial = reader.getCurrentSource();
@@ -371,11 +368,11 @@ public abstract class BoundedSource<T> extends Source<T> {
* corresponding to the given fraction. In this case, the method MUST have no effect
* (the reader must behave as if the method hadn't been called at all).
*
- * <h5>Statefulness</h5>
+ * <h3>Statefulness</h3>
* Since this method (if successful) affects the reader's source, in subsequent invocations
* "fraction" should be interpreted relative to the new current source.
*
- * <h5>Thread safety and blocking</h5>
+ * <h3>Thread safety and blocking</h3>
* This method will be called concurrently to other methods (however there will not be multiple
* concurrent invocations of this method itself), and it is critical for it to be implemented
* in a thread-safe way (otherwise data loss is possible).
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
index f33b9bd..af0c8130 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
@@ -74,7 +74,6 @@ public class CompressedSource<T> extends FileBasedSource<T> {
public interface DecompressingChannelFactory extends Serializable {
/**
* Given a channel, create a channel that decompresses the content read from the channel.
- * @throws IOException
*/
ReadableByteChannel createDecompressingChannel(ReadableByteChannel channel)
throws IOException;
@@ -88,7 +87,6 @@ public class CompressedSource<T> extends FileBasedSource<T> {
extends DecompressingChannelFactory {
/**
* Given a channel, create a channel that decompresses the content read from the channel.
- * @throws IOException
*/
ReadableByteChannel createDecompressingChannel(String fileName, ReadableByteChannel channel)
throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
index 3d0fe04..101ff61 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileBasedSink.java
@@ -133,7 +133,8 @@ public abstract class FileBasedSink<T> extends Sink<T> {
/**
* The {@link WritableByteChannelFactory} that is used to wrap the raw data output to the
- * underlying channel. The default is to not compress the output using {@link #UNCOMPRESSED}.
+ * underlying channel. The default is to not compress the output using
+ * {@link CompressionType#UNCOMPRESSED}.
*/
protected final WritableByteChannelFactory writableByteChannelFactory;
@@ -206,11 +207,6 @@ public abstract class FileBasedSink<T> extends Sink<T> {
return baseOutputFilename;
}
- /**
- * Perform pipeline-construction-time validation. The default implementation is a no-op.
- * Subclasses should override to ensure the sink is valid and can be written to. It is recommended
- * to use {@link Preconditions#checkState(boolean)} in the implementation of this method.
- */
@Override
public void validate(PipelineOptions options) {}
@@ -802,7 +798,6 @@ public abstract class FileBasedSink<T> extends Sink<T> {
/**
* @param channel the {@link WritableByteChannel} to wrap
* @return the {@link WritableByteChannel} to be used during output
- * @throws IOException
*/
WritableByteChannel create(WritableByteChannel channel) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/XmlSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/XmlSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/XmlSource.java
index aaee75a..1458f74 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/XmlSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/XmlSource.java
@@ -108,7 +108,7 @@ import org.codehaus.stax2.XMLInputFactory2;
* <p>These dependencies have been declared as optional in the sdks/java/core/pom.xml file of
* Apache Beam.
*
- * <p><h3>Permissions</h3>
+ * <h3>Permissions</h3>
* Permission requirements depend on the
* {@link org.apache.beam.sdk.runners.PipelineRunner PipelineRunner} that is
* used to execute the Beam pipeline. Please refer to the documentation of corresponding
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java
index 3063441..e30f8af 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/range/ByteKeyRange.java
@@ -133,7 +133,7 @@ public final class ByteKeyRange implements Serializable {
* Specifically, if this range is unsplittable (e.g., because the start and end keys are equal
* up to padding by zero bytes), the list returned will only contain the start and end key.
*
- * @throws IllegalArgumentException if the specified number of splits is < 1
+ * @throws IllegalArgumentException if the specified number of splits is less than 1
* @see ByteKeyRange the ByteKeyRange class Javadoc for more information about split semantics.
*/
public List<ByteKey> split(int numSplits) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
index 4f0280b..188b371 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/ValueProvider.java
@@ -133,7 +133,7 @@ public interface ValueProvider<T> {
* construction time.
*
* <p>To enforce this contract, if there is no default, users must only call
- * {@link #get()} at execution time (after a call to {@link Pipeline#run}),
+ * {@link #get()} at execution time (after a call to {@link org.apache.beam.sdk.Pipeline#run}),
* which will provide the value of {@code optionsMap}.
*/
class RuntimeValueProvider<T> implements ValueProvider<T>, Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
index 427ecfc..43f53a8 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java
@@ -29,13 +29,11 @@ import org.apache.beam.sdk.util.ExecutionContext;
* typically from the {@link DoFn} constructor. Elements can be added to the
* {@code Aggregator} by calling {@link Aggregator#addValue}.
*
- * <p>Aggregators are visible in the monitoring UI, when the pipeline is run using
- * {@link DataflowRunner} along with their current value.
- * Aggregators may not become visible until the system begins executing the ParDo transform
- * that created them and/or their initial value is changed.
+ * <p>It is runner-dependent whether aggregators are accessible during pipeline execution or only
+ * after jobs have completed.
*
* <p>Example:
- * <pre> {@code
+ * <pre>{@code
* class MyDoFn extends DoFn<String, String> {
* private Aggregator<Integer, Integer> myAggregator;
*
@@ -43,12 +41,12 @@ import org.apache.beam.sdk.util.ExecutionContext;
* myAggregator = createAggregator("myAggregator", new Sum.SumIntegerFn());
* }
*
- * @ProcessElement
+ * {@literal @}ProcessElement
* public void processElement(ProcessContext c) {
* myAggregator.addValue(1);
* }
* }
- * } </pre>
+ * }</pre>
*
* @param <InputT> the type of input values
* @param <OutputT> the type of output values
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
index 1b3e525..e4e1c50 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/CombineFns.java
@@ -106,7 +106,7 @@ public class CombineFns {
* <p>The same {@link TupleTag} cannot be used in a composition multiple times.
*
* <p>Example:
- * <pre><{@code
+ * <pre>{@code
* PCollection<Integer> globalLatencies = ...;
*
* TupleTag<Integer> maxLatencyTag = new TupleTag<Integer>();
@@ -114,7 +114,7 @@ public class CombineFns {
*
* SimpleFunction<Integer, Integer> identityFn =
* new SimpleFunction<Integer, Integer>() {
- * @Override
+ * {@literal @}Override
* public Integer apply(Integer input) {
* return input;
* }};
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 018877f..2b3962e 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -76,15 +76,15 @@ import org.joda.time.Instant;
*
* <p>Example usage:
*
- * <pre> {@code
+ * <pre>{@code
* PCollection<String> lines = ... ;
* PCollection<String> words =
* lines.apply(ParDo.of(new DoFn<String, String>() {
- * @ProcessElement
+ * {@literal @}ProcessElement
* public void processElement(ProcessContext c, BoundedWindow window) {
*
* }}));
- * } </pre>
+ * }</pre>
*
* @param <InputT> the type of the (main) input elements
* @param <OutputT> the type of the (main) output elements
@@ -436,14 +436,14 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
*
* <pre>{@code
* new DoFn<KV<Key, Foo>, Baz>() {
- * @StateId("my-state-id")
+ * {@literal @}StateId("my-state-id")
* private final StateSpec<K, ValueState<MyState>> myStateSpec =
* StateSpecs.value(new MyStateCoder());
*
- * @ProcessElement
+ * {@literal @}ProcessElement
* public void processElement(
* ProcessContext c,
- * @StateId("my-state-id") ValueState<MyState> myState) {
+ * {@literal @}StateId("my-state-id") ValueState<MyState> myState) {
* myState.read();
* myState.write(...);
* }
@@ -480,17 +480,17 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
*
* <pre>{@code
* new DoFn<KV<Key, Foo>, Baz>() {
- * @TimerId("my-timer-id")
+ * {@literal @}TimerId("my-timer-id")
* private final TimerSpec myTimer = TimerSpecs.timerForDomain(TimeDomain.EVENT_TIME);
*
- * @ProcessElement
+ * {@literal @}ProcessElement
* public void processElement(
* ProcessContext c,
- * @TimerId("my-timer-id") Timer myTimer) {
+ * {@literal @}TimerId("my-timer-id") Timer myTimer) {
* myTimer.setForNowPlus(Duration.standardSeconds(...));
* }
*
- * @OnTimer("my-timer-id")
+ * {@literal @}OnTimer("my-timer-id")
* public void onMyTimer() {
* ...
* }
@@ -578,7 +578,7 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
* <ul>
* <li>Its first argument must be a {@link DoFn.ProcessContext}.
* <li>If one of its arguments is a subtype of {@link RestrictionTracker}, then it is a <a
- * href="https://s.apache.org/splittable-do-fn>splittable</a> {@link DoFn} subject to the
+ * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn} subject to the
* separate requirements described below. Items below are assuming this is not a splittable
* {@link DoFn}.
* <li>If one of its arguments is {@link BoundedWindow}, this argument corresponds to the window
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
index a3a306a..0684a5c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/ParDo.java
@@ -96,7 +96,7 @@ import org.apache.beam.sdk.values.TypedPValue;
*
* <p>For example:
*
- * <pre><{@code
+ * <pre>{@code
* PCollection<String> lines = ...;
* PCollection<String> words =
* lines.apply(ParDo.of(new DoFn<String, String>() {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
index d750efc..5fafc0a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/View.java
@@ -213,7 +213,7 @@ public class View {
/**
* Returns a {@link View.AsMultimap} transform that takes a
- * {@link PCollection PCollection<KV<K, V>>}
+ * {@link PCollection PCollection<KV<K, V>>}
* as input and produces a {@link PCollectionView} mapping
* each window to its contents as a {@link Map Map<K, Iterable<V>>}
* for use as a side input.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
index 46cf94a..1a2de97 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/display/DisplayData.java
@@ -87,9 +87,8 @@ public class DisplayData implements Serializable {
*
* <p>Use this method if the type of metadata is not known at compile time. For example:
*
- * <pre>
- * {@code
- * @Override
+ * <pre>{@code
+ * {@literal @}Override
* public void populateDisplayData(DisplayData.Builder builder) {
* Optional<DisplayData.Type> type = DisplayData.inferType(foo);
* if (type.isPresent()) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGroupByKey.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGroupByKey.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGroupByKey.java
index 9546b29..b760e2c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGroupByKey.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/join/CoGroupByKey.java
@@ -43,7 +43,7 @@ import org.apache.beam.sdk.values.PCollectionList;
* <p>Example of performing a {@link CoGroupByKey} followed by a
* {@link ParDo} that consumes
* the results:
- * <pre> {@code
+ * <pre>{@code
* PCollection<KV<K, V1>> pt1 = ...;
* PCollection<KV<K, V2>> pt2 = ...;
*
@@ -57,7 +57,7 @@ import org.apache.beam.sdk.values.PCollectionList;
* PCollection<T> finalResultCollection =
* coGbkResultCollection.apply(ParDo.of(
* new DoFn<KV<K, CoGbkResult>, T>() {
- * @Override
+ * {@literal @}ProcessElement
* public void processElement(ProcessContext c) {
* KV<K, CoGbkResult> e = c.element();
* Iterable<V1> pt1Vals = e.getValue().getAll(t1);
@@ -66,7 +66,7 @@ import org.apache.beam.sdk.values.PCollectionList;
* c.output(...some T...);
* }
* }));
- * } </pre>
+ * }</pre>
*
* @param <K> the type of the keys in the input and output
* {@code PCollection}s
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index 7087efa..11f6aa7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -415,7 +415,7 @@ public abstract class DoFnSignature {
/**
* Describes a timer declaration; a field of type {@link TimerSpec} annotated with
- * {@DoFn.TimerId}.
+ * {@link DoFn.TimerId}.
*/
@AutoValue
public abstract static class TimerDeclaration {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
index 6b249ee..268766b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/RestrictionTracker.java
@@ -21,7 +21,7 @@ import org.apache.beam.sdk.transforms.DoFn;
/**
* Manages concurrent access to the restriction and keeps track of its claimed part for a <a
- * href="https://s.apache.org/splittable-do-fn>splittable</a> {@link DoFn}.
+ * href="https://s.apache.org/splittable-do-fn">splittable</a> {@link DoFn}.
*/
public interface RestrictionTracker<RestrictionT> {
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
index 1ceb880..4523032 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/splittabledofn/package-info.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
/**
- * Defines utilities related to <a href="https://s.apache.org/splittable-do-fn>splittable</a>
- * {@link org.apache.beam.sdk.transforms.DoFn}'s.
+ * Defines utilities related to <a href="https://s.apache.org/splittable-do-fn">splittable</a>
+ * {@link org.apache.beam.sdk.transforms.DoFn}.
*/
package org.apache.beam.sdk.transforms.splittabledofn;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
index 45bbe75..e26f2b0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BaseExecutionContext.java
@@ -38,7 +38,7 @@ import org.apache.beam.sdk.values.TupleTag;
* of {@link StepContext} from {@link #getOrCreateStepContext(String, String)} and
* {@link #getAllStepContexts()} without forcing each subclass to override the method, e.g.
* <pre>{@code
- * @Override
+ * {@literal @}Override
* StreamingModeExecutionContext.StepContext getOrCreateStepContext(...) {
* return (StreamingModeExecutionContext.StepContext) super.getOrCreateStepContext(...);
* }
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java
index c17d92d..b0784ca 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BufferedElementCountingOutputStream.java
@@ -49,7 +49,7 @@ import org.apache.beam.sdk.coders.Coder.Context;
* </pre>
*
* <p>To read this stream:
- * <pre><code>
+ * <pre>{@code
* InputStream is = ...
* long count;
* do {
@@ -58,7 +58,7 @@ import org.apache.beam.sdk.coders.Coder.Context;
* // read an element from is
* }
* } while(count > 0);
- * </code></pre>
+ * }</pre>
*
* <p>The counts are encoded as variable length longs. See {@link VarInt#encode(long, OutputStream)}
* for more details. The end of the iterable is detected by reading a count of 0.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutionContext.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutionContext.java
index 82d900c..f2a79bd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutionContext.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExecutionContext.java
@@ -41,14 +41,14 @@ public interface ExecutionContext {
/**
* Hook for subclasses to implement that will be called whenever
- * {@link OldDoFn.Context#output}
+ * {@link org.apache.beam.sdk.transforms.OldDoFn.Context#output}
* is called.
*/
void noteOutput(WindowedValue<?> output);
/**
* Hook for subclasses to implement that will be called whenever
- * {@link OldDoFn.Context#sideOutput}
+ * {@link org.apache.beam.sdk.transforms.OldDoFn.Context#sideOutput}
* is called.
*/
void noteSideOutput(TupleTag<?> tag, WindowedValue<?> output);
@@ -70,14 +70,14 @@ public interface ExecutionContext {
/**
* Hook for subclasses to implement that will be called whenever
- * {@link OldDoFn.Context#output}
+ * {@link org.apache.beam.sdk.transforms.OldDoFn.Context#output}
* is called.
*/
void noteOutput(WindowedValue<?> output);
/**
* Hook for subclasses to implement that will be called whenever
- * {@link OldDoFn.Context#sideOutput}
+ * {@link org.apache.beam.sdk.transforms.OldDoFn.Context#sideOutput}
* is called.
*/
void noteSideOutput(TupleTag<?> tag, WindowedValue<?> output);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java
index 020c453..662ea38 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayInputStream.java
@@ -30,8 +30,9 @@ public class ExposedByteArrayInputStream extends ByteArrayInputStream{
super(buf);
}
- /** Read all remaining bytes.
- * @throws IOException */
+ /**
+ * Read all remaining bytes.
+ */
@SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "Returns internal buffer by design")
public byte[] readAll() throws IOException {
if (pos == 0 && count == buf.length) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStream.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStream.java
index e2c7e42..e4a8062 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStream.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/ExposedByteArrayOutputStream.java
@@ -62,8 +62,6 @@ public class ExposedByteArrayOutputStream extends ByteArrayOutputStream {
* no content copy will be involved.
*
* <p><i>Note: After passing any byte array to this method, it must not be modified again.</i>
- *
- * @throws IOException
*/
@SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "Takes ownership of input buffer")
public void writeAndOwn(byte[] b) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index ce4604b..54a963b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -301,7 +301,6 @@ public class GcsUtil {
*
* @param path the GCS filename to read from
* @return a SeekableByteChannel that can read the object data
- * @throws IOException
*/
public SeekableByteChannel open(GcsPath path)
throws IOException {
@@ -319,7 +318,6 @@ public class GcsUtil {
* @param path the GCS file to write to
* @param type the type of object, eg "text/plain".
* @return a Callable object that encloses the operation.
- * @throws IOException
*/
public WritableByteChannel create(GcsPath path,
String type) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java
index 7b30d34..a8239ce 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/MovingFunction.java
@@ -25,8 +25,8 @@ import org.apache.beam.sdk.transforms.Combine;
/**
* Keep track of the moving minimum/maximum/sum of sampled long values. The minimum/maximum/sum
- * is over at most the last {@link #samplePeriodMs}, and is updated every
- * {@link #sampleUpdateMs}.
+ * is over at most the user-specified last {@code samplePeriodMs}, and is updated every
+ * {@code sampleUpdateMs}.
*/
public class MovingFunction {
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubClient.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubClient.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubClient.java
index 1ac5511..06b776b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubClient.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/PubsubClient.java
@@ -90,10 +90,10 @@ public abstract class PubsubClient implements Closeable {
* <p>If {@code timestampLabel} is non-{@literal null} then the message attributes must contain
* that label, and the value of that label will be taken as the timestamp.
* Otherwise the timestamp will be taken from the Pubsub publish timestamp {@code
- * pubsubTimestamp}. Throw {@link IllegalArgumentException} if the timestamp cannot be
- * recognized as a ms-since-unix-epoch or RFC3339 time.
+ * pubsubTimestamp}.
*
- * @throws IllegalArgumentException
+ * @throws IllegalArgumentException if the timestamp cannot be recognized as a ms-since-unix-epoch
+ * or RFC3339 time.
*/
protected static long extractTimestamp(
@Nullable String timestampLabel,
@@ -115,7 +115,7 @@ public abstract class PubsubClient implements Closeable {
"Cannot interpret value of label %s as timestamp: %s",
timestampLabel, value);
}
- return timestampMsSinceEpoch == null ? 0 : timestampMsSinceEpoch;
+ return timestampMsSinceEpoch;
}
/**
@@ -443,8 +443,6 @@ public abstract class PubsubClient implements Closeable {
/**
* Publish {@code outgoingMessages} to Pubsub {@code topic}. Return number of messages
* published.
- *
- * @throws IOException
*/
public abstract int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages)
throws IOException;
@@ -454,8 +452,6 @@ public abstract class PubsubClient implements Closeable {
* Return the received messages, or empty collection if none were available. Does not
* wait for messages to arrive if {@code returnImmediately} is {@literal true}.
* Returned messages will record their request time as {@code requestTimeMsSinceEpoch}.
- *
- * @throws IOException
*/
public abstract List<IncomingMessage> pull(
long requestTimeMsSinceEpoch,
@@ -466,8 +462,6 @@ public abstract class PubsubClient implements Closeable {
/**
* Acknowldege messages from {@code subscription} with {@code ackIds}.
- *
- * @throws IOException
*/
public abstract void acknowledge(SubscriptionPath subscription, List<String> ackIds)
throws IOException;
@@ -475,8 +469,6 @@ public abstract class PubsubClient implements Closeable {
/**
* Modify the ack deadline for messages from {@code subscription} with {@code ackIds} to
* be {@code deadlineSeconds} from now.
- *
- * @throws IOException
*/
public abstract void modifyAckDeadline(
SubscriptionPath subscription, List<String> ackIds,
@@ -484,29 +476,21 @@ public abstract class PubsubClient implements Closeable {
/**
* Create {@code topic}.
- *
- * @throws IOException
*/
public abstract void createTopic(TopicPath topic) throws IOException;
/*
* Delete {@code topic}.
- *
- * @throws IOException
*/
public abstract void deleteTopic(TopicPath topic) throws IOException;
/**
* Return a list of topics for {@code project}.
- *
- * @throws IOException
*/
public abstract List<TopicPath> listTopics(ProjectPath project) throws IOException;
/**
* Create {@code subscription} to {@code topic}.
- *
- * @throws IOException
*/
public abstract void createSubscription(
TopicPath topic, SubscriptionPath subscription, int ackDeadlineSeconds) throws IOException;
@@ -514,8 +498,6 @@ public abstract class PubsubClient implements Closeable {
/**
* Create a random subscription for {@code topic}. Return the {@link SubscriptionPath}. It
* is the responsibility of the caller to later delete the subscription.
- *
- * @throws IOException
*/
public SubscriptionPath createRandomSubscription(
ProjectPath project, TopicPath topic, int ackDeadlineSeconds) throws IOException {
@@ -529,23 +511,17 @@ public abstract class PubsubClient implements Closeable {
/**
* Delete {@code subscription}.
- *
- * @throws IOException
*/
public abstract void deleteSubscription(SubscriptionPath subscription) throws IOException;
/**
* Return a list of subscriptions for {@code topic} in {@code project}.
- *
- * @throws IOException
*/
public abstract List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic)
throws IOException;
/**
* Return the ack deadline, in seconds, for {@code subscription}.
- *
- * @throws IOException
*/
public abstract int ackDeadlineSeconds(SubscriptionPath subscription) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java
index 1c52c1f..3ff8448 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/StringUtils.java
@@ -119,8 +119,8 @@ public class StringUtils {
*
* <p>Examples:
* <ul>
- * <li>{@code some.package.Word.SummaryDoFn} -> "Summary"
- * <li>{@code another.package.PairingFn} -> "Pairing"
+ * <li>{@code some.package.Word.SummaryDoFn} becomes "Summary"
+ * <li>{@code another.package.PairingFn} becomes "Pairing"
* </ul>
*
* @throws IllegalArgumentException if the class is anonymous
@@ -143,8 +143,8 @@ public class StringUtils {
*
* <p>Examples:
* <ul>
- * <li>{@code some.package.Word.Summary} -> "Word.Summary"
- * <li>{@code another.package.Pairing.Bound} -> "Pairing"
+ * <li>{@code some.package.Word.Summary} becomes "Word.Summary"
+ * <li>{@code another.package.Pairing.Bound} becomes "Pairing"
* </ul>
*/
public static String approximatePTransformName(Class<?> clazz) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
index 8015116..743f3f7 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/TimerInternals.java
@@ -128,15 +128,17 @@ public interface TimerInternals {
* </ol>
*
* <p>In pictures:
- * <pre>
+ * <pre>{@code
* | | | | |
* | | D | C | B | A
* | | | | |
* GIWM <= GOWM <= LOWM <= LIWM <= GIWM
* (next stage)
* -------------------------------------------------> event time
- * </pre>
- * where
+ * }</pre>
+ *
+ * <p>where
+ *
* <ul>
* <li> LOWM = local output water mark.
* <li> GOWM = global output water mark.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateNamespace.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateNamespace.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateNamespace.java
index 9b27b1d..04440cb 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateNamespace.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/state/StateNamespace.java
@@ -22,17 +22,17 @@ import java.io.IOException;
/**
* A namespace used for scoping state stored with {@link StateInternals}.
*
- * <p>Instances of {@code StateNamespace} are guaranteed to have a {@link #hashCode} and
- * {@link #equals} that uniquely identify the namespace.
+ * <p>Instances of {@link StateNamespace} are guaranteed to have a {@link Object#hashCode} and
+ * {@link Object#equals} that uniquely identify the namespace.
*/
public interface StateNamespace {
/**
* Return a {@link String} representation of the key. It is guaranteed that this
- * {@code String} will uniquely identify the key.
+ * {@link String} will uniquely identify the key.
*
- * <p>This will encode the actual namespace as a {@code String}. It is
- * preferable to use the {@code StateNamespace} object when possible.
+ * <p>This will encode the actual namespace as a {@link String}. It is
+ * preferable to use the {@link StateNamespace} object when possible.
*
* <p>The string produced by the standard implementations will not contain a '+' character. This
* enables adding a '+' between the actual namespace and other information, if needed, to separate
@@ -46,11 +46,11 @@ public interface StateNamespace {
void appendTo(Appendable sb) throws IOException;
/**
- * Return an {@code Object} to use as a key in a cache.
+ * Return an {@link Object} to use as a key in a cache.
*
* <p>Different namespaces may use the same key in order to be treated as a unit in the cache.
- * The {@code Object}'s {@code hashCode} and {@code equals} methods will be used to determine
- * equality.
+ * The {@link Object}'s {@link Object#hashCode} and {@link Object#equals} methods will be used to
+ * determine equality.
*/
Object getCacheKey();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
index 83d6a92..9e8cae4 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/values/PDone.java
@@ -24,7 +24,7 @@ import org.apache.beam.sdk.transforms.PTransform;
/**
* {@link PDone} is the output of a {@link PTransform} that has a trivial result,
- * such as a {@link Write}.
+ * such as a {@link org.apache.beam.sdk.io.Write}.
*/
public class PDone extends POutputValueBase {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index 3727f92..1d1075c 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -165,7 +165,7 @@ import org.slf4j.LoggerFactory;
* read <a href="https://cloud.google.com/datastore/docs/concepts/entities">Entities, Properties,
* and Keys</a> for more information about {@code Entity} keys.
*
- * <p><h3>Permissions</h3>
+ * <h3>Permissions</h3>
* Permission requirements depend on the {@code PipelineRunner} that is used to execute the
* Dataflow job. Please refer to the documentation of corresponding {@code PipelineRunner}s for
* more details.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/717b431c/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
index 945eff6..91939d4 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisIO.java
@@ -40,32 +40,29 @@ import org.joda.time.Instant;
* It follows the usage conventions laid out by other *IO classes like
* BigQueryIO or PubsubIOLet's see how you can set up a simple Pipeline, which reads from Kinesis:
*
- * <pre>{@code}
+ * <pre>{@code
* p.
* apply(KinesisIO.Read.
* from("streamName", InitialPositionInStream.LATEST).
* using("AWS_KEY", _"AWS_SECRET", STREAM_REGION).
* apply( ... ) // other transformations
- *</pre>
- * </p>
+ * }</pre>
*
- * <p>
- * As you can see you need to provide 3 things:
+ * <p>As you can see you need to provide 3 things:
* <ul>
* <li>name of the stream you're going to read</li>
- * <li>position in the stream where reading should start. There are two options:</li>
+ * <li>position in the stream where reading should start. There are two options:
* <ul>
* <li>{@link InitialPositionInStream#LATEST} - reading will begin from end of the stream</li>
* <li>{@link InitialPositionInStream#TRIM_HORIZON} - reading will begin at
* the very beginning of the stream</li>
- * </ul>
- * <li>data used to initialize {@link AmazonKinesis} client</li>
+ * </ul></li>
+ * <li>data used to initialize {@link AmazonKinesis} client:
* <ul>
* <li>credentials (aws key, aws secret)</li>
* <li>region where the stream is located</li>
- * </ul>
+ * </ul></li>
* </ul>
- * </p>
*
* <p>In case when you want to set up {@link AmazonKinesis} client by your own
* (for example if you're using more sophisticated authorization methods like Amazon STS, etc.)
@@ -73,35 +70,33 @@ import org.joda.time.Instant;
*
* <pre>{@code
* public class MyCustomKinesisClientProvider implements KinesisClientProvider {
- * @Override
+ * {@literal @}Override
* public AmazonKinesis get() {
* // set up your client here
* }
- * }}
- * </pre>
+ * }
+ * }</pre>
*
- * Usage is pretty straightforward:
+ * <p>Usage is pretty straightforward:
*
- * <pre>{@code}
+ * <pre>{@code
* p.
* apply(KinesisIO.Read.
* from("streamName", InitialPositionInStream.LATEST).
* using(MyCustomKinesisClientProvider()).
* apply( ... ) // other transformations
- * </pre>
- * </p>
+ * }</pre>
*
* <p>There\u2019s also possibility to start reading using arbitrary point in time -
* in this case you need to provide {@link Instant} object:
*
- * <pre>{@code}
+ * <pre>{@code
* p.
* apply(KinesisIO.Read.
* from("streamName", instant).
* using(MyCustomKinesisClientProvider()).
* apply( ... ) // other transformations
- * </pre>
- * </p>
+ * }</pre>
*
*/
public final class KinesisIO {
[29/50] incubator-beam git commit: [BEAM-929] Fix Findbugs issues in
Kinesis
Posted by ke...@apache.org.
[BEAM-929] Fix Findbugs issues in Kinesis
* Fix equals and hashcode
* Add tests
* Remove Serializable from KinesisRecord, as it is in fact coded not serialized
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/367fcac6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/367fcac6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/367fcac6
Branch: refs/heads/gearpump-runner
Commit: 367fcac68b38cc613525815181707a071eb8a51c
Parents: 9b47228
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 12:31:19 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 13:33:04 2016 -0800
----------------------------------------------------------------------
sdks/java/io/kinesis/pom.xml | 21 ++++++--------------
.../beam/sdk/io/kinesis/CustomOptional.java | 21 ++++++++++++++++----
.../beam/sdk/io/kinesis/KinesisRecord.java | 3 +--
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 ++++++++++
4 files changed, 34 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/367fcac6/sdks/java/io/kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index e0b57db..36c7039 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -30,19 +30,6 @@
<description>Library to read Kinesis streams.</description>
<build>
- <pluginManagement>
- <plugins>
- <!-- BEAM-929 -->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
-
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -139,7 +126,6 @@
<artifactId>annotations</artifactId>
</dependency>
-
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -160,6 +146,12 @@
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava-testlib</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
@@ -171,6 +163,5 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/367fcac6/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
index 4317a59..4bed0e3 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/CustomOptional.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.kinesis;
import java.util.NoSuchElementException;
+import java.util.Objects;
/**
* Similar to Guava {@code Optional}, but throws {@link NoSuchElementException} for missing element.
@@ -53,17 +54,19 @@ abstract class CustomOptional<T> {
return value;
}
-
@Override
public boolean equals(Object o) {
- Present<?> present = (Present<?>) o;
+ if (!(o instanceof Present)) {
+ return false;
+ }
- return value != null ? value.equals(present.value) : present.value == null;
+ Present<?> present = (Present<?>) o;
+ return Objects.equals(value, present.value);
}
@Override
public int hashCode() {
- return value != null ? value.hashCode() : 0;
+ return Objects.hash(value);
}
}
@@ -82,5 +85,15 @@ abstract class CustomOptional<T> {
public T get() {
throw new NoSuchElementException();
}
+
+ @Override
+ public boolean equals(Object o) {
+ return o instanceof Absent;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/367fcac6/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
index fe2a33d..02b5370 100644
--- a/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
+++ b/sdks/java/io/kinesis/src/main/java/org/apache/beam/sdk/io/kinesis/KinesisRecord.java
@@ -22,7 +22,6 @@ import static org.apache.commons.lang.builder.HashCodeBuilder.reflectionHashCode
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord;
import com.google.common.base.Charsets;
-import java.io.Serializable;
import java.nio.ByteBuffer;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.joda.time.Instant;
@@ -30,7 +29,7 @@ import org.joda.time.Instant;
/**
* {@link UserRecord} enhanced with utility methods.
*/
-public class KinesisRecord implements Serializable {
+public class KinesisRecord {
private Instant readTime;
private String streamName;
private String shardId;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/367fcac6/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
index 20e8372..00acffe 100644
--- a/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
+++ b/sdks/java/io/kinesis/src/test/java/org/apache/beam/sdk/io/kinesis/CustomOptionalTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.io.kinesis;
+import com.google.common.testing.EqualsTester;
import java.util.NoSuchElementException;
import org.junit.Test;
@@ -28,4 +29,13 @@ public class CustomOptionalTest {
public void absentThrowsNoSuchElementExceptionOnGet() {
CustomOptional.absent().get();
}
+
+ @Test
+ public void testEqualsAndHashCode() {
+ new EqualsTester()
+ .addEqualityGroup(CustomOptional.absent(), CustomOptional.absent())
+ .addEqualityGroup(CustomOptional.of(3), CustomOptional.of(3))
+ .addEqualityGroup(CustomOptional.of(11))
+ .addEqualityGroup(CustomOptional.of("3")).testEquals();
+ }
}
[03/50] incubator-beam git commit: Closes #1285
Posted by ke...@apache.org.
Closes #1285
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f06deac8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f06deac8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f06deac8
Branch: refs/heads/gearpump-runner
Commit: f06deac8c10f10b4b64ddc51358b1b551845e595
Parents: faf55c7 f50b214
Author: Dan Halperin <dh...@google.com>
Authored: Fri Nov 4 10:51:29 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Nov 4 10:51:29 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 5 -----
examples/java8/pom.xml | 5 -----
examples/pom.xml | 13 ++++++++++++-
pom.xml | 3 +++
runners/core-java/pom.xml | 5 -----
runners/direct-java/pom.xml | 5 -----
runners/flink/examples/pom.xml | 5 -----
runners/flink/runner/pom.xml | 5 -----
runners/google-cloud-dataflow-java/pom.xml | 7 -------
runners/pom.xml | 11 +++++++++++
runners/spark/pom.xml | 4 ----
sdks/java/build-tools/pom.xml | 4 ++--
.../src/main/resources/beam/checkstyle.xml | 2 +-
sdks/java/core/pom.xml | 18 +++++++++---------
sdks/java/extensions/join-library/pom.xml | 4 ----
sdks/java/extensions/sorter/pom.xml | 4 ----
sdks/java/io/google-cloud-platform/pom.xml | 4 ----
sdks/java/io/hdfs/pom.xml | 4 ----
sdks/java/io/jdbc/pom.xml | 6 +-----
sdks/java/io/jms/pom.xml | 4 ----
sdks/java/io/kafka/pom.xml | 4 ----
sdks/java/io/kinesis/pom.xml | 4 ----
sdks/java/io/mongodb/pom.xml | 4 ----
sdks/java/java8tests/pom.xml | 5 -----
sdks/java/microbenchmarks/pom.xml | 5 -----
sdks/java/pom.xml | 4 +++-
sdks/pom.xml | 16 +++++++++++++++-
27 files changed, 57 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
[42/50] incubator-beam git commit: Track Minimum Element Timestamps
within Bundles
Posted by ke...@apache.org.
Track Minimum Element Timestamps within Bundles
This allows the Watermark Manager to track pending elements by bundles
of elements rather than per-element, which significantly reduces the
amount of work done per-element to track watermarks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a58f1eba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a58f1eba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a58f1eba
Branch: refs/heads/gearpump-runner
Commit: a58f1eba1a3b5ae453f4d65ad409785cb717b2ae
Parents: 317b5e6
Author: Thomas Groh <tg...@google.com>
Authored: Fri Nov 4 12:54:00 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 15:47:02 2016 -0800
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 5 ++
.../direct/ImmutableListBundleFactory.java | 21 ++++++++-
.../beam/runners/direct/WatermarkManager.java | 48 +++++++++-----------
.../direct/ImmutableListBundleFactoryTest.java | 15 +++++-
4 files changed, 60 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a58f1eba/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
index 44d1986..4d5a449 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/DirectRunner.java
@@ -151,6 +151,11 @@ public class DirectRunner
Iterable<WindowedValue<T>> getElements();
/**
+ * Returns the minimum timestamp among all of the elements of this {@link CommittedBundle}.
+ */
+ Instant getMinTimestamp();
+
+ /**
* Returns the processing time output watermark at the time the producing {@link PTransform}
* committed this bundle. Downstream synchronized processing time watermarks cannot progress
* past this point before consuming this bundle.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a58f1eba/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
index abc6dd8..6b342d6 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/ImmutableListBundleFactory.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList;
import javax.annotation.Nullable;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.runners.direct.DirectRunner.UncommittedBundle;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.joda.time.Instant;
@@ -64,6 +65,7 @@ class ImmutableListBundleFactory implements BundleFactory {
private final StructuralKey<?> key;
private boolean committed = false;
private ImmutableList.Builder<WindowedValue<T>> elements;
+ private Instant minSoFar = BoundedWindow.TIMESTAMP_MAX_VALUE;
/**
* Create a new {@link UncommittedImmutableListBundle} for the specified {@link PCollection}.
@@ -93,6 +95,9 @@ class ImmutableListBundleFactory implements BundleFactory {
element,
pcollection);
elements.add(element);
+ if (element.getTimestamp().isBefore(minSoFar)) {
+ minSoFar = element.getTimestamp();
+ }
return this;
}
@@ -102,7 +107,7 @@ class ImmutableListBundleFactory implements BundleFactory {
committed = true;
final Iterable<WindowedValue<T>> committedElements = elements.build();
return CommittedImmutableListBundle.create(
- pcollection, key, committedElements, synchronizedCompletionTime);
+ pcollection, key, committedElements, minSoFar, synchronizedCompletionTime);
}
}
@@ -112,9 +117,10 @@ class ImmutableListBundleFactory implements BundleFactory {
@Nullable PCollection<T> pcollection,
StructuralKey<?> key,
Iterable<WindowedValue<T>> committedElements,
+ Instant minElementTimestamp,
Instant synchronizedCompletionTime) {
return new AutoValue_ImmutableListBundleFactory_CommittedImmutableListBundle<>(
- pcollection, key, committedElements, synchronizedCompletionTime);
+ pcollection, key, committedElements, minElementTimestamp, synchronizedCompletionTime);
}
@Override
@@ -123,6 +129,7 @@ class ImmutableListBundleFactory implements BundleFactory {
getPCollection(),
getKey(),
ImmutableList.copyOf(elements),
+ minTimestamp(elements),
getSynchronizedProcessingOutputWatermark());
}
@@ -136,4 +143,14 @@ class ImmutableListBundleFactory implements BundleFactory {
return this == obj;
}
}
+
+ private static Instant minTimestamp(Iterable<? extends WindowedValue<?>> elements) {
+ Instant minTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
+ for (WindowedValue<?> element : elements) {
+ if (element.getTimestamp().isBefore(minTs)) {
+ minTs = element.getTimestamp();
+ }
+ }
+ return minTs;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a58f1eba/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index 2228cd5..f235af0 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -27,8 +27,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
-import com.google.common.collect.SortedMultiset;
-import com.google.common.collect.TreeMultiset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -55,7 +53,6 @@ import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.TimeDomain;
import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.TimerInternals.TimerData;
-import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PValue;
import org.joda.time.Instant;
@@ -123,6 +120,10 @@ import org.joda.time.Instant;
* </pre>
*/
public class WatermarkManager {
+ // The number of updates to apply in #tryApplyPendingUpdates
+ private static final int MAX_INCREMENTAL_UPDATES = 10;
+
+
/**
* The watermark of some {@link Pipeline} element, usually a {@link PTransform} or a
* {@link PCollection}.
@@ -203,7 +204,7 @@ public class WatermarkManager {
*/
private static class AppliedPTransformInputWatermark implements Watermark {
private final Collection<? extends Watermark> inputWatermarks;
- private final SortedMultiset<WindowedValue<?>> pendingElements;
+ private final NavigableSet<CommittedBundle<?>> pendingElements;
private final Map<StructuralKey<?>, NavigableSet<TimerData>> objectTimers;
private AtomicReference<Instant> currentWatermark;
@@ -213,10 +214,10 @@ public class WatermarkManager {
// The ordering must order elements by timestamp, and must not compare two distinct elements
// as equal. This is built on the assumption that any element added as a pending element will
// be consumed without modifications.
- Ordering<WindowedValue<?>> pendingElementComparator =
- new WindowedValueByTimestampComparator().compound(Ordering.arbitrary());
+ Ordering<CommittedBundle<?>> pendingBundleComparator =
+ new BundleByElementTimestampComparator().compound(Ordering.arbitrary());
this.pendingElements =
- TreeMultiset.create(pendingElementComparator);
+ new TreeSet<>(pendingBundleComparator);
this.objectTimers = new HashMap<>();
currentWatermark = new AtomicReference<>(BoundedWindow.TIMESTAMP_MIN_VALUE);
}
@@ -249,25 +250,20 @@ public class WatermarkManager {
minInputWatermark = INSTANT_ORDERING.min(minInputWatermark, inputWatermark.get());
}
if (!pendingElements.isEmpty()) {
- minInputWatermark = INSTANT_ORDERING.min(
- minInputWatermark, pendingElements.firstEntry().getElement().getTimestamp());
+ minInputWatermark =
+ INSTANT_ORDERING.min(minInputWatermark, pendingElements.first().getMinTimestamp());
}
Instant newWatermark = INSTANT_ORDERING.max(oldWatermark, minInputWatermark);
currentWatermark.set(newWatermark);
return WatermarkUpdate.fromTimestamps(oldWatermark, newWatermark);
}
- private synchronized void addPendingElements(Iterable<? extends WindowedValue<?>> newPending) {
- for (WindowedValue<?> pendingElement : newPending) {
- pendingElements.add(pendingElement);
- }
+ private synchronized void addPending(CommittedBundle<?> newPending) {
+ pendingElements.add(newPending);
}
- private synchronized void removePendingElements(
- Iterable<? extends WindowedValue<?>> finishedElements) {
- for (WindowedValue<?> finishedElement : finishedElements) {
- pendingElements.remove(finishedElement);
- }
+ private synchronized void removePending(CommittedBundle<?> completed) {
+ pendingElements.remove(completed);
}
private synchronized void updateTimers(TimerUpdate update) {
@@ -856,7 +852,7 @@ public class WatermarkManager {
private void tryApplyPendingUpdates() {
if (refreshLock.tryLock()) {
try {
- applyNUpdates(10);
+ applyNUpdates(MAX_INCREMENTAL_UPDATES);
} finally {
refreshLock.unlock();
}
@@ -867,7 +863,7 @@ public class WatermarkManager {
* Applies all pending updates to this {@link WatermarkManager}, causing the pending state
* of all {@link TransformWatermarks} to be advanced as far as possible.
*/
- private void applyPendingUpdates() {
+ private void applyAllPendingUpdates() {
refreshLock.lock();
try {
applyNUpdates(-1);
@@ -944,7 +940,7 @@ public class WatermarkManager {
synchronized void refreshAll() {
refreshLock.lock();
try {
- applyPendingUpdates();
+ applyAllPendingUpdates();
Set<AppliedPTransform<?, ?, ?>> toRefresh = pendingRefreshes;
while (!toRefresh.isEmpty()) {
toRefresh = refreshAllOf(toRefresh);
@@ -1180,12 +1176,12 @@ public class WatermarkManager {
}
private void removePending(CommittedBundle<?> bundle) {
- inputWatermark.removePendingElements(bundle.getElements());
+ inputWatermark.removePending(bundle);
synchronizedProcessingInputWatermark.removePending(bundle);
}
private void addPending(CommittedBundle<?> bundle) {
- inputWatermark.addPendingElements(bundle.getElements());
+ inputWatermark.addPending(bundle);
synchronizedProcessingInputWatermark.addPending(bundle);
}
@@ -1434,11 +1430,11 @@ public class WatermarkManager {
}
}
- private static class WindowedValueByTimestampComparator extends Ordering<WindowedValue<?>> {
+ private static class BundleByElementTimestampComparator extends Ordering<CommittedBundle<?>> {
@Override
- public int compare(WindowedValue<?> o1, WindowedValue<?> o2) {
+ public int compare(CommittedBundle<?> o1, CommittedBundle<?> o2) {
return ComparisonChain.start()
- .compare(o1.getTimestamp(), o2.getTimestamp())
+ .compare(o1.getMinTimestamp(), o2.getMinTimestamp())
.result();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a58f1eba/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
index 4a7477f..a36c408 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/ImmutableListBundleFactoryTest.java
@@ -19,6 +19,7 @@ package org.apache.beam.runners.direct;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
import com.google.common.collect.ImmutableList;
@@ -35,6 +36,7 @@ import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
@@ -95,15 +97,25 @@ public class ImmutableListBundleFactoryTest {
afterCommitGetElementsShouldHaveAddedElements(Iterable<WindowedValue<T>> elems) {
UncommittedBundle<T> bundle = bundleFactory.createRootBundle();
Collection<Matcher<? super WindowedValue<T>>> expectations = new ArrayList<>();
+ Instant minElementTs = BoundedWindow.TIMESTAMP_MAX_VALUE;
for (WindowedValue<T> elem : elems) {
bundle.add(elem);
expectations.add(equalTo(elem));
+ if (elem.getTimestamp().isBefore(minElementTs)) {
+ minElementTs = elem.getTimestamp();
+ }
}
Matcher<Iterable<? extends WindowedValue<T>>> containsMatcher =
Matchers.<WindowedValue<T>>containsInAnyOrder(expectations);
- CommittedBundle<T> committed = bundle.commit(Instant.now());
+ Instant commitTime = Instant.now();
+ CommittedBundle<T> committed = bundle.commit(commitTime);
assertThat(committed.getElements(), containsMatcher);
+ // Sanity check that the test is meaningful.
+ assertThat(minElementTs, not(equalTo(commitTime)));
+ assertThat(committed.getMinTimestamp(), equalTo(minElementTs));
+ assertThat(committed.getSynchronizedProcessingOutputWatermark(), equalTo(commitTime));
+
return committed;
}
@@ -149,6 +161,7 @@ public class ImmutableListBundleFactoryTest {
assertThat(
withed.getSynchronizedProcessingOutputWatermark(),
equalTo(committed.getSynchronizedProcessingOutputWatermark()));
+ assertThat(withed.getMinTimestamp(), equalTo(new Instant(2048L)));
}
@Test
[50/50] incubator-beam git commit: This closes #1306
Posted by ke...@apache.org.
This closes #1306
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a14927f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a14927f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a14927f4
Branch: refs/heads/gearpump-runner
Commit: a14927f400d8d28f1cff3b21e384a66c647ae3a5
Parents: 323ec11 2a96a17
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Nov 7 19:33:32 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 19:33:32 2016 -0800
----------------------------------------------------------------------
.travis.yml | 22 +-
examples/java/pom.xml | 92 +--
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/MinimalWordCount.java | 2 +-
.../org/apache/beam/examples/WordCount.java | 2 +-
.../beam/examples/common/ExampleUtils.java | 3 +-
.../apache/beam/examples/complete/TfIdf.java | 8 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/DeDupExample.java | 10 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/WindowedWordCountIT.java | 15 +-
.../org/apache/beam/examples/WordCountIT.java | 18 +-
.../org/apache/beam/examples/WordCountTest.java | 2 +-
.../examples/complete/AutoCompleteTest.java | 6 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 2 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 15 +-
.../examples/cookbook/DeDupExampleTest.java | 82 --
.../examples/cookbook/DistinctExampleTest.java | 82 ++
.../examples/cookbook/JoinExamplesTest.java | 2 +-
.../examples/cookbook/TriggerExampleTest.java | 2 +-
examples/java8/pom.xml | 32 +-
.../beam/examples/MinimalWordCountJava8.java | 2 +-
.../beam/examples/complete/game/GameStats.java | 55 +-
.../examples/complete/game/HourlyTeamScore.java | 30 +-
.../examples/complete/game/LeaderBoard.java | 45 +-
.../beam/examples/complete/game/UserScore.java | 18 +-
.../complete/game/utils/WriteToBigQuery.java | 49 +-
.../game/utils/WriteWindowedToBigQuery.java | 14 +-
.../examples/MinimalWordCountJava8Test.java | 2 +-
.../examples/complete/game/GameStatsTest.java | 2 +-
.../complete/game/HourlyTeamScoreTest.java | 2 +-
.../examples/complete/game/LeaderBoardTest.java | 10 +-
.../examples/complete/game/UserScoreTest.java | 6 +-
examples/pom.xml | 18 +-
pom.xml | 123 ++-
runners/core-java/pom.xml | 77 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 3 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 3 +-
.../core/GroupByKeyViaGroupByKeyOnly.java | 13 +-
.../core/PushbackSideInputDoFnRunner.java | 28 +-
.../beam/runners/core/SimpleDoFnRunner.java | 12 +
.../beam/runners/core/SplittableParDo.java | 12 +
.../beam/runners/core/ReduceFnTester.java | 6 +-
.../UnboundedReadFromBoundedSourceTest.java | 4 +-
runners/direct-java/pom.xml | 115 +--
.../runners/direct/AggregatorContainer.java | 20 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGroupByKey.java | 42 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 25 +-
.../beam/runners/direct/DirectRunner.java | 7 +-
.../runners/direct/DirectTimerInternals.java | 13 +
.../beam/runners/direct/EvaluationContext.java | 18 +-
.../direct/ExecutorServiceParallelExecutor.java | 41 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 249 ++++--
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 31 +-
.../direct/ImmutableListBundleFactory.java | 21 +-
.../direct/PTransformOverrideFactory.java | 8 +-
.../beam/runners/direct/ParDoEvaluator.java | 28 +-
.../runners/direct/ParDoEvaluatorFactory.java | 126 +++
.../direct/ParDoMultiEvaluatorFactory.java | 107 ---
.../direct/ParDoMultiEvaluatorHooks.java | 55 ++
.../runners/direct/ParDoOverrideFactory.java | 14 +-
.../direct/ParDoSingleEvaluatorFactory.java | 110 ---
.../direct/ParDoSingleEvaluatorHooks.java | 58 ++
.../direct/TestStreamEvaluatorFactory.java | 14 +-
.../direct/TransformEvaluatorRegistry.java | 10 +-
.../direct/UncommittedBundleOutputManager.java | 50 --
.../runners/direct/ViewEvaluatorFactory.java | 19 +-
.../direct/WatermarkCallbackExecutor.java | 3 +-
.../beam/runners/direct/WatermarkManager.java | 243 ++++--
.../direct/WriteWithShardingFactory.java | 15 +-
.../runners/direct/EvaluationContextTest.java | 23 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 29 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 31 +-
.../direct/ImmutableListBundleFactoryTest.java | 15 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 16 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 439 ----------
.../direct/ParDoMultiEvaluatorHooksTest.java | 439 ++++++++++
.../direct/ParDoSingleEvaluatorFactoryTest.java | 335 --------
.../direct/ParDoSingleEvaluatorHooksTest.java | 335 ++++++++
.../runners/direct/WatermarkManagerTest.java | 102 +--
.../direct/WriteWithShardingFactoryTest.java | 26 +-
runners/flink/examples/pom.xml | 17 -
.../beam/runners/flink/examples/TFIDF.java | 6 +-
.../flink/examples/streaming/AutoComplete.java | 8 +-
.../flink/examples/streaming/JoinExamples.java | 12 +-
.../examples/streaming/KafkaIOExamples.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 8 +-
runners/flink/pom.xml | 9 +
runners/flink/runner/pom.xml | 17 -
.../wrappers/streaming/DoFnOperator.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 13 +
.../gearpump/GearpumpPipelineResult.java | 4 +-
runners/google-cloud-dataflow-java/pom.xml | 142 +---
.../dataflow/BlockingDataflowRunner.java | 181 -----
.../dataflow/DataflowJobCancelledException.java | 39 -
.../dataflow/DataflowJobExecutionException.java | 35 -
.../dataflow/DataflowJobUpdatedException.java | 51 --
.../runners/dataflow/DataflowPipelineJob.java | 61 +-
.../dataflow/DataflowPipelineRegistrar.java | 11 +-
.../dataflow/DataflowPipelineTranslator.java | 8 +-
.../beam/runners/dataflow/DataflowRunner.java | 49 +-
.../runners/dataflow/internal/IsmFormat.java | 5 +-
.../dataflow/internal/ReadTranslator.java | 12 +-
.../BlockingDataflowPipelineOptions.java | 28 -
.../options/DataflowPipelineOptions.java | 8 +
.../testing/TestDataflowPipelineOptions.java | 6 +-
.../dataflow/testing/TestDataflowRunner.java | 7 +-
.../dataflow/util/DataflowTemplateJob.java | 70 ++
.../dataflow/util/DataflowTransport.java | 9 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 53 +-
.../runners/dataflow/util/MonitoringUtil.java | 1 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../dataflow/BlockingDataflowRunnerTest.java | 300 -------
.../dataflow/DataflowPipelineJobTest.java | 30 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 7 +-
.../DataflowPipelineTranslatorTest.java | 28 +
.../runners/dataflow/DataflowRunnerTest.java | 45 ++
runners/pom.xml | 16 +
runners/spark/pom.xml | 87 +-
.../runners/spark/SparkPipelineOptions.java | 14 +-
.../apache/beam/runners/spark/SparkRunner.java | 19 -
.../metrics/AggregatorMetricSource.java | 9 +-
.../metrics/WithNamedAggregatorsSupport.java | 7 +-
.../coders/BeamSparkRunnerRegistrator.java | 46 ++
.../runners/spark/io/EmptyCheckpointMark.java | 52 ++
.../apache/beam/runners/spark/io/KafkaIO.java | 131 ---
.../beam/runners/spark/io/MicrobatchSource.java | 262 ++++++
.../beam/runners/spark/io/SourceDStream.java | 156 ++++
.../apache/beam/runners/spark/io/SourceRDD.java | 77 +-
.../runners/spark/io/SparkUnboundedSource.java | 167 ++++
.../spark/stateful/StateSpecFunctions.java | 167 ++++
.../runners/spark/stateful/package-info.java | 22 +
.../spark/translation/EvaluationContext.java | 6 +-
.../translation/GroupCombineFunctions.java | 65 +-
.../ReifyTimestampsAndWindowsFunction.java | 47 ++
.../spark/translation/SparkContextFactory.java | 5 +-
.../spark/translation/SparkProcessContext.java | 46 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../spark/translation/TransformTranslator.java | 43 +-
.../spark/translation/TranslationUtils.java | 17 +
.../SparkRunnerStreamingContextFactory.java | 30 +-
.../streaming/StreamingEvaluationContext.java | 6 +-
.../streaming/StreamingTransformTranslator.java | 100 +--
.../spark/util/SparkSideInputReader.java | 2 +-
.../runners/spark/ClearAggregatorsRule.java | 33 -
.../runners/spark/InMemoryMetricsSinkRule.java | 32 -
.../metrics/sink/ClearAggregatorsRule.java | 33 +
.../metrics/sink/InMemoryMetrics.java | 15 +-
.../metrics/sink/InMemoryMetricsSinkRule.java | 31 +
.../metrics/sink/NamedAggregatorsTest.java | 92 +++
.../streaming/EmptyStreamAssertionTest.java | 3 +-
.../streaming/FlattenStreamingTest.java | 6 +-
.../streaming/KafkaStreamingTest.java | 152 +++-
.../ResumeFromCheckpointStreamingTest.java | 37 +-
.../streaming/SimpleStreamingWordCountTest.java | 3 +-
.../streaming/utils/EmbeddedKafkaCluster.java | 51 +-
.../utils/KafkaWriteOnBatchCompleted.java | 105 +++
.../utils/TestOptionsForStreaming.java | 12 +-
sdks/java/build-tools/pom.xml | 4 +-
.../src/main/resources/beam/checkstyle.xml | 9 +-
sdks/java/core/pom.xml | 157 +---
.../org/apache/beam/sdk/PipelineResult.java | 10 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 147 +++-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 11 +-
.../apache/beam/sdk/io/CompressedSource.java | 4 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 43 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 59 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 123 ++-
.../main/java/org/apache/beam/sdk/io/Sink.java | 4 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 43 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 136 +---
.../sdk/options/PipelineOptionsFactory.java | 248 ++++--
.../sdk/options/ProxyInvocationHandler.java | 5 +-
.../apache/beam/sdk/options/ValueProvider.java | 38 +-
.../beam/sdk/options/ValueProviderUtils.java | 60 ++
.../beam/sdk/testing/BigqueryMatcher.java | 15 +-
.../apache/beam/sdk/transforms/Aggregator.java | 13 +-
.../apache/beam/sdk/transforms/CombineFns.java | 4 +-
.../sdk/transforms/DelegatingAggregator.java | 7 +-
.../apache/beam/sdk/transforms/Distinct.java | 158 ++++
.../org/apache/beam/sdk/transforms/DoFn.java | 42 +-
.../beam/sdk/transforms/DoFnAdapters.java | 22 +
.../org/apache/beam/sdk/transforms/Flatten.java | 3 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 4 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 45 +-
.../org/apache/beam/sdk/transforms/Regex.java | 454 +++++++++++
.../beam/sdk/transforms/RemoveDuplicates.java | 159 ----
.../org/apache/beam/sdk/transforms/View.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 5 +-
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../sdk/transforms/reflect/DoFnInvokers.java | 150 ++--
.../sdk/transforms/reflect/DoFnSignature.java | 113 ++-
.../sdk/transforms/reflect/DoFnSignatures.java | 794 ++++++++++++-------
.../splittabledofn/RestrictionTracker.java | 2 +-
.../transforms/splittabledofn/package-info.java | 4 +-
.../beam/sdk/transforms/windowing/Window.java | 130 +--
.../beam/sdk/transforms/windowing/WindowFn.java | 12 +
.../beam/sdk/util/BaseExecutionContext.java | 2 +-
.../BufferedElementCountingOutputStream.java | 4 +-
.../apache/beam/sdk/util/CredentialFactory.java | 4 +-
.../org/apache/beam/sdk/util/Credentials.java | 192 -----
.../apache/beam/sdk/util/ExecutableTrigger.java | 131 ---
.../apache/beam/sdk/util/ExecutionContext.java | 8 +-
.../sdk/util/ExposedByteArrayInputStream.java | 5 +-
.../sdk/util/ExposedByteArrayOutputStream.java | 2 -
.../beam/sdk/util/FileIOChannelFactory.java | 8 +-
.../apache/beam/sdk/util/GatherAllPanes.java | 18 +-
.../beam/sdk/util/GcpCredentialFactory.java | 41 +-
.../beam/sdk/util/GcsIOChannelFactory.java | 8 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 -
.../apache/beam/sdk/util/IOChannelFactory.java | 4 +
.../apache/beam/sdk/util/MovingFunction.java | 4 +-
.../beam/sdk/util/NoopCredentialFactory.java | 9 +-
.../org/apache/beam/sdk/util/PubsubClient.java | 32 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 9 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 32 +
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 41 -
.../sdk/util/ReifyTimestampsAndWindows.java | 63 --
.../org/apache/beam/sdk/util/StringUtils.java | 8 +-
.../apache/beam/sdk/util/TestCredential.java | 44 +-
.../apache/beam/sdk/util/TimerInternals.java | 35 +-
.../org/apache/beam/sdk/util/Transport.java | 9 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 10 +-
.../beam/sdk/util/common/ReflectHelpers.java | 17 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 13 +-
.../sdk/util/state/InMemoryTimerInternals.java | 12 +
.../beam/sdk/util/state/StateNamespace.java | 16 +-
.../java/org/apache/beam/sdk/values/PDone.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptor.java | 48 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 1 -
.../apache/beam/sdk/values/TypeParameter.java | 62 ++
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 29 +
.../apache/beam/sdk/io/CountingInputTest.java | 6 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 8 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 23 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 79 ++
.../org/apache/beam/sdk/io/XmlSinkTest.java | 16 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 239 ++++++
.../sdk/options/ProxyInvocationHandlerTest.java | 29 +-
.../beam/sdk/options/ValueProviderTest.java | 50 +-
.../sdk/options/ValueProviderUtilsTest.java | 78 ++
.../apache/beam/sdk/testing/ExpectedLogs.java | 15 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 30 +-
.../apache/beam/sdk/testing/PAssertTest.java | 27 +
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/DistinctTest.java | 130 +++
.../apache/beam/sdk/transforms/ParDoTest.java | 61 ++
.../apache/beam/sdk/transforms/RegexTest.java | 248 ++++++
.../sdk/transforms/RemoveDuplicatesTest.java | 130 ---
.../transforms/reflect/DoFnInvokersTest.java | 100 ++-
.../DoFnSignaturesProcessElementTest.java | 18 +-
.../DoFnSignaturesSplittableDoFnTest.java | 24 +-
.../transforms/reflect/DoFnSignaturesTest.java | 43 +-
.../reflect/DoFnSignaturesTestUtils.java | 13 +-
.../sdk/transforms/windowing/WindowTest.java | 6 +-
.../beam/sdk/util/ExecutableTriggerTest.java | 109 ---
.../beam/sdk/util/PubsubGrpcClientTest.java | 9 +-
.../sdk/util/common/ReflectHelpersTest.java | 25 +
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 25 +
.../beam/sdk/values/TypeDescriptorTest.java | 12 +
sdks/java/extensions/join-library/pom.xml | 8 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/README.md | 42 +
sdks/java/extensions/sorter/pom.xml | 159 ++++
.../sorter/BufferedExternalSorter.java | 125 +++
.../sdk/extensions/sorter/ExternalSorter.java | 225 ++++++
.../sdk/extensions/sorter/InMemorySorter.java | 166 ++++
.../beam/sdk/extensions/sorter/SortValues.java | 213 +++++
.../beam/sdk/extensions/sorter/Sorter.java | 47 ++
.../sdk/extensions/sorter/package-info.java | 23 +
.../sorter/BufferedExternalSorterTest.java | 177 +++++
.../extensions/sorter/ExternalSorterTest.java | 87 ++
.../extensions/sorter/InMemorySorterTest.java | 144 ++++
.../sdk/extensions/sorter/SortValuesTest.java | 128 +++
.../sdk/extensions/sorter/SorterTestUtils.java | 129 +++
sdks/java/io/google-cloud-platform/pom.xml | 17 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 27 +-
.../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 132 +++
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 25 +-
sdks/java/io/hdfs/pom.xml | 8 -
sdks/java/io/jdbc/pom.xml | 14 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 39 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 33 +
sdks/java/io/jms/pom.xml | 21 +-
sdks/java/io/kafka/pom.xml | 28 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 6 +-
sdks/java/io/kinesis/pom.xml | 23 +-
.../beam/sdk/io/kinesis/CustomOptional.java | 21 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +-
.../beam/sdk/io/kinesis/KinesisRecord.java | 3 +-
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 +
sdks/java/io/mongodb/pom.xml | 25 +-
sdks/java/java8tests/pom.xml | 11 -
.../beam/sdk/transforms/DistinctJava8Test.java | 104 +++
.../transforms/RemoveDuplicatesJava8Test.java | 104 ---
.../main/resources/archetype-resources/pom.xml | 32 -
sdks/java/microbenchmarks/pom.xml | 23 +-
sdks/java/pom.xml | 9 +-
sdks/pom.xml | 21 +-
311 files changed, 10047 insertions(+), 5854 deletions(-)
----------------------------------------------------------------------
[48/50] incubator-beam git commit: Merge branch 'master' into
gearpump-runner
Posted by ke...@apache.org.
Merge branch 'master' into gearpump-runner
# Conflicts:
# runners/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/0c362282
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/0c362282
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/0c362282
Branch: refs/heads/gearpump-runner
Commit: 0c362282d3f2ca224c937c813f467723d7b1e908
Parents: 323ec11 99505e1
Author: manuzhang <ow...@gmail.com>
Authored: Tue Nov 8 10:30:33 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Tue Nov 8 10:30:33 2016 +0800
----------------------------------------------------------------------
.travis.yml | 22 +-
examples/java/pom.xml | 92 +--
.../beam/examples/DebuggingWordCount.java | 2 +-
.../apache/beam/examples/MinimalWordCount.java | 2 +-
.../org/apache/beam/examples/WordCount.java | 2 +-
.../beam/examples/common/ExampleUtils.java | 3 +-
.../apache/beam/examples/complete/TfIdf.java | 8 +-
.../examples/complete/TopWikipediaSessions.java | 2 +-
.../examples/cookbook/BigQueryTornadoes.java | 2 +-
.../cookbook/CombinePerKeyExamples.java | 2 +-
.../beam/examples/cookbook/DeDupExample.java | 10 +-
.../beam/examples/cookbook/FilterExamples.java | 2 +-
.../beam/examples/cookbook/JoinExamples.java | 2 +-
.../examples/cookbook/MaxPerKeyExamples.java | 2 +-
.../org/apache/beam/examples/cookbook/README.md | 2 +-
.../beam/examples/WindowedWordCountIT.java | 15 +-
.../org/apache/beam/examples/WordCountIT.java | 18 +-
.../org/apache/beam/examples/WordCountTest.java | 2 +-
.../examples/complete/AutoCompleteTest.java | 6 +-
.../beam/examples/complete/TfIdfTest.java | 6 +-
.../complete/TopWikipediaSessionsTest.java | 2 +-
.../examples/cookbook/BigQueryTornadoesIT.java | 15 +-
.../examples/cookbook/DeDupExampleTest.java | 82 --
.../examples/cookbook/DistinctExampleTest.java | 82 ++
.../examples/cookbook/JoinExamplesTest.java | 2 +-
.../examples/cookbook/TriggerExampleTest.java | 2 +-
examples/java8/pom.xml | 32 +-
.../beam/examples/MinimalWordCountJava8.java | 2 +-
.../beam/examples/complete/game/GameStats.java | 55 +-
.../examples/complete/game/HourlyTeamScore.java | 30 +-
.../examples/complete/game/LeaderBoard.java | 45 +-
.../beam/examples/complete/game/UserScore.java | 18 +-
.../complete/game/utils/WriteToBigQuery.java | 49 +-
.../game/utils/WriteWindowedToBigQuery.java | 14 +-
.../examples/MinimalWordCountJava8Test.java | 2 +-
.../examples/complete/game/GameStatsTest.java | 2 +-
.../complete/game/HourlyTeamScoreTest.java | 2 +-
.../examples/complete/game/LeaderBoardTest.java | 10 +-
.../examples/complete/game/UserScoreTest.java | 6 +-
examples/pom.xml | 18 +-
pom.xml | 123 ++-
runners/core-java/pom.xml | 77 +-
.../core/GroupAlsoByWindowViaWindowSetDoFn.java | 3 +-
.../GroupAlsoByWindowsViaOutputBufferDoFn.java | 3 +-
.../core/GroupByKeyViaGroupByKeyOnly.java | 13 +-
.../core/PushbackSideInputDoFnRunner.java | 28 +-
.../beam/runners/core/SimpleDoFnRunner.java | 12 +
.../beam/runners/core/SplittableParDo.java | 12 +
.../beam/runners/core/ReduceFnTester.java | 6 +-
.../UnboundedReadFromBoundedSourceTest.java | 4 +-
runners/direct-java/pom.xml | 115 +--
.../runners/direct/AggregatorContainer.java | 20 +-
...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +-
.../beam/runners/direct/DirectGroupByKey.java | 42 +-
.../direct/DirectGroupByKeyOverrideFactory.java | 25 +-
.../beam/runners/direct/DirectRunner.java | 7 +-
.../runners/direct/DirectTimerInternals.java | 13 +
.../beam/runners/direct/EvaluationContext.java | 18 +-
.../direct/ExecutorServiceParallelExecutor.java | 41 +-
.../GroupAlsoByWindowEvaluatorFactory.java | 249 ++++--
.../direct/GroupByKeyOnlyEvaluatorFactory.java | 31 +-
.../direct/ImmutableListBundleFactory.java | 21 +-
.../direct/PTransformOverrideFactory.java | 8 +-
.../beam/runners/direct/ParDoEvaluator.java | 28 +-
.../runners/direct/ParDoEvaluatorFactory.java | 126 +++
.../direct/ParDoMultiEvaluatorFactory.java | 107 ---
.../direct/ParDoMultiEvaluatorHooks.java | 55 ++
.../runners/direct/ParDoOverrideFactory.java | 14 +-
.../direct/ParDoSingleEvaluatorFactory.java | 110 ---
.../direct/ParDoSingleEvaluatorHooks.java | 58 ++
.../direct/TestStreamEvaluatorFactory.java | 14 +-
.../direct/TransformEvaluatorRegistry.java | 10 +-
.../direct/UncommittedBundleOutputManager.java | 50 --
.../runners/direct/ViewEvaluatorFactory.java | 19 +-
.../direct/WatermarkCallbackExecutor.java | 3 +-
.../beam/runners/direct/WatermarkManager.java | 243 ++++--
.../direct/WriteWithShardingFactory.java | 15 +-
.../runners/direct/EvaluationContextTest.java | 23 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 29 +-
.../GroupByKeyOnlyEvaluatorFactoryTest.java | 31 +-
.../direct/ImmutableListBundleFactoryTest.java | 15 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 16 +-
.../direct/ParDoMultiEvaluatorFactoryTest.java | 439 ----------
.../direct/ParDoMultiEvaluatorHooksTest.java | 439 ++++++++++
.../direct/ParDoSingleEvaluatorFactoryTest.java | 335 --------
.../direct/ParDoSingleEvaluatorHooksTest.java | 335 ++++++++
.../runners/direct/WatermarkManagerTest.java | 102 +--
.../direct/WriteWithShardingFactoryTest.java | 26 +-
runners/flink/examples/pom.xml | 17 -
.../beam/runners/flink/examples/TFIDF.java | 6 +-
.../flink/examples/streaming/AutoComplete.java | 8 +-
.../flink/examples/streaming/JoinExamples.java | 12 +-
.../examples/streaming/KafkaIOExamples.java | 2 +-
.../examples/streaming/WindowedWordCount.java | 8 +-
runners/flink/pom.xml | 9 +
runners/flink/runner/pom.xml | 17 -
.../wrappers/streaming/DoFnOperator.java | 10 +-
.../wrappers/streaming/WindowDoFnOperator.java | 13 +
runners/google-cloud-dataflow-java/pom.xml | 142 +---
.../dataflow/BlockingDataflowRunner.java | 181 -----
.../dataflow/DataflowJobCancelledException.java | 39 -
.../dataflow/DataflowJobExecutionException.java | 35 -
.../dataflow/DataflowJobUpdatedException.java | 51 --
.../runners/dataflow/DataflowPipelineJob.java | 61 +-
.../dataflow/DataflowPipelineRegistrar.java | 11 +-
.../dataflow/DataflowPipelineTranslator.java | 8 +-
.../beam/runners/dataflow/DataflowRunner.java | 49 +-
.../runners/dataflow/internal/IsmFormat.java | 5 +-
.../dataflow/internal/ReadTranslator.java | 12 +-
.../BlockingDataflowPipelineOptions.java | 28 -
.../options/DataflowPipelineOptions.java | 8 +
.../testing/TestDataflowPipelineOptions.java | 6 +-
.../dataflow/testing/TestDataflowRunner.java | 7 +-
.../dataflow/util/DataflowTemplateJob.java | 70 ++
.../dataflow/util/DataflowTransport.java | 9 +-
.../beam/runners/dataflow/util/DoFnInfo.java | 53 +-
.../runners/dataflow/util/MonitoringUtil.java | 1 -
.../runners/dataflow/util/RandomAccessData.java | 2 +-
.../dataflow/BlockingDataflowRunnerTest.java | 300 -------
.../dataflow/DataflowPipelineJobTest.java | 30 +-
.../dataflow/DataflowPipelineRegistrarTest.java | 7 +-
.../DataflowPipelineTranslatorTest.java | 28 +
.../runners/dataflow/DataflowRunnerTest.java | 45 ++
runners/pom.xml | 16 +
runners/spark/pom.xml | 87 +-
.../runners/spark/SparkPipelineOptions.java | 14 +-
.../apache/beam/runners/spark/SparkRunner.java | 19 -
.../metrics/AggregatorMetricSource.java | 9 +-
.../metrics/WithNamedAggregatorsSupport.java | 7 +-
.../coders/BeamSparkRunnerRegistrator.java | 46 ++
.../runners/spark/io/EmptyCheckpointMark.java | 52 ++
.../apache/beam/runners/spark/io/KafkaIO.java | 131 ---
.../beam/runners/spark/io/MicrobatchSource.java | 262 ++++++
.../beam/runners/spark/io/SourceDStream.java | 156 ++++
.../apache/beam/runners/spark/io/SourceRDD.java | 77 +-
.../runners/spark/io/SparkUnboundedSource.java | 167 ++++
.../spark/stateful/StateSpecFunctions.java | 167 ++++
.../runners/spark/stateful/package-info.java | 22 +
.../spark/translation/EvaluationContext.java | 6 +-
.../translation/GroupCombineFunctions.java | 65 +-
.../ReifyTimestampsAndWindowsFunction.java | 47 ++
.../spark/translation/SparkContextFactory.java | 5 +-
.../spark/translation/SparkProcessContext.java | 46 +-
.../spark/translation/SparkRuntimeContext.java | 2 +-
.../spark/translation/TransformTranslator.java | 43 +-
.../spark/translation/TranslationUtils.java | 17 +
.../SparkRunnerStreamingContextFactory.java | 30 +-
.../streaming/StreamingEvaluationContext.java | 6 +-
.../streaming/StreamingTransformTranslator.java | 100 +--
.../spark/util/SparkSideInputReader.java | 2 +-
.../runners/spark/ClearAggregatorsRule.java | 33 -
.../runners/spark/InMemoryMetricsSinkRule.java | 32 -
.../metrics/sink/ClearAggregatorsRule.java | 33 +
.../metrics/sink/InMemoryMetrics.java | 15 +-
.../metrics/sink/InMemoryMetricsSinkRule.java | 31 +
.../metrics/sink/NamedAggregatorsTest.java | 92 +++
.../streaming/EmptyStreamAssertionTest.java | 3 +-
.../streaming/FlattenStreamingTest.java | 6 +-
.../streaming/KafkaStreamingTest.java | 152 +++-
.../ResumeFromCheckpointStreamingTest.java | 37 +-
.../streaming/SimpleStreamingWordCountTest.java | 3 +-
.../streaming/utils/EmbeddedKafkaCluster.java | 51 +-
.../utils/KafkaWriteOnBatchCompleted.java | 105 +++
.../utils/TestOptionsForStreaming.java | 12 +-
sdks/java/build-tools/pom.xml | 4 +-
.../src/main/resources/beam/checkstyle.xml | 9 +-
sdks/java/core/pom.xml | 157 +---
.../org/apache/beam/sdk/PipelineResult.java | 10 +-
.../java/org/apache/beam/sdk/io/AvroIO.java | 147 +++-
.../sdk/io/BoundedReadFromUnboundedSource.java | 4 +-
.../org/apache/beam/sdk/io/BoundedSource.java | 11 +-
.../apache/beam/sdk/io/CompressedSource.java | 4 +-
.../org/apache/beam/sdk/io/FileBasedSink.java | 43 +-
.../org/apache/beam/sdk/io/FileBasedSource.java | 59 +-
.../beam/sdk/io/PubsubUnboundedSource.java | 123 ++-
.../main/java/org/apache/beam/sdk/io/Sink.java | 4 +-
.../java/org/apache/beam/sdk/io/TextIO.java | 43 +-
.../java/org/apache/beam/sdk/io/XmlSource.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRange.java | 2 +-
.../org/apache/beam/sdk/options/GcpOptions.java | 136 +---
.../sdk/options/PipelineOptionsFactory.java | 248 ++++--
.../sdk/options/ProxyInvocationHandler.java | 5 +-
.../apache/beam/sdk/options/ValueProvider.java | 38 +-
.../beam/sdk/options/ValueProviderUtils.java | 60 ++
.../beam/sdk/testing/BigqueryMatcher.java | 15 +-
.../apache/beam/sdk/transforms/Aggregator.java | 13 +-
.../apache/beam/sdk/transforms/CombineFns.java | 4 +-
.../sdk/transforms/DelegatingAggregator.java | 7 +-
.../apache/beam/sdk/transforms/Distinct.java | 158 ++++
.../org/apache/beam/sdk/transforms/DoFn.java | 42 +-
.../beam/sdk/transforms/DoFnAdapters.java | 22 +
.../org/apache/beam/sdk/transforms/Flatten.java | 3 +-
.../apache/beam/sdk/transforms/GroupByKey.java | 4 +-
.../org/apache/beam/sdk/transforms/ParDo.java | 45 +-
.../org/apache/beam/sdk/transforms/Regex.java | 454 +++++++++++
.../beam/sdk/transforms/RemoveDuplicates.java | 159 ----
.../org/apache/beam/sdk/transforms/View.java | 2 +-
.../sdk/transforms/display/DisplayData.java | 5 +-
.../beam/sdk/transforms/join/CoGroupByKey.java | 6 +-
.../sdk/transforms/reflect/DoFnInvokers.java | 150 ++--
.../sdk/transforms/reflect/DoFnSignature.java | 113 ++-
.../sdk/transforms/reflect/DoFnSignatures.java | 794 ++++++++++++-------
.../splittabledofn/RestrictionTracker.java | 2 +-
.../transforms/splittabledofn/package-info.java | 4 +-
.../beam/sdk/transforms/windowing/Window.java | 130 +--
.../beam/sdk/transforms/windowing/WindowFn.java | 12 +
.../beam/sdk/util/BaseExecutionContext.java | 2 +-
.../BufferedElementCountingOutputStream.java | 4 +-
.../apache/beam/sdk/util/CredentialFactory.java | 4 +-
.../org/apache/beam/sdk/util/Credentials.java | 192 -----
.../apache/beam/sdk/util/ExecutableTrigger.java | 131 ---
.../apache/beam/sdk/util/ExecutionContext.java | 8 +-
.../sdk/util/ExposedByteArrayInputStream.java | 5 +-
.../sdk/util/ExposedByteArrayOutputStream.java | 2 -
.../beam/sdk/util/FileIOChannelFactory.java | 8 +-
.../apache/beam/sdk/util/GatherAllPanes.java | 18 +-
.../beam/sdk/util/GcpCredentialFactory.java | 41 +-
.../beam/sdk/util/GcsIOChannelFactory.java | 8 +-
.../java/org/apache/beam/sdk/util/GcsUtil.java | 2 -
.../apache/beam/sdk/util/IOChannelFactory.java | 4 +
.../apache/beam/sdk/util/MovingFunction.java | 4 +-
.../beam/sdk/util/NoopCredentialFactory.java | 9 +-
.../org/apache/beam/sdk/util/PubsubClient.java | 32 +-
.../apache/beam/sdk/util/PubsubGrpcClient.java | 18 +-
.../apache/beam/sdk/util/PubsubJsonClient.java | 9 +-
.../apache/beam/sdk/util/PubsubTestClient.java | 32 +
.../sdk/util/ReifyTimestampAndWindowsDoFn.java | 41 -
.../sdk/util/ReifyTimestampsAndWindows.java | 63 --
.../org/apache/beam/sdk/util/StringUtils.java | 8 +-
.../apache/beam/sdk/util/TestCredential.java | 44 +-
.../apache/beam/sdk/util/TimerInternals.java | 35 +-
.../org/apache/beam/sdk/util/Transport.java | 9 +-
.../apache/beam/sdk/util/WindowingStrategy.java | 10 +-
.../beam/sdk/util/common/ReflectHelpers.java | 17 +
.../org/apache/beam/sdk/util/gcsfs/GcsPath.java | 13 +-
.../sdk/util/state/InMemoryTimerInternals.java | 12 +
.../beam/sdk/util/state/StateNamespace.java | 16 +-
.../java/org/apache/beam/sdk/values/PDone.java | 2 +-
.../apache/beam/sdk/values/TypeDescriptor.java | 48 +-
.../apache/beam/sdk/values/TypeDescriptors.java | 1 -
.../apache/beam/sdk/values/TypeParameter.java | 62 ++
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 29 +
.../apache/beam/sdk/io/CountingInputTest.java | 6 +-
.../apache/beam/sdk/io/CountingSourceTest.java | 8 +-
.../apache/beam/sdk/io/FileBasedSinkTest.java | 23 +-
.../beam/sdk/io/PubsubUnboundedSourceTest.java | 79 ++
.../org/apache/beam/sdk/io/XmlSinkTest.java | 16 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 239 ++++++
.../sdk/options/ProxyInvocationHandlerTest.java | 29 +-
.../beam/sdk/options/ValueProviderTest.java | 50 +-
.../sdk/options/ValueProviderUtilsTest.java | 78 ++
.../apache/beam/sdk/testing/ExpectedLogs.java | 15 +-
.../beam/sdk/testing/ExpectedLogsTest.java | 30 +-
.../apache/beam/sdk/testing/PAssertTest.java | 27 +
.../sdk/transforms/ApproximateUniqueTest.java | 2 +-
.../beam/sdk/transforms/DistinctTest.java | 130 +++
.../apache/beam/sdk/transforms/ParDoTest.java | 61 ++
.../apache/beam/sdk/transforms/RegexTest.java | 248 ++++++
.../sdk/transforms/RemoveDuplicatesTest.java | 130 ---
.../transforms/reflect/DoFnInvokersTest.java | 100 ++-
.../DoFnSignaturesProcessElementTest.java | 18 +-
.../DoFnSignaturesSplittableDoFnTest.java | 24 +-
.../transforms/reflect/DoFnSignaturesTest.java | 43 +-
.../reflect/DoFnSignaturesTestUtils.java | 13 +-
.../sdk/transforms/windowing/WindowTest.java | 6 +-
.../beam/sdk/util/ExecutableTriggerTest.java | 109 ---
.../beam/sdk/util/PubsubGrpcClientTest.java | 9 +-
.../sdk/util/common/ReflectHelpersTest.java | 25 +
.../apache/beam/sdk/util/gcsfs/GcsPathTest.java | 25 +
.../beam/sdk/values/TypeDescriptorTest.java | 12 +
sdks/java/extensions/join-library/pom.xml | 8 -
sdks/java/extensions/pom.xml | 1 +
sdks/java/extensions/sorter/README.md | 42 +
sdks/java/extensions/sorter/pom.xml | 159 ++++
.../sorter/BufferedExternalSorter.java | 125 +++
.../sdk/extensions/sorter/ExternalSorter.java | 225 ++++++
.../sdk/extensions/sorter/InMemorySorter.java | 166 ++++
.../beam/sdk/extensions/sorter/SortValues.java | 213 +++++
.../beam/sdk/extensions/sorter/Sorter.java | 47 ++
.../sdk/extensions/sorter/package-info.java | 23 +
.../sorter/BufferedExternalSorterTest.java | 177 +++++
.../extensions/sorter/ExternalSorterTest.java | 87 ++
.../extensions/sorter/InMemorySorterTest.java | 144 ++++
.../sdk/extensions/sorter/SortValuesTest.java | 128 +++
.../sdk/extensions/sorter/SorterTestUtils.java | 129 +++
sdks/java/io/google-cloud-platform/pom.xml | 17 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 27 +-
.../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 132 +++
.../beam/sdk/io/gcp/datastore/V1TestUtil.java | 25 +-
sdks/java/io/hdfs/pom.xml | 8 -
sdks/java/io/jdbc/pom.xml | 14 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 39 +
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 33 +
sdks/java/io/jms/pom.xml | 21 +-
sdks/java/io/kafka/pom.xml | 28 +-
.../apache/beam/sdk/io/kafka/KafkaIOTest.java | 6 +-
sdks/java/io/kinesis/pom.xml | 23 +-
.../beam/sdk/io/kinesis/CustomOptional.java | 21 +-
.../apache/beam/sdk/io/kinesis/KinesisIO.java | 35 +-
.../beam/sdk/io/kinesis/KinesisRecord.java | 3 +-
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 +
sdks/java/io/mongodb/pom.xml | 25 +-
sdks/java/java8tests/pom.xml | 11 -
.../beam/sdk/transforms/DistinctJava8Test.java | 104 +++
.../transforms/RemoveDuplicatesJava8Test.java | 104 ---
.../main/resources/archetype-resources/pom.xml | 32 -
sdks/java/microbenchmarks/pom.xml | 23 +-
sdks/java/pom.xml | 9 +-
sdks/pom.xml | 21 +-
310 files changed, 10045 insertions(+), 5852 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/0c362282/runners/pom.xml
----------------------------------------------------------------------
diff --cc runners/pom.xml
index 4be31d0,8084d0b..457bae5
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@@ -42,14 -42,21 +42,30 @@@
<profiles>
<profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
++ <profile>
+ <id>java8</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <modules>
+ <module>gearpump</module>
+ </modules>
+ </profile>
<!-- A profile that adds an integration test phase if and only if
the runnableOnServicePipelineOptions maven property has been set.
[19/50] incubator-beam git commit: Rename RegexTransform to just Regex
Posted by ke...@apache.org.
Rename RegexTransform to just Regex
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/6954abe4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/6954abe4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/6954abe4
Branch: refs/heads/gearpump-runner
Commit: 6954abe4e28103f5ddd3e1eebe998b765cd9de11
Parents: f6a9733
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Nov 7 10:10:32 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 10:10:32 2016 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/Regex.java | 505 +++++++++++++++++++
.../beam/sdk/transforms/RegexTransform.java | 505 -------------------
.../apache/beam/sdk/transforms/RegexTest.java | 262 ++++++++++
.../beam/sdk/transforms/RegexTransformTest.java | 262 ----------
4 files changed, 767 insertions(+), 767 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6954abe4/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
new file mode 100644
index 0000000..27104f6
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/**
+ * {@code PTransorm}s to use Regular Expressions to process elements in a
+ * {@link PCollection}.
+ *
+ * <p>
+ * {@link Regex#matches(String, int)} can be used to see if an entire line matches
+ * a Regex. {@link Regex#matchesKV(String, int, int)} can be used to see if an entire
+ * line matches a Regex and output certain groups as a {@link KV}.
+ * </p>
+ * <p>
+ * {@link Regex#find(String, int)} can be used to see if a portion of a line
+ * matches a Regex. {@link Regex#matchesKV(String, int, int)} can be used to see if a
+ * portion of a line matches a Regex and output certain groups as a {@link KV}.
+ * </p>
+ * <p>
+ * Lines that do not match the Regex will not be output.
+ * </p>
+ */
+public class Regex {
+ private Regex() {
+ // do not instantiate
+ }
+
+ /**
+ * Returns a {@link Regex.Matches} {@link PTransform} that checks if
+ * the entire line matches the Regex. Returns the entire line (group 0) as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Matches matches(String regex) {
+ return matches(regex, 0);
+ }
+
+ /**
+ * Returns a {@link Regex.Matches} {@link PTransform} that checks if
+ * the entire line matches the Regex. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param group
+ * The Regex group to return as a PCollection
+ */
+ public static Matches matches(String regex, int group) {
+ return new Matches(regex, group);
+ }
+
+ /**
+ * Returns a {@link Regex.MatchesKV} {@link PTransform} that checks
+ * if the entire line matches the Regex. Returns the specified groups as the
+ * key and value as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param keyGroup
+ * The Regex group to use as the key
+ * @param valueGroup
+ * The Regex group to use the value
+ */
+ public static MatchesKV matchesKV(String regex, int keyGroup,
+ int valueGroup) {
+ return new MatchesKV(regex, keyGroup, valueGroup);
+ }
+
+ /**
+ * Returns a {@link Regex.Find} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the entire line (group 0) as
+ * a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Find find(String regex) {
+ return find(regex, 0);
+ }
+
+ /**
+ * Returns a {@link Regex.Find} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param group
+ * The Regex group to return as a PCollection
+ */
+ public static Find find(String regex, int group) {
+ return new Find(regex, group);
+ }
+
+ /**
+ * Returns a {@link Regex.FindKV} {@link PTransform} that checks if a
+ * portion of the line matches the Regex. Returns the specified groups as the
+ * key and value as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param keyGroup
+ * The Regex group to use as the key
+ * @param valueGroup
+ * The Regex group to use the value
+ */
+ public static FindKV findKV(String regex, int keyGroup, int valueGroup) {
+ return new FindKV(regex, keyGroup, valueGroup);
+ }
+
+ /**
+ * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a
+ * portion of the line matches the Regex and replaces all matches with the replacement
+ * String. Returns the group as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param replacement
+ * The string to be substituted for each match
+ */
+ public static ReplaceAll replaceAll(String regex, String replacement) {
+ return new ReplaceAll(regex, replacement);
+ }
+
+ /**
+ * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a
+ * portion of the line matches the Regex and replaces the first match with the replacement
+ * String. Returns the group as a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param replacement
+ * The string to be substituted for each match
+ */
+ public static ReplaceFirst replaceFirst(String regex, String replacement) {
+ return new ReplaceFirst(regex, replacement);
+ }
+
+ /**
+ * Returns a {@link Regex.Split} {@link PTransform} that splits a string
+ * on the regular expression and then outputs each item. It will not output empty
+ * items. Returns the group as a {@link PCollection}.
+ * a {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ */
+ public static Split split(String regex) {
+ return split(regex, false);
+ }
+
+ /**
+ * Returns a {@link Regex.Split} {@link PTransform} that splits a string
+ * on the regular expression and then outputs each item. Returns the group as a
+ * {@link PCollection}.
+ * @param regex
+ * The regular expression to run
+ * @param outputEmpty
+ * Should empty be output. True to output empties and false if not.
+ */
+ public static Split split(String regex, boolean outputEmpty) {
+ return new Split(regex, outputEmpty);
+ }
+
+ /**
+ * {@code Regex.Matches<String>} takes a {@code PCollection<String>}
+ * and returns a {@code PCollection<String>} representing the value
+ * extracted from the Regex groups of the input {@code PCollection}
+ * to the number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If the entire line
+ * does not match the Regex, the line will not be output. If it does match the
+ * entire line, the group in the Regex will be used. The output will be the
+ * Regex group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(Regex.matches("myregex (mygroup)", 1));
+ * }
+ * </pre>
+ */
+ public static class Matches
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ int group;
+
+ public Matches(String regex, int group) {
+ this.pattern = Pattern.compile(regex);
+ this.group = group;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in
+ .apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.matches()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.MatchesKV<KV<String, String>>} takes a
+ * {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value
+ * extracted from the Regex groups of the input {@code PCollection} to the
+ * number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If the entire line
+ * does not match the Regex, the line will not be output. If it does match the
+ * entire line, the groups in the Regex will be used. The key will be the
+ * key's group and the value will be the value's group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<KV<String, String>> keysAndValues =
+ * words.apply(Regex.matchesKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
+ * }
+ * </pre>
+ */
+ public static class MatchesKV
+ extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
+ Pattern pattern;
+ int keyGroup, valueGroup;
+
+ public MatchesKV(String regex, int keyGroup, int valueGroup) {
+ this.pattern = Pattern.compile(regex);
+ this.keyGroup = keyGroup;
+ this.valueGroup = valueGroup;
+ }
+
+ public PCollection<KV<String, String>> apply(PCollection<String> in) {
+ return in.apply(ParDo
+ .of(new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.Find<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} representing the value extracted
+ * from the Regex groups of the input {@code PCollection} to
+ * the number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will not be output. If it does
+ * match a portion of the line, the group in the Regex will be used. The
+ * output will be the Regex group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(Regex.find("myregex (mygroup)", 1));
+ * }
+ * </pre>
+ */
+ public static class Find
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ int group;
+
+ public Find(String regex, int group) {
+ this.pattern = Pattern.compile(regex);
+ this.group = group;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.MatchesKV<KV<String, String>>} takes a
+ * {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value
+ * extracted from the Regex groups of the input {@code PCollection} to the
+ * number of times that element occurs in the input.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will not be output. If it does
+ * match a portion of the line, the groups in the Regex will be used. The key
+ * will be the key's group and the value will be the value's group.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<KV<String, String>> keysAndValues =
+ * words.apply(Regex.findKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
+ * }
+ * </pre>
+ */
+ public static class FindKV
+ extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
+ Pattern pattern;
+ int keyGroup, valueGroup;
+
+ public FindKV(String regex, int keyGroup, int valueGroup) {
+ this.pattern = Pattern.compile(regex);
+ this.keyGroup = keyGroup;
+ this.valueGroup = valueGroup;
+ }
+
+ public PCollection<KV<String, String>> apply(PCollection<String> in) {
+ return in.apply(
+ ParDo.of(new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.ReplaceAll<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with all Strings that matched the
+ * Regex being replaced with the replacement string.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will be output without changes. If it does
+ * match a portion of the line, all portions matching the Regex will be replaced
+ * with the replacement String.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(Regex.replaceAll("myregex", "myreplacement"));
+ * }
+ * </pre>
+ */
+ public static class ReplaceAll
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ String replacement;
+
+ public ReplaceAll(String regex, String replacement) {
+ this.pattern = Pattern.compile(regex);
+ this.replacement = replacement;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceAll(replacement));
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.ReplaceFirst<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with the first Strings that matched the
+ * Regex being replaced with the replacement string.
+ *
+ * <p>
+ * This transform runs a Regex on the entire input line. If a portion of the
+ * line does not match the Regex, the line will be output without changes. If it does
+ * match a portion of the line, the first portion matching the Regex will be replaced
+ * with the replacement String.
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(Regex.replaceFirst("myregex", "myreplacement"));
+ * }
+ * </pre>
+ */
+ public static class ReplaceFirst
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ String replacement;
+
+ public ReplaceFirst(String regex, String replacement) {
+ this.pattern = Pattern.compile(regex);
+ this.replacement = replacement;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceFirst(replacement));
+ }
+ }));
+ }
+ }
+
+ /**
+ * {@code Regex.Split<String>} takes a {@code PCollection<String>} and
+ * returns a {@code PCollection<String>} with the input string split into
+ * individual items in a list. Each item is then output as a separate string.
+ *
+ * <p>
+ * This transform runs a Regex as part of a splint the entire input line. The split
+ * gives back an array of items. Each item is output as a separate item in the
+ * {@code PCollection<String>}.
+ * </p>
+ *
+ * <p>
+ * Depending on the Regex, a split can be an empty or
+ * "" string. You can pass in a parameter if you want empty strings or not.
+ * </p>
+ *
+ * <p>
+ * Example of use:
+ * <pre>
+ * {@code
+ * PCollection<String> words = ...;
+ * PCollection<String> values =
+ * words.apply(Regex.split("\W*"));
+ * }
+ * </pre>
+ */
+ public static class Split
+ extends PTransform<PCollection<String>, PCollection<String>> {
+ Pattern pattern;
+ boolean outputEmpty;
+
+ public Split(String regex, boolean outputEmpty) {
+ this.pattern = Pattern.compile(regex);
+ this.outputEmpty = outputEmpty;
+ }
+
+ public PCollection<String> apply(PCollection<String> in) {
+ return in.apply(ParDo.of(new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ String[] items = pattern.split(c.element());
+
+ for (String item : items) {
+ if (outputEmpty || !item.isEmpty()) {
+ c.output(item);
+ }
+ }
+ }
+ }));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6954abe4/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
deleted file mode 100644
index bd7848a..0000000
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/RegexTransform.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.transforms;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-/**
- * {@code PTransorm}s to use Regular Expressions to process elements in a
- * {@link PCollection}.
- *
- * <p>
- * {@link RegexTransform#matches(String, int)} can be used to see if an entire line matches
- * a Regex. {@link RegexTransform#matchesKV(String, int, int)} can be used to see if an entire
- * line matches a Regex and output certain groups as a {@link KV}.
- * </p>
- * <p>
- * {@link RegexTransform#find(String, int)} can be used to see if a portion of a line
- * matches a Regex. {@link RegexTransform#matchesKV(String, int, int)} can be used to see if a
- * portion of a line matches a Regex and output certain groups as a {@link KV}.
- * </p>
- * <p>
- * Lines that do not match the Regex will not be output.
- * </p>
- */
-public class RegexTransform {
- private RegexTransform() {
- // do not instantiate
- }
-
- /**
- * Returns a {@link RegexTransform.Matches} {@link PTransform} that checks if
- * the entire line matches the Regex. Returns the entire line (group 0) as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- */
- public static Matches matches(String regex) {
- return matches(regex, 0);
- }
-
- /**
- * Returns a {@link RegexTransform.Matches} {@link PTransform} that checks if
- * the entire line matches the Regex. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param group
- * The Regex group to return as a PCollection
- */
- public static Matches matches(String regex, int group) {
- return new Matches(regex, group);
- }
-
- /**
- * Returns a {@link RegexTransform.MatchesKV} {@link PTransform} that checks
- * if the entire line matches the Regex. Returns the specified groups as the
- * key and value as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param keyGroup
- * The Regex group to use as the key
- * @param valueGroup
- * The Regex group to use the value
- */
- public static MatchesKV matchesKV(String regex, int keyGroup,
- int valueGroup) {
- return new MatchesKV(regex, keyGroup, valueGroup);
- }
-
- /**
- * Returns a {@link RegexTransform.Find} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the entire line (group 0) as
- * a {@link PCollection}.
- * @param regex
- * The regular expression to run
- */
- public static Find find(String regex) {
- return find(regex, 0);
- }
-
- /**
- * Returns a {@link RegexTransform.Find} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param group
- * The Regex group to return as a PCollection
- */
- public static Find find(String regex, int group) {
- return new Find(regex, group);
- }
-
- /**
- * Returns a {@link RegexTransform.FindKV} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the specified groups as the
- * key and value as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param keyGroup
- * The Regex group to use as the key
- * @param valueGroup
- * The Regex group to use the value
- */
- public static FindKV findKV(String regex, int keyGroup, int valueGroup) {
- return new FindKV(regex, keyGroup, valueGroup);
- }
-
- /**
- * Returns a {@link RegexTransform.ReplaceAll} {@link PTransform} that checks if a
- * portion of the line matches the Regex and replaces all matches with the replacement
- * String. Returns the group as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param replacement
- * The string to be substituted for each match
- */
- public static ReplaceAll replaceAll(String regex, String replacement) {
- return new ReplaceAll(regex, replacement);
- }
-
- /**
- * Returns a {@link RegexTransform.ReplaceAll} {@link PTransform} that checks if a
- * portion of the line matches the Regex and replaces the first match with the replacement
- * String. Returns the group as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param replacement
- * The string to be substituted for each match
- */
- public static ReplaceFirst replaceFirst(String regex, String replacement) {
- return new ReplaceFirst(regex, replacement);
- }
-
- /**
- * Returns a {@link RegexTransform.Split} {@link PTransform} that splits a string
- * on the regular expression and then outputs each item. It will not output empty
- * items. Returns the group as a {@link PCollection}.
- * a {@link PCollection}.
- * @param regex
- * The regular expression to run
- */
- public static Split split(String regex) {
- return split(regex, false);
- }
-
- /**
- * Returns a {@link RegexTransform.Split} {@link PTransform} that splits a string
- * on the regular expression and then outputs each item. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param outputEmpty
- * Should empty be output. True to output empties and false if not.
- */
- public static Split split(String regex, boolean outputEmpty) {
- return new Split(regex, outputEmpty);
- }
-
- /**
- * {@code RegexTransform.Matches<String>} takes a {@code PCollection<String>}
- * and returns a {@code PCollection<String>} representing the value
- * extracted from the Regex groups of the input {@code PCollection}
- * to the number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If the entire line
- * does not match the Regex, the line will not be output. If it does match the
- * entire line, the group in the Regex will be used. The output will be the
- * Regex group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<String> values =
- * words.apply(RegexTransform.matches("myregex (mygroup)", 1));
- * }
- * </pre>
- */
- public static class Matches
- extends PTransform<PCollection<String>, PCollection<String>> {
- Pattern pattern;
- int group;
-
- public Matches(String regex, int group) {
- this.pattern = Pattern.compile(regex);
- this.group = group;
- }
-
- public PCollection<String> apply(PCollection<String> in) {
- return in
- .apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.matches()) {
- c.output(m.group(group));
- }
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.MatchesKV<KV<String, String>>} takes a
- * {@code PCollection<String>} and returns a
- * {@code PCollection<KV<String, String>>} representing the key and value
- * extracted from the Regex groups of the input {@code PCollection} to the
- * number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If the entire line
- * does not match the Regex, the line will not be output. If it does match the
- * entire line, the groups in the Regex will be used. The key will be the
- * key's group and the value will be the value's group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<KV<String, String>> keysAndValues =
- * words.apply(RegexTransform.matchesKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
- * }
- * </pre>
- */
- public static class MatchesKV
- extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
- Pattern pattern;
- int keyGroup, valueGroup;
-
- public MatchesKV(String regex, int keyGroup, int valueGroup) {
- this.pattern = Pattern.compile(regex);
- this.keyGroup = keyGroup;
- this.valueGroup = valueGroup;
- }
-
- public PCollection<KV<String, String>> apply(PCollection<String> in) {
- return in.apply(ParDo
- .of(new DoFn<String, KV<String, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
- }
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.Find<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} representing the value extracted
- * from the Regex groups of the input {@code PCollection} to
- * the number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will not be output. If it does
- * match a portion of the line, the group in the Regex will be used. The
- * output will be the Regex group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<String> values =
- * words.apply(RegexTransform.find("myregex (mygroup)", 1));
- * }
- * </pre>
- */
- public static class Find
- extends PTransform<PCollection<String>, PCollection<String>> {
- Pattern pattern;
- int group;
-
- public Find(String regex, int group) {
- this.pattern = Pattern.compile(regex);
- this.group = group;
- }
-
- public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(m.group(group));
- }
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.MatchesKV<KV<String, String>>} takes a
- * {@code PCollection<String>} and returns a
- * {@code PCollection<KV<String, String>>} representing the key and value
- * extracted from the Regex groups of the input {@code PCollection} to the
- * number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will not be output. If it does
- * match a portion of the line, the groups in the Regex will be used. The key
- * will be the key's group and the value will be the value's group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<KV<String, String>> keysAndValues =
- * words.apply(RegexTransform.findKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
- * }
- * </pre>
- */
- public static class FindKV
- extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
- Pattern pattern;
- int keyGroup, valueGroup;
-
- public FindKV(String regex, int keyGroup, int valueGroup) {
- this.pattern = Pattern.compile(regex);
- this.keyGroup = keyGroup;
- this.valueGroup = valueGroup;
- }
-
- public PCollection<KV<String, String>> apply(PCollection<String> in) {
- return in.apply(
- ParDo.of(new DoFn<String, KV<String, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
- }
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.ReplaceAll<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with all Strings that matched the
- * Regex being replaced with the replacement string.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will be output without changes. If it does
- * match a portion of the line, all portions matching the Regex will be replaced
- * with the replacement String.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<String> values =
- * words.apply(RegexTransform.replaceAll("myregex", "myreplacement"));
- * }
- * </pre>
- */
- public static class ReplaceAll
- extends PTransform<PCollection<String>, PCollection<String>> {
- Pattern pattern;
- String replacement;
-
- public ReplaceAll(String regex, String replacement) {
- this.pattern = Pattern.compile(regex);
- this.replacement = replacement;
- }
-
- public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
- c.output(m.replaceAll(replacement));
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.ReplaceFirst<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with the first Strings that matched the
- * Regex being replaced with the replacement string.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will be output without changes. If it does
- * match a portion of the line, the first portion matching the Regex will be replaced
- * with the replacement String.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<String> values =
- * words.apply(RegexTransform.replaceFirst("myregex", "myreplacement"));
- * }
- * </pre>
- */
- public static class ReplaceFirst
- extends PTransform<PCollection<String>, PCollection<String>> {
- Pattern pattern;
- String replacement;
-
- public ReplaceFirst(String regex, String replacement) {
- this.pattern = Pattern.compile(regex);
- this.replacement = replacement;
- }
-
- public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
- c.output(m.replaceFirst(replacement));
- }
- }));
- }
- }
-
- /**
- * {@code RegexTransform.Split<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with the input string split into
- * individual items in a list. Each item is then output as a separate string.
- *
- * <p>
- * This transform runs a Regex as part of a splint the entire input line. The split
- * gives back an array of items. Each item is output as a separate item in the
- * {@code PCollection<String>}.
- * </p>
- *
- * <p>
- * Depending on the Regex, a split can be an empty or
- * "" string. You can pass in a parameter if you want empty strings or not.
- * </p>
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
- * PCollection<String> words = ...;
- * PCollection<String> values =
- * words.apply(RegexTransform.split("\W*"));
- * }
- * </pre>
- */
- public static class Split
- extends PTransform<PCollection<String>, PCollection<String>> {
- Pattern pattern;
- boolean outputEmpty;
-
- public Split(String regex, boolean outputEmpty) {
- this.pattern = Pattern.compile(regex);
- this.outputEmpty = outputEmpty;
- }
-
- public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- String[] items = pattern.split(c.element());
-
- for (String item : items) {
- if (outputEmpty || !item.isEmpty()) {
- c.output(item);
- }
- }
- }
- }));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6954abe4/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
new file mode 100644
index 0000000..71f080e
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms;
+
+import java.io.Serializable;
+
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for {@link Regex}.
+ */
+@RunWith(JUnit4.class)
+public class RegexTest implements Serializable {
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFind() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("aj", "xj", "yj", "zj"))
+ .apply(Regex.find("[xyz]"));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFindGroup() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("aj", "xj", "yj", "zj"))
+ .apply(Regex.find("([xyz])", 1));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testFindNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "b", "c", "d"))
+ .apply(Regex.find("[xyz]"));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVFind() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("a b c"))
+ .apply(Regex.findKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVFindNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("x y z"))
+ .apply(Regex.findKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatches() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "x", "y", "z"))
+ .apply(Regex.matches("[xyz]"));
+
+ PAssert.that(output).containsInAnyOrder("x", "y", "z");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchesNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "b", "c", "d"))
+ .apply(Regex.matches("[xyz]"));
+
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testMatchesGroup() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("a", "x xxx", "x yyy", "x zzz"))
+ .apply(Regex.matches("x ([xyz]*)", 1));
+
+ PAssert.that(output).containsInAnyOrder("xxx", "yyy", "zzz");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVMatches() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("a b c"))
+ .apply(Regex.matchesKV("a (b) (c)", 1, 2));
+
+ PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testKVMatchesNone() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<KV<String, String>> output = p
+ .apply(Create.of("x y z"))
+ .apply(Regex.matchesKV("a (b) (c)", 1, 2));
+ PAssert.that(output).empty();
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceAll() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("xj", "yj", "zj"))
+ .apply(Regex.replaceAll("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("newj", "newj", "newj");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceAllMixed() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("abc", "xj", "yj", "zj", "def"))
+ .apply(Regex.replaceAll("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("abc", "newj", "newj", "newj", "def");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceFirst() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("xjx", "yjy", "zjz"))
+ .apply(Regex.replaceFirst("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("newjx", "newjy", "newjz");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testReplaceFirstMixed() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("abc", "xjx", "yjy", "zjz", "def"))
+ .apply(Regex.replaceFirst("[xyz]", "new"));
+
+ PAssert.that(output).containsInAnyOrder("abc", "newjx", "newjy", "newjz", "def");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplits() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\W+"));
+
+ PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
+ "fox", "jumps", "over", "the", "lazy", "dog");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplitsWithEmpty() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\s", true));
+
+ String[] outputStr = "The quick brown fox jumps over the lazy dog".split("\\s");
+
+ PAssert.that(output).containsInAnyOrder("The", "", "quick", "brown", "", "",
+ "fox", "jumps", "over", "", "", "", "the", "lazy", "dog");
+ p.run();
+ }
+
+ @Test
+ @Category(NeedsRunner.class)
+ public void testSplitsWithoutEmpty() {
+ TestPipeline p = TestPipeline.create();
+
+ PCollection<String> output = p
+ .apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\s", false));
+
+ PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
+ "fox", "jumps", "over", "the", "lazy", "dog");
+ p.run();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/6954abe4/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
deleted file mode 100644
index 63d02d7..0000000
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTransformTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.transforms;
-
-import java.io.Serializable;
-
-import org.apache.beam.sdk.testing.NeedsRunner;
-import org.apache.beam.sdk.testing.PAssert;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.PCollection;
-
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/**
- * Tests for {@link RegexTransform}.
- */
-@RunWith(JUnit4.class)
-public class RegexTransformTest implements Serializable {
- @Test
- @Category(NeedsRunner.class)
- public void testFind() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("aj", "xj", "yj", "zj"))
- .apply(RegexTransform.find("[xyz]"));
-
- PAssert.that(output).containsInAnyOrder("x", "y", "z");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testFindGroup() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("aj", "xj", "yj", "zj"))
- .apply(RegexTransform.find("([xyz])", 1));
-
- PAssert.that(output).containsInAnyOrder("x", "y", "z");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testFindNone() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("a", "b", "c", "d"))
- .apply(RegexTransform.find("[xyz]"));
-
- PAssert.that(output).empty();
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testKVFind() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<KV<String, String>> output = p
- .apply(Create.of("a b c"))
- .apply(RegexTransform.findKV("a (b) (c)", 1, 2));
-
- PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testKVFindNone() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<KV<String, String>> output = p
- .apply(Create.of("x y z"))
- .apply(RegexTransform.findKV("a (b) (c)", 1, 2));
-
- PAssert.that(output).empty();
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testMatches() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("a", "x", "y", "z"))
- .apply(RegexTransform.matches("[xyz]"));
-
- PAssert.that(output).containsInAnyOrder("x", "y", "z");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testMatchesNone() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("a", "b", "c", "d"))
- .apply(RegexTransform.matches("[xyz]"));
-
- PAssert.that(output).empty();
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testMatchesGroup() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("a", "x xxx", "x yyy", "x zzz"))
- .apply(RegexTransform.matches("x ([xyz]*)", 1));
-
- PAssert.that(output).containsInAnyOrder("xxx", "yyy", "zzz");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testKVMatches() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<KV<String, String>> output = p
- .apply(Create.of("a b c"))
- .apply(RegexTransform.matchesKV("a (b) (c)", 1, 2));
-
- PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testKVMatchesNone() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<KV<String, String>> output = p
- .apply(Create.of("x y z"))
- .apply(RegexTransform.matchesKV("a (b) (c)", 1, 2));
- PAssert.that(output).empty();
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReplaceAll() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("xj", "yj", "zj"))
- .apply(RegexTransform.replaceAll("[xyz]", "new"));
-
- PAssert.that(output).containsInAnyOrder("newj", "newj", "newj");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReplaceAllMixed() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("abc", "xj", "yj", "zj", "def"))
- .apply(RegexTransform.replaceAll("[xyz]", "new"));
-
- PAssert.that(output).containsInAnyOrder("abc", "newj", "newj", "newj", "def");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReplaceFirst() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("xjx", "yjy", "zjz"))
- .apply(RegexTransform.replaceFirst("[xyz]", "new"));
-
- PAssert.that(output).containsInAnyOrder("newjx", "newjy", "newjz");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testReplaceFirstMixed() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("abc", "xjx", "yjy", "zjz", "def"))
- .apply(RegexTransform.replaceFirst("[xyz]", "new"));
-
- PAssert.that(output).containsInAnyOrder("abc", "newjx", "newjy", "newjz", "def");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testSplits() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(RegexTransform.split("\\W+"));
-
- PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
- "fox", "jumps", "over", "the", "lazy", "dog");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testSplitsWithEmpty() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(RegexTransform.split("\\s", true));
-
- String[] outputStr = "The quick brown fox jumps over the lazy dog".split("\\s");
-
- PAssert.that(output).containsInAnyOrder("The", "", "quick", "brown", "", "",
- "fox", "jumps", "over", "", "", "", "the", "lazy", "dog");
- p.run();
- }
-
- @Test
- @Category(NeedsRunner.class)
- public void testSplitsWithoutEmpty() {
- TestPipeline p = TestPipeline.create();
-
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(RegexTransform.split("\\s", false));
-
- PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
- "fox", "jumps", "over", "the", "lazy", "dog");
- p.run();
- }
-}
[10/50] incubator-beam git commit: [BEAM-889] Let Spark handle the
user-provided checkpointDir, but warn if not a reliable fs.
Posted by ke...@apache.org.
[BEAM-889] Let Spark handle the user-provided checkpointDir, but warn if not a reliable fs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/90a75d1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/90a75d1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/90a75d1f
Branch: refs/heads/gearpump-runner
Commit: 90a75d1fb0706ec4cc25a9eeeca8ade1b3b7de28
Parents: 46fbfe0
Author: Sela <an...@paypal.com>
Authored: Thu Nov 3 18:22:20 2016 +0200
Committer: Sela <an...@paypal.com>
Committed: Fri Nov 4 23:59:40 2016 +0200
----------------------------------------------------------------------
.../runners/spark/SparkPipelineOptions.java | 3 +--
.../SparkRunnerStreamingContextFactory.java | 23 +++++---------------
.../streaming/EmptyStreamAssertionTest.java | 3 +--
.../streaming/FlattenStreamingTest.java | 6 ++---
.../streaming/KafkaStreamingTest.java | 6 ++---
.../ResumeFromCheckpointStreamingTest.java | 3 +--
.../streaming/SimpleStreamingWordCountTest.java | 3 +--
.../utils/TestOptionsForStreaming.java | 12 +++++-----
8 files changed, 19 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 08e14fe..4eada35 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -77,8 +77,7 @@ public interface SparkPipelineOptions extends PipelineOptions, StreamingOptions,
class TmpCheckpointDirFactory implements DefaultValueFactory<String> {
@Override
public String create(PipelineOptions options) {
- SparkPipelineOptions sparkPipelineOptions = options.as(SparkPipelineOptions.class);
- return "file:///tmp/" + sparkPipelineOptions.getJobName();
+ return "/tmp/" + options.as(SparkPipelineOptions.class).getJobName();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
index 2378788..a670f61 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/SparkRunnerStreamingContextFactory.java
@@ -20,11 +20,6 @@ package org.apache.beam.runners.spark.translation.streaming;
import static com.google.common.base.Preconditions.checkArgument;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Arrays;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.SparkRunner;
import org.apache.beam.runners.spark.translation.SparkContextFactory;
@@ -48,7 +43,7 @@ import org.slf4j.LoggerFactory;
public class SparkRunnerStreamingContextFactory implements JavaStreamingContextFactory {
private static final Logger LOG =
LoggerFactory.getLogger(SparkRunnerStreamingContextFactory.class);
- private static final Iterable<String> KNOWN_RELIABLE_FS = Arrays.asList("hdfs", "s3", "gs");
+ private static final String KNOWN_RELIABLE_FS_PATTERN = "^(hdfs|s3|gs)";
private final Pipeline pipeline;
private final SparkPipelineOptions options;
@@ -83,19 +78,11 @@ public class SparkRunnerStreamingContextFactory implements JavaStreamingContextF
// set checkpoint dir.
String checkpointDir = options.getCheckpointDir();
- LOG.info("Checkpoint dir set to: {}", checkpointDir);
- try {
- // validate checkpoint dir and warn if not of a known durable filesystem.
- URL checkpointDirUrl = new URL(checkpointDir);
- if (!Iterables.any(KNOWN_RELIABLE_FS, Predicates.equalTo(checkpointDirUrl.getProtocol()))) {
- LOG.warn("Checkpoint dir URL {} does not match a reliable filesystem, in case of failures "
- + "this job may not recover properly or even at all.", checkpointDirUrl);
- }
- } catch (MalformedURLException e) {
- throw new RuntimeException("Failed to form checkpoint dir URL. CheckpointDir should be in "
- + "the form of hdfs:///path/to/dir or other reliable fs protocol, "
- + "or file:///path/to/dir for local mode.", e);
+ if (!checkpointDir.matches(KNOWN_RELIABLE_FS_PATTERN)) {
+ LOG.warn("The specified checkpoint dir {} does not match a reliable filesystem so in case "
+ + "of failures this job may not recover properly or even at all.", checkpointDir);
}
+ LOG.info("Checkpoint dir set to: {}", checkpointDir);
jssc.checkpoint(checkpointDir);
// register listeners.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
index 4f2a7c6..3e95b4d 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/EmptyStreamAssertionTest.java
@@ -55,8 +55,7 @@ public class EmptyStreamAssertionTest implements Serializable {
@Test
public void testAssertion() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
Duration windowDuration = new Duration(options.getBatchIntervalMillis());
Pipeline pipeline = Pipeline.create(options);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index e6872f1..319b5e9 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -61,8 +61,7 @@ public class FlattenStreamingTest {
@Test
public void testFlattenUnbounded() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
Pipeline p = Pipeline.create(options);
PCollection<String> w1 =
@@ -81,8 +80,7 @@ public class FlattenStreamingTest {
@Test
public void testFlattenBoundedUnbounded() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
Pipeline p = Pipeline.create(options);
PCollection<String> w1 =
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index fe2d04e..f01059f 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -74,8 +74,7 @@ public class KafkaStreamingTest {
@Test
public void testEarliest2Topics() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
// It seems that the consumer's first "position" lookup (in unit test) takes +200 msec,
// so to be on the safe side we'll set to 750 msec.
options.setMinReadTimeMillis(750L);
@@ -122,8 +121,7 @@ public class KafkaStreamingTest {
@Test
public void testLatest() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
//--- setup
final String topic = "topic";
// messages.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
index ca0b668..34ffbe2 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/ResumeFromCheckpointStreamingTest.java
@@ -108,8 +108,7 @@ public class ResumeFromCheckpointStreamingTest {
@Test
public void testRun() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
// It seems that the consumer's first "position" lookup (in unit test) takes +200 msec,
// so to be on the safe side we'll set to 750 msec.
options.setMinReadTimeMillis(750L);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
index 4bc9a3d..edba507 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
@@ -62,8 +62,7 @@ public class SimpleStreamingWordCountTest implements Serializable {
@Test
public void testFixedWindows() throws Exception {
- SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(
- checkpointParentDir.newFolder(getClass().getSimpleName()));
+ SparkPipelineOptions options = commonOptions.withTmpCheckpointDir(checkpointParentDir);
// override defaults
options.setBatchIntervalMillis(BATCH_INTERVAL.getMillis());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/90a75d1f/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/TestOptionsForStreaming.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/TestOptionsForStreaming.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/TestOptionsForStreaming.java
index d695df0..2861d9f 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/TestOptionsForStreaming.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/utils/TestOptionsForStreaming.java
@@ -19,12 +19,13 @@
package org.apache.beam.runners.spark.translation.streaming.utils;
-import java.io.File;
-import java.net.MalformedURLException;
+import java.io.IOException;
import org.apache.beam.runners.spark.SparkPipelineOptions;
import org.apache.beam.runners.spark.SparkRunner;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+
/**
@@ -41,11 +42,10 @@ public class TestOptionsForStreaming extends ExternalResource {
options.setTimeout(1000L);
}
- public SparkPipelineOptions withTmpCheckpointDir(File checkpointDir)
- throws MalformedURLException {
+ public SparkPipelineOptions withTmpCheckpointDir(TemporaryFolder parent)
+ throws IOException {
// tests use JUnit's TemporaryFolder path in the form of: /.../junit/...
- // so need to add the missing protocol.
- options.setCheckpointDir(checkpointDir.toURI().toURL().toString());
+ options.setCheckpointDir(parent.newFolder(options.getJobName()).toURI().toURL().toString());
return options;
}
[12/50] incubator-beam git commit: Enable javadoc for all modules by
default
Posted by ke...@apache.org.
Enable javadoc for all modules by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/aae65db6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/aae65db6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/aae65db6
Branch: refs/heads/gearpump-runner
Commit: aae65db663c27de5b9a21adedc2110cc21cd4061
Parents: 14e093a
Author: Dan Halperin <dh...@google.com>
Authored: Fri Nov 4 15:28:43 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Nov 4 18:29:21 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 68 ------------------------
examples/java8/pom.xml | 8 ---
pom.xml | 11 +++-
runners/core-java/pom.xml | 23 --------
runners/direct-java/pom.xml | 61 ---------------------
runners/flink/examples/pom.xml | 8 ---
runners/flink/runner/pom.xml | 8 ---
runners/google-cloud-dataflow-java/pom.xml | 70 -------------------------
runners/spark/pom.xml | 4 --
sdks/java/core/pom.xml | 70 -------------------------
sdks/java/io/jdbc/pom.xml | 4 --
sdks/java/io/kafka/pom.xml | 7 ---
sdks/java/io/kinesis/pom.xml | 7 ---
sdks/java/io/mongodb/pom.xml | 4 --
14 files changed, 10 insertions(+), 343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 12a114f..857373a 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -285,74 +285,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <windowtitle>Apache Beam Examples</windowtitle>
- <doctitle>Apache Beam Examples</doctitle>
-
- <subpackages>org.apache.beam.examples</subpackages>
- <additionalparam>-exclude org.apache.beam.sdk.runners.worker:org.apache.beam.sdk.runners.dataflow:org.apache.beam.sdk.util ${beam.javadoc_opts}</additionalparam>
- <use>false</use>
- <quiet>true</quiet>
- <bottom><![CDATA[<br>]]></bottom>
-
- <offlineLinks>
- <!-- The Dataflow SDK docs -->
- <offlineLink>
- <url>https://cloud.google.com/dataflow/java-sdk/JavaDoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/dataflow-sdk-docs</location>
- </offlineLink>
- <!-- Other dependencies -->
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/apiclient-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://avro.apache.org/docs/${avro.version}/api/java/</url>
- <location>${basedir}/../../sdks/java/javadoc/avro-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/</url>
- <location>${basedir}/../../sdks/java/javadoc/bq-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://cloud.google.com/datastore/docs/apis/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/datastore-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://docs.guava-libraries.googlecode.com/git-history/release18/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/guava-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-annotations/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-annotations-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-databind/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-databind-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://www.joda.org/joda-time/apidocs</url>
- <location>${basedir}/../../sdks/java/javadoc/joda-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-oauth-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/oauth-docs</location>
- </offlineLink>
- </offlineLinks>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>jar</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 18257d9..0a5dcb0 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -120,14 +120,6 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:missing</additionalparam>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 14c38fd..77e87ba 100644
--- a/pom.xml
+++ b/pom.xml
@@ -160,6 +160,9 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <additionalparam>${beam.javadoc_opts}</additionalparam>
+ </configuration>
<executions>
<execution>
<id>javadoc</id>
@@ -212,6 +215,11 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
@@ -222,9 +230,10 @@
<jdk>[1.8,)</jdk>
</activation>
<properties>
- <beam.javadoc_opts>-Xdoclint:-missing</beam.javadoc_opts>
+ <beam.javadoc_opts>-Xdoclint:all -Xdoclint:-missing</beam.javadoc_opts>
</properties>
</profile>
+
<profile>
<id>java7-packageinfo</id>
<activation>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index fa64660..3d8a84b 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -65,29 +65,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <windowtitle>Beam Runners Core utilities ${project.version} API</windowtitle>
- <doctitle>Beam Runners Core utilities for Java, version ${project.version}</doctitle>
- <overview>../javadoc/overview.html</overview>
-
- <subpackages>org.apache.beam.runners.core</subpackages>
- <use>false</use>
- <quiet>true</quiet>
- <bottom><![CDATA[<br>]]></bottom>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>jar</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index b781c6e..64e9b5a 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -94,67 +94,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <windowtitle>Apache Beam Direct Runner ${project.version}</windowtitle>
- <doctitle>Apache Beam Direct Runner, version ${project.version}</doctitle>
-
- <subpackages>org.apache.beam.runners.direct</subpackages>
- <use>false</use>
- <quiet>true</quiet>
- <bottom><![CDATA[<br>]]></bottom>
-
- <offlineLinks>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/apiclient-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://avro.apache.org/docs/${avro.version}/api/java/</url>
- <location>${basedir}/../../sdks/java/javadoc/avro-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/</url>
- <location>${basedir}/../../sdks/java/javadoc/bq-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://cloud.google.com/datastore/docs/apis/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/datastore-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://docs.guava-libraries.googlecode.com/git-history/release19/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/guava-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://hamcrest.org/JavaHamcrest/javadoc/1.3/</url>
- <location>${basedir}/../../sdks/java/javadoc/hamcrest-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-annotations/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-annotations-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-databind/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-databind-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://www.joda.org/joda-time/apidocs</url>
- <location>${basedir}/../../sdks/java/javadoc/joda-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://junit.sourceforge.net/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/junit-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-oauth-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/oauth-docs</location>
- </offlineLink>
- </offlineLinks>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/flink/examples/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml
index 409d77b..051ec2d 100644
--- a/runners/flink/examples/pom.xml
+++ b/runners/flink/examples/pom.xml
@@ -110,14 +110,6 @@
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:missing</additionalparam>
- </configuration>
- </plugin>
-
- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<configuration>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/flink/runner/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/runner/pom.xml b/runners/flink/runner/pom.xml
index 928889e..9729316 100644
--- a/runners/flink/runner/pom.xml
+++ b/runners/flink/runner/pom.xml
@@ -242,14 +242,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:missing</additionalparam>
- </configuration>
- </plugin>
-
<!-- Integration Tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index d8f1b82..0a2c9f6 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -85,76 +85,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <windowtitle>Google Cloud Dataflow Java Runner ${project.version}</windowtitle>
- <doctitle>Google Cloud Dataflow Runner for Java, version ${project.version}</doctitle>
-
- <subpackages>org.apache.beam.runners.dataflow</subpackages>
- <additionalparam>-exclude org.apache.beam.sdk.runners.dataflow.internal:org.apache.beam.sdk.runners.dataflow.testing:org.apache.beam.sdk.runners.dataflow.util ${beam.javadoc_opts}</additionalparam>
- <use>false</use>
- <quiet>true</quiet>
- <bottom><![CDATA[<br>]]></bottom>
-
- <offlineLinks>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/apiclient-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://avro.apache.org/docs/${avro.version}/api/java/</url>
- <location>${basedir}/../../sdks/java/javadoc/avro-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/</url>
- <location>${basedir}/../../sdks/java/javadoc/bq-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://cloud.google.com/datastore/docs/apis/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/datastore-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://docs.guava-libraries.googlecode.com/git-history/release19/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/guava-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://hamcrest.org/JavaHamcrest/javadoc/1.3/</url>
- <location>${basedir}/../../sdks/java/javadoc/hamcrest-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-annotations/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-annotations-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-databind/javadoc/2.7/</url>
- <location>${basedir}/../../sdks/java/javadoc/jackson-databind-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://www.joda.org/joda-time/apidocs</url>
- <location>${basedir}/../../sdks/java/javadoc/joda-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://junit.sourceforge.net/javadoc/</url>
- <location>${basedir}/../../sdks/java/javadoc/junit-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-oauth-java-client/reference/1.20.0/</url>
- <location>${basedir}/../../sdks/java/javadoc/oauth-docs</location>
- </offlineLink>
- </offlineLinks>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>jar</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 0edafe8..a3ca296 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -415,10 +415,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 40a511c..7d4d9ab 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -44,71 +44,6 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <windowtitle>Apache Beam SDK ${project.version} API</windowtitle>
- <doctitle>Apache Beam SDK for Java, version ${project.version}</doctitle>
- <overview>../javadoc/overview.html</overview>
-
- <subpackages>org.apache.beam.sdk</subpackages>
- <additionalparam>-exclude
- org.apache.beam.sdk.runners.worker:org.apache.beam.sdk.runners.dataflow:org.apache.beam.sdk.util:org.apache.beam.sdk.runners.inprocess
- ${beam.javadoc_opts}</additionalparam>
- <use>false</use>
- <quiet>true</quiet>
- <bottom><![CDATA[<br>]]></bottom>
-
- <offlineLinks>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.20.0/</url>
- <location>${basedir}/../javadoc/apiclient-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://avro.apache.org/docs/${avro.version}/api/java/</url>
- <location>${basedir}/../javadoc/avro-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/</url>
- <location>${basedir}/../javadoc/bq-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://cloud.google.com/datastore/docs/apis/javadoc/</url>
- <location>${basedir}/../javadoc/datastore-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://docs.guava-libraries.googlecode.com/git-history/release19/javadoc/</url>
- <location>${basedir}/../javadoc/guava-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://hamcrest.org/JavaHamcrest/javadoc/1.3/</url>
- <location>${basedir}/../javadoc/hamcrest-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-annotations/javadoc/2.7/</url>
- <location>${basedir}/../javadoc/jackson-annotations-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://fasterxml.github.io/jackson-databind/javadoc/2.7/</url>
- <location>${basedir}/../javadoc/jackson-databind-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://www.joda.org/joda-time/apidocs</url>
- <location>${basedir}/../javadoc/joda-docs</location>
- </offlineLink>
- <offlineLink>
- <url>http://junit.sourceforge.net/javadoc/</url>
- <location>${basedir}/../javadoc/junit-docs</location>
- </offlineLink>
- <offlineLink>
- <url>https://developers.google.com/api-client-library/java/google-oauth-java-client/reference/1.20.0/</url>
- <location>${basedir}/../javadoc/oauth-docs</location>
- </offlineLink>
- </offlineLinks>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<!-- Set testSourceDirectory in order to exclude generated-test-sources -->
@@ -172,11 +107,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/sdks/java/io/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index a65ed46..2eb24bd 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -48,10 +48,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/sdks/java/io/kafka/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index 184c1c9..2004952 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -52,13 +52,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:missing</additionalparam>
- </configuration>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/sdks/java/io/kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index 76df329..439995d 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -52,13 +52,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <configuration>
- <additionalparam>-Xdoclint:missing</additionalparam>
- </configuration>
- </plugin>
<!-- Integration Tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/aae65db6/sdks/java/io/mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index d674e63..d0ae6ac 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -48,10 +48,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
</plugins>
</build>
[15/50] incubator-beam git commit: move source jar creation into
release profile
Posted by ke...@apache.org.
move source jar creation into release profile
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/68f26386
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/68f26386
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/68f26386
Branch: refs/heads/gearpump-runner
Commit: 68f26386e6ba7c5a883ba34accccff4618234811
Parents: 5864a38
Author: Dan Halperin <dh...@google.com>
Authored: Sat Nov 5 01:45:57 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 08:45:50 2016 -0800
----------------------------------------------------------------------
examples/java/pom.xml | 6 ------
examples/java8/pom.xml | 6 ------
pom.xml | 5 +++++
runners/core-java/pom.xml | 6 ------
runners/direct-java/pom.xml | 6 ------
runners/flink/examples/pom.xml | 4 ----
runners/flink/runner/pom.xml | 4 ----
runners/google-cloud-dataflow-java/pom.xml | 6 ------
runners/spark/pom.xml | 4 ----
sdks/java/core/pom.xml | 6 ------
sdks/java/extensions/join-library/pom.xml | 4 ----
sdks/java/extensions/sorter/pom.xml | 4 ----
sdks/java/io/google-cloud-platform/pom.xml | 4 ----
sdks/java/io/hdfs/pom.xml | 4 ----
sdks/java/io/jdbc/pom.xml | 4 ----
sdks/java/io/jms/pom.xml | 4 ----
sdks/java/io/kafka/pom.xml | 4 ----
sdks/java/io/kinesis/pom.xml | 4 ----
sdks/java/io/mongodb/pom.xml | 4 ----
sdks/java/java8tests/pom.xml | 6 ------
sdks/java/microbenchmarks/pom.xml | 5 -----
21 files changed, 5 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 857373a..5d69bfe 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -277,12 +277,6 @@
</configuration>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 0a5dcb0..72f24ca 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -140,12 +140,6 @@
</configuration>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 77e87ba..c1bd5c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -220,6 +220,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index 3d8a84b..40ebf58 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -57,12 +57,6 @@
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 64e9b5a..8983b1c 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -51,12 +51,6 @@
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/flink/examples/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml
index 051ec2d..dccb5cb 100644
--- a/runners/flink/examples/pom.xml
+++ b/runners/flink/examples/pom.xml
@@ -90,10 +90,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/flink/runner/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/runner/pom.xml b/runners/flink/runner/pom.xml
index 9729316..adcb3de 100644
--- a/runners/flink/runner/pom.xml
+++ b/runners/flink/runner/pom.xml
@@ -223,10 +223,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 0a2c9f6..a5ffe0e 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -77,12 +77,6 @@
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index a3ca296..5e94848 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -411,10 +411,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 7d4d9ab..7906afb 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -99,12 +99,6 @@
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/extensions/join-library/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/join-library/pom.xml b/sdks/java/extensions/join-library/pom.xml
index 16326b0..554792a 100644
--- a/sdks/java/extensions/join-library/pom.xml
+++ b/sdks/java/extensions/join-library/pom.xml
@@ -46,10 +46,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/extensions/sorter/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/pom.xml b/sdks/java/extensions/sorter/pom.xml
index 1a994de..8d1e89d 100644
--- a/sdks/java/extensions/sorter/pom.xml
+++ b/sdks/java/extensions/sorter/pom.xml
@@ -50,10 +50,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/google-cloud-platform/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index fa51072..9f7af52 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -52,10 +52,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hdfs/pom.xml b/sdks/java/io/hdfs/pom.xml
index e79996f..a551455 100644
--- a/sdks/java/io/hdfs/pom.xml
+++ b/sdks/java/io/hdfs/pom.xml
@@ -47,10 +47,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index 2eb24bd..d48ff66 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -38,10 +38,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/jms/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jms/pom.xml b/sdks/java/io/jms/pom.xml
index 0b46154..7063ce9 100644
--- a/sdks/java/io/jms/pom.xml
+++ b/sdks/java/io/jms/pom.xml
@@ -46,10 +46,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/kafka/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index 2004952..1ad31aa 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -37,10 +37,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index 439995d..f874c21 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -37,10 +37,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/io/mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index d0ae6ac..4b100a9 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -38,10 +38,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/java8tests/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/java8tests/pom.xml b/sdks/java/java8tests/pom.xml
index 989fba2..d9dfdce 100644
--- a/sdks/java/java8tests/pom.xml
+++ b/sdks/java/java8tests/pom.xml
@@ -61,12 +61,6 @@
</configuration>
</plugin>
- <!-- Source plugin for generating source and test-source JARs. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/68f26386/sdks/java/microbenchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/microbenchmarks/pom.xml b/sdks/java/microbenchmarks/pom.xml
index 05f06aa..c95309c 100644
--- a/sdks/java/microbenchmarks/pom.xml
+++ b/sdks/java/microbenchmarks/pom.xml
@@ -40,11 +40,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
[46/50] incubator-beam git commit: Revert "Update Watermarks even if
a Reader is empty"
Posted by ke...@apache.org.
Revert "Update Watermarks even if a Reader is empty"
This reverts commit ff7fe07be96de393b763e7b3d213734040aa3795.
Updated test appears to be broken.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e2856fbf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e2856fbf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e2856fbf
Branch: refs/heads/gearpump-runner
Commit: e2856fbf076d34b7856391eafdfcfeb71bc6d7b2
Parents: 339dee9
Author: Thomas Groh <tg...@google.com>
Authored: Mon Nov 7 18:02:44 2016 -0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 18:02:44 2016 -0800
----------------------------------------------------------------------
.../direct/UnboundedReadEvaluatorFactory.java | 6 ++----
.../UnboundedReadEvaluatorFactoryTest.java | 19 ++++++-------------
2 files changed, 8 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2856fbf/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
index fb09b3e..e529088 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactory.java
@@ -35,7 +35,6 @@ import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
import org.apache.beam.sdk.io.UnboundedSource.UnboundedReader;
import org.apache.beam.sdk.transforms.AppliedPTransform;
-import org.apache.beam.sdk.transforms.PTransform;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
@@ -144,13 +143,12 @@ class UnboundedReadEvaluatorFactory implements TransformEvaluatorFactory {
// If the reader had no elements available, but the shard is not done, reuse it later
resultBuilder.addUnprocessedElements(
Collections.<WindowedValue<?>>singleton(
- WindowedValue.timestampedValueInGlobalWindow(
+ element.withValue(
UnboundedSourceShard.of(
shard.getSource(),
shard.getDeduplicator(),
reader,
- shard.getCheckpoint()),
- reader.getWatermark())));
+ shard.getCheckpoint()))));
}
} catch (IOException e) {
if (reader != null) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2856fbf/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
index 18c7cec..9a7fec3 100644
--- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
+++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/UnboundedReadEvaluatorFactoryTest.java
@@ -260,7 +260,6 @@ public class UnboundedReadEvaluatorFactoryTest {
(WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>>)
Iterables.getOnlyElement(result.getUnprocessedElements());
secondEvaluator.processElement(residual);
-
TransformResult secondResult = secondEvaluator.finishBundle();
// Sanity check that nothing was output (The test would have to run for more than a day to do
@@ -269,14 +268,11 @@ public class UnboundedReadEvaluatorFactoryTest {
secondOutput.commit(Instant.now()).getElements(),
Matchers.<WindowedValue<Long>>emptyIterable());
- // Test that even though the reader produced no outputs, there is still a residual shard with
- // the updated watermark.
- WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>> unprocessed =
- (WindowedValue<UnboundedSourceShard<Long, TestCheckpointMark>>)
- Iterables.getOnlyElement(secondResult.getUnprocessedElements());
- assertThat(
- unprocessed.getTimestamp(), Matchers.<ReadableInstant>greaterThan(residual.getTimestamp()));
- assertThat(unprocessed.getValue().getExistingReader(), not(nullValue()));
+ // Test that even though the reader produced no outputs, there is still a residual shard.
+ UnboundedSourceShard<Long, TestCheckpointMark> residualShard =
+ (UnboundedSourceShard<Long, TestCheckpointMark>)
+ Iterables.getOnlyElement(secondResult.getUnprocessedElements()).getValue();
+ assertThat(residualShard.getExistingReader(), not(nullValue()));
}
@Test
@@ -381,8 +377,6 @@ public class UnboundedReadEvaluatorFactoryTest {
}
private static class TestUnboundedSource<T> extends UnboundedSource<T, TestCheckpointMark> {
- private static int getWatermarkCalls = 0;
-
static int readerClosedCount;
static int readerAdvancedCount;
private final Coder<T> coder;
@@ -453,8 +447,7 @@ public class UnboundedReadEvaluatorFactoryTest {
@Override
public Instant getWatermark() {
- getWatermarkCalls++;
- return new Instant(index + getWatermarkCalls);
+ return Instant.now();
}
@Override
[40/50] incubator-beam git commit: This closes #1282
Posted by ke...@apache.org.
This closes #1282
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9de9ce69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9de9ce69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9de9ce69
Branch: refs/heads/gearpump-runner
Commit: 9de9ce69fa010f46911ac0f9cfbe4df2e475772b
Parents: ac252a7 9c3e59f
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Nov 7 15:25:04 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:04 2016 -0800
----------------------------------------------------------------------
.../beam/runners/core/SimpleDoFnRunner.java | 12 ++
.../beam/runners/core/SplittableParDo.java | 12 ++
.../org/apache/beam/sdk/transforms/DoFn.java | 20 +++
.../beam/sdk/transforms/DoFnAdapters.java | 22 +++
.../org/apache/beam/sdk/transforms/ParDo.java | 43 +++++-
.../sdk/transforms/reflect/DoFnInvokers.java | 143 +++++++++++++------
.../sdk/transforms/reflect/DoFnSignature.java | 67 ++++++---
.../sdk/transforms/reflect/DoFnSignatures.java | 56 +++++++-
.../beam/sdk/transforms/windowing/WindowFn.java | 12 ++
.../apache/beam/sdk/transforms/ParDoTest.java | 61 ++++++++
.../transforms/reflect/DoFnInvokersTest.java | 100 ++++++++-----
.../DoFnSignaturesSplittableDoFnTest.java | 5 +-
.../transforms/reflect/DoFnSignaturesTest.java | 3 +-
13 files changed, 445 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
[36/50] incubator-beam git commit: Add DoFnInvoker dispatch for State
and Timer parameters
Posted by ke...@apache.org.
Add DoFnInvoker dispatch for State and Timer parameters
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e2db8268
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e2db8268
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e2db8268
Branch: refs/heads/gearpump-runner
Commit: e2db82686008aea224ca5cf1ef1acc2831c46ceb
Parents: c052d2a
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Nov 3 19:18:24 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:03 2016 -0800
----------------------------------------------------------------------
.../beam/runners/core/SimpleDoFnRunner.java | 12 +++
.../beam/runners/core/SplittableParDo.java | 12 +++
.../org/apache/beam/sdk/transforms/DoFn.java | 20 ++++
.../beam/sdk/transforms/DoFnAdapters.java | 22 ++++
.../sdk/transforms/reflect/DoFnInvokers.java | 104 +++++++++++--------
.../transforms/reflect/DoFnInvokersTest.java | 59 ++++++++++-
6 files changed, 187 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index dec9905..3abb06b 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -48,11 +48,13 @@ import org.apache.beam.sdk.transforms.windowing.WindowFn;
import org.apache.beam.sdk.util.ExecutionContext.StepContext;
import org.apache.beam.sdk.util.SideInputReader;
import org.apache.beam.sdk.util.SystemDoFnInternal;
+import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingInternals;
import org.apache.beam.sdk.util.WindowingStrategy;
+import org.apache.beam.sdk.util.state.State;
import org.apache.beam.sdk.util.state.StateInternals;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
@@ -532,6 +534,16 @@ public class SimpleDoFnRunner<InputT, OutputT> implements DoFnRunner<InputT, Out
}
@Override
+ public State state(String timerId) {
+ throw new UnsupportedOperationException("State parameters are not supported.");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("Timer parameters are not supported.");
+ }
+
+ @Override
public WindowingInternals<InputT, OutputT> windowingInternals() {
return new WindowingInternals<InputT, OutputT>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
----------------------------------------------------------------------
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
index 33d0ab7..d8ee1d5 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SplittableParDo.java
@@ -46,9 +46,11 @@ import org.apache.beam.sdk.transforms.windowing.OutputTimeFns;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.KeyedWorkItem;
import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.TimerInternals;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.util.WindowingInternals;
+import org.apache.beam.sdk.util.state.State;
import org.apache.beam.sdk.util.state.StateNamespace;
import org.apache.beam.sdk.util.state.StateNamespaces;
import org.apache.beam.sdk.util.state.StateTag;
@@ -432,6 +434,16 @@ public class SplittableParDo<
public TrackerT restrictionTracker() {
return tracker;
}
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException("State cannot be used with a splittable DoFn");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("Timers cannot be used with a splittable DoFn");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 2b3962e..876dfe2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -381,6 +381,16 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
* the current {@link ProcessElement} call.
*/
<RestrictionT> RestrictionTracker<RestrictionT> restrictionTracker();
+
+ /**
+ * Returns the state cell for the given {@link StateId}.
+ */
+ State state(String stateId);
+
+ /**
+ * Returns the timer for the given {@link TimerId}.
+ */
+ Timer timer(String timerId);
}
/** Receives values of the given type. */
@@ -416,6 +426,16 @@ public abstract class DoFn<InputT, OutputT> implements Serializable, HasDisplayD
return null;
}
+ @Override
+ public State state(String stateId) {
+ return null;
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ return null;
+ }
+
public <RestrictionT> RestrictionTracker<RestrictionT> restrictionTracker() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
index ca724cd..420304b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnAdapters.java
@@ -28,7 +28,9 @@ import org.apache.beam.sdk.transforms.reflect.DoFnSignatures;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.WindowingInternals;
+import org.apache.beam.sdk.util.state.State;
import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -343,6 +345,16 @@ public class DoFnAdapters {
public <RestrictionT> RestrictionTracker<RestrictionT> restrictionTracker() {
throw new UnsupportedOperationException("This is a non-splittable DoFn");
}
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException("State is not supported by this runner");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("Timers are not supported by this runner");
+ }
}
/**
@@ -436,5 +448,15 @@ public class DoFnAdapters {
public <RestrictionT> RestrictionTracker<RestrictionT> restrictionTracker() {
throw new UnsupportedOperationException("This is a non-splittable DoFn");
}
+
+ @Override
+ public State state(String stateId) {
+ throw new UnsupportedOperationException("State is not supported by this runner");
+ }
+
+ @Override
+ public Timer timer(String timerId) {
+ throw new UnsupportedOperationException("Timers are not supported by this runner");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
index ad2b766..b7f75ed 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
@@ -23,8 +23,6 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,6 +50,7 @@ import net.bytebuddy.implementation.bytecode.StackManipulation;
import net.bytebuddy.implementation.bytecode.Throw;
import net.bytebuddy.implementation.bytecode.assign.Assigner;
import net.bytebuddy.implementation.bytecode.assign.TypeCasting;
+import net.bytebuddy.implementation.bytecode.constant.TextConstant;
import net.bytebuddy.implementation.bytecode.member.FieldAccess;
import net.bytebuddy.implementation.bytecode.member.MethodInvocation;
import net.bytebuddy.implementation.bytecode.member.MethodReturn;
@@ -77,6 +76,7 @@ import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Restrictio
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
+import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.TypeDescriptor;
@@ -503,15 +503,15 @@ public class DoFnInvokers {
}
}
- private static StackManipulation simpleExtraContextParameter(
- String methodName,
- StackManipulation pushExtraContextFactory) {
+ /**
+ * This wrapper exists to convert checked exceptions to unchecked exceptions, since if this fails
+ * the library itself is malformed.
+ */
+ private static MethodDescription getExtraContextFactoryMethodDescription(
+ String methodName, Class<?>... parameterTypes) {
try {
- return new StackManipulation.Compound(
- pushExtraContextFactory,
- MethodInvocation.invoke(
- new MethodDescription.ForLoadedMethod(
- DoFn.ExtraContextFactory.class.getMethod(methodName))));
+ return new MethodDescription.ForLoadedMethod(
+ DoFn.ExtraContextFactory.class.getMethod(methodName, parameterTypes));
} catch (Exception e) {
throw new IllegalStateException(
String.format(
@@ -521,47 +521,69 @@ public class DoFnInvokers {
}
}
+ private static StackManipulation simpleExtraContextParameter(
+ String methodName,
+ StackManipulation pushExtraContextFactory) {
+ return new StackManipulation.Compound(
+ pushExtraContextFactory,
+ MethodInvocation.invoke(getExtraContextFactoryMethodDescription(methodName)));
+ }
+
private static StackManipulation getExtraContextParameter(
DoFnSignature.Parameter parameter,
final StackManipulation pushExtraContextFactory) {
- return parameter.match(new Cases<StackManipulation>() {
+ return parameter.match(
+ new Cases<StackManipulation>() {
- @Override
- public StackManipulation dispatch(BoundedWindowParameter p) {
- return simpleExtraContextParameter("window", pushExtraContextFactory);
- }
+ @Override
+ public StackManipulation dispatch(BoundedWindowParameter p) {
+ return simpleExtraContextParameter("window", pushExtraContextFactory);
+ }
- @Override
- public StackManipulation dispatch(InputProviderParameter p) {
- return simpleExtraContextParameter("inputProvider", pushExtraContextFactory);
- }
+ @Override
+ public StackManipulation dispatch(InputProviderParameter p) {
+ return simpleExtraContextParameter("inputProvider", pushExtraContextFactory);
+ }
- @Override
- public StackManipulation dispatch(OutputReceiverParameter p) {
- return simpleExtraContextParameter("outputReceiver", pushExtraContextFactory);
- }
+ @Override
+ public StackManipulation dispatch(OutputReceiverParameter p) {
+ return simpleExtraContextParameter("outputReceiver", pushExtraContextFactory);
+ }
- @Override
- public StackManipulation dispatch(RestrictionTrackerParameter p) {
- // ExtraContextFactory.restrictionTracker() returns a RestrictionTracker,
- // but the @ProcessElement method expects a concrete subtype of it.
- // Insert a downcast.
- return new StackManipulation.Compound(
- simpleExtraContextParameter("restrictionTracker", pushExtraContextFactory),
- TypeCasting.to(new TypeDescription.ForLoadedType(p.trackerT().getRawType())));
- }
+ @Override
+ public StackManipulation dispatch(RestrictionTrackerParameter p) {
+ // ExtraContextFactory.restrictionTracker() returns a RestrictionTracker,
+ // but the @ProcessElement method expects a concrete subtype of it.
+ // Insert a downcast.
+ return new StackManipulation.Compound(
+ simpleExtraContextParameter("restrictionTracker", pushExtraContextFactory),
+ TypeCasting.to(new TypeDescription.ForLoadedType(p.trackerT().getRawType())));
+ }
- @Override
- public StackManipulation dispatch(StateParameter p) {
- throw new UnsupportedOperationException("State parameters are not yet supported.");
- }
+ @Override
+ public StackManipulation dispatch(StateParameter p) {
+ return new StackManipulation.Compound(
+ // TOP = extraContextFactory.state(<id>)
+ pushExtraContextFactory,
+ new TextConstant(p.referent().id()),
+ MethodInvocation.invoke(
+ getExtraContextFactoryMethodDescription("state", String.class)),
+ TypeCasting.to(
+ new TypeDescription.ForLoadedType(p.referent().stateType().getRawType())));
+ }
- @Override
- public StackManipulation dispatch(TimerParameter p) {
- throw new UnsupportedOperationException("Timer parameters are not yet supported.");
- }
- });
+ @Override
+ public StackManipulation dispatch(TimerParameter p) {
+ return new StackManipulation.Compound(
+ // TOP = extraContextFactory.state(<id>)
+ pushExtraContextFactory,
+ new TextConstant(p.referent().id()),
+ MethodInvocation.invoke(
+ getExtraContextFactoryMethodDescription("timer", String.class)),
+ TypeCasting.to(new TypeDescription.ForLoadedType(Timer.class)));
+ }
+ });
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e2db8268/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index dbb7955..60f82a8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -37,16 +37,23 @@ import java.util.Arrays;
import java.util.List;
import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.CustomCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.GetInitialRestriction;
import org.apache.beam.sdk.transforms.DoFn.ExtraContextFactory;
import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.reflect.testhelper.DoFnInvokersTestHelper;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.TimeDomain;
+import org.apache.beam.sdk.util.Timer;
+import org.apache.beam.sdk.util.TimerSpec;
+import org.apache.beam.sdk.util.TimerSpecs;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.util.WindowingInternals;
+import org.apache.beam.sdk.util.state.StateSpec;
+import org.apache.beam.sdk.util.state.StateSpecs;
+import org.apache.beam.sdk.util.state.ValueState;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -173,6 +180,56 @@ public class DoFnInvokersTest {
verify(fn).processElement(mockContext, mockWindow);
}
+ /**
+ * Tests that the generated {@link DoFnInvoker} passes the state parameter that it
+ * should.
+ */
+ @Test
+ public void testDoFnWithState() throws Exception {
+ ValueState<Integer> mockState = mock(ValueState.class);
+ final String stateId = "my-state-id-here";
+ when(extraContextFactory.state(stateId)).thenReturn(mockState);
+
+ class MockFn extends DoFn<String, String> {
+ @StateId(stateId)
+ private final StateSpec<Object, ValueState<Integer>> spec =
+ StateSpecs.value(VarIntCoder.of());
+
+ @ProcessElement
+ public void processElement(ProcessContext c, @StateId(stateId) ValueState<Integer> valueState)
+ throws Exception {}
+ }
+ MockFn fn = mock(MockFn.class);
+ assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
+ verify(fn).processElement(mockContext, mockState);
+ }
+
+ /**
+ * Tests that the generated {@link DoFnInvoker} passes the timer parameter that it
+ * should.
+ */
+ @Test
+ public void testDoFnWithTimer() throws Exception {
+ Timer mockTimer = mock(Timer.class);
+ final String timerId = "my-timer-id-here";
+ when(extraContextFactory.timer(timerId)).thenReturn(mockTimer);
+
+ class MockFn extends DoFn<String, String> {
+ @TimerId(timerId)
+ private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);
+
+ @ProcessElement
+ public void processElement(ProcessContext c, @TimerId(timerId) Timer timer)
+ throws Exception {}
+
+ @OnTimer(timerId)
+ public void onTimer() {}
+ }
+ MockFn fn = mock(MockFn.class);
+ assertEquals(ProcessContinuation.stop(), invokeProcessElement(fn));
+ verify(fn).processElement(mockContext, mockTimer);
+ }
+
@Test
public void testDoFnWithOutputReceiver() throws Exception {
class MockFn extends DoFn<String, String> {
[08/50] incubator-beam git commit: Remove @Default from IT options
Posted by ke...@apache.org.
Remove @Default from IT options
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1eccd29b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1eccd29b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1eccd29b
Branch: refs/heads/gearpump-runner
Commit: 1eccd29b72aedc87e69763fbcc828b5694335e1f
Parents: 99062d1
Author: Mark Liu <ma...@markliu0.mtv.corp.google.com>
Authored: Thu Nov 3 16:45:43 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Nov 4 14:19:43 2016 -0700
----------------------------------------------------------------------
.../beam/examples/WindowedWordCountIT.java | 11 ++++++++---
.../org/apache/beam/examples/WordCountIT.java | 19 +++++++++++--------
.../examples/cookbook/BigQueryTornadoesIT.java | 11 ++++++++---
3 files changed, 27 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1eccd29b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index 6742654..d545ad2 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -17,9 +17,9 @@
*/
package org.apache.beam.examples;
+import com.google.common.base.Strings;
import java.io.IOException;
import org.apache.beam.examples.WindowedWordCount.Options;
-import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.testing.BigqueryMatcher;
@@ -37,12 +37,13 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class WindowedWordCountIT {
+ private static final String DEFAULT_OUTPUT_CHECKSUM = "ff54f6f42b2afeb146206c1e8e915deaee0362b4";
+
/**
* Options for the {@link WindowedWordCount} Integration Test.
*/
public interface WindowedWordCountITOptions
extends Options, TestPipelineOptions, StreamingOptions {
- @Default.String("ff54f6f42b2afeb146206c1e8e915deaee0362b4")
String getChecksum();
void setChecksum(String value);
}
@@ -66,9 +67,13 @@ public class WindowedWordCountIT {
String query = String.format("SELECT word, SUM(count) FROM [%s:%s.%s] GROUP BY word",
options.getProject(), options.getBigQueryDataset(), options.getBigQueryTable());
+ String outputChecksum =
+ Strings.isNullOrEmpty(options.getChecksum())
+ ? DEFAULT_OUTPUT_CHECKSUM
+ : options.getChecksum();
options.setOnSuccessMatcher(
new BigqueryMatcher(
- options.getAppName(), options.getProject(), query, options.getChecksum()));
+ options.getAppName(), options.getProject(), query, outputChecksum));
WindowedWordCount.main(TestPipeline.convertToArgs(options));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1eccd29b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 2f2ea46..8f170af 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -18,9 +18,9 @@
package org.apache.beam.examples;
+import com.google.common.base.Strings;
import java.util.Date;
import org.apache.beam.examples.WordCount.WordCountOptions;
-import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.FileChecksumMatcher;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -36,6 +36,8 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class WordCountIT {
+ private static final String DEFAULT_OUTPUT_CHECKSUM = "8ae94f799f97cfd1cb5e8125951b32dfb52e1f12";
+
/**
* Options for the WordCount Integration Test.
*
@@ -43,9 +45,8 @@ public class WordCountIT {
* with customized input.
*/
public interface WordCountITOptions extends TestPipelineOptions, WordCountOptions {
- @Default.String("c04722202dee29c442b55ead54c6000693e85e77")
- String getOutputChecksum();
- void setOutputChecksum(String value);
+ String getChecksum();
+ void setChecksum(String value);
}
@Test
@@ -58,11 +59,13 @@ public class WordCountIT {
String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
"output",
"results"));
- options.setOnSuccessMatcher(
- new FileChecksumMatcher(options.getOutputChecksum(), options.getOutput() + "*"));
- String e2eTestInputPath = "gs://apache-beam-samples/apache/LICENSE";
- options.setInputFile(e2eTestInputPath);
+ String outputChecksum =
+ Strings.isNullOrEmpty(options.getChecksum())
+ ? DEFAULT_OUTPUT_CHECKSUM
+ : options.getChecksum();
+ options.setOnSuccessMatcher(
+ new FileChecksumMatcher(outputChecksum, options.getOutput() + "*"));
WordCount.main(TestPipeline.convertToArgs(options));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1eccd29b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
index 27a5a8f..209c713 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/cookbook/BigQueryTornadoesIT.java
@@ -18,8 +18,8 @@
package org.apache.beam.examples.cookbook;
+import com.google.common.base.Strings;
import org.apache.beam.sdk.options.BigQueryOptions;
-import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.BigqueryMatcher;
import org.apache.beam.sdk.testing.TestPipeline;
@@ -34,12 +34,13 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class BigQueryTornadoesIT {
+ private static final String DEFAULT_OUTPUT_CHECKSUM = "1ab4c7ec460b94bbb3c3885b178bf0e6bed56e1f";
+
/**
* Options for the BigQueryTornadoes Integration Test.
*/
public interface BigQueryTornadoesITOptions
extends TestPipelineOptions, BigQueryTornadoes.Options, BigQueryOptions {
- @Default.String("1ab4c7ec460b94bbb3c3885b178bf0e6bed56e1f")
String getChecksum();
void setChecksum(String value);
}
@@ -54,9 +55,13 @@ public class BigQueryTornadoesIT {
String query =
String.format("SELECT month, tornado_count FROM [%s]", options.getOutput());
+ String outputChecksum =
+ Strings.isNullOrEmpty(options.getChecksum())
+ ? DEFAULT_OUTPUT_CHECKSUM
+ : options.getChecksum();
options.setOnSuccessMatcher(
new BigqueryMatcher(
- options.getAppName(), options.getProject(), query, options.getChecksum()));
+ options.getAppName(), options.getProject(), query, outputChecksum));
BigQueryTornadoes.main(TestPipeline.convertToArgs(options));
}
[20/50] incubator-beam git commit: Format Regex according to style
guidelines
Posted by ke...@apache.org.
Format Regex according to style guidelines
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/79b04551
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/79b04551
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/79b04551
Branch: refs/heads/gearpump-runner
Commit: 79b04551c7c9f964908ab4a1d95119ef8a7fff84
Parents: 6954abe
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Nov 7 10:10:59 2016 -0800
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 10:10:59 2016 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/transforms/Regex.java | 525 +++++++++----------
.../apache/beam/sdk/transforms/RegexTest.java | 106 ++--
2 files changed, 283 insertions(+), 348 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/79b04551/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
index 27104f6..a94130d 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Regex.java
@@ -24,22 +24,17 @@ import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
/**
- * {@code PTransorm}s to use Regular Expressions to process elements in a
- * {@link PCollection}.
+ * {@code PTransorm}s to use Regular Expressions to process elements in a {@link PCollection}.
*
- * <p>
- * {@link Regex#matches(String, int)} can be used to see if an entire line matches
- * a Regex. {@link Regex#matchesKV(String, int, int)} can be used to see if an entire
- * line matches a Regex and output certain groups as a {@link KV}.
- * </p>
- * <p>
- * {@link Regex#find(String, int)} can be used to see if a portion of a line
- * matches a Regex. {@link Regex#matchesKV(String, int, int)} can be used to see if a
- * portion of a line matches a Regex and output certain groups as a {@link KV}.
- * </p>
- * <p>
- * Lines that do not match the Regex will not be output.
- * </p>
+ * <p>{@link Regex#matches(String, int)} can be used to see if an entire line matches a Regex.
+ * {@link Regex#matchesKV(String, int, int)} can be used to see if an entire line matches a Regex
+ * and output certain groups as a {@link KV}.
+ *
+ * <p>{@link Regex#find(String, int)} can be used to see if a portion of a line matches a Regex.
+ * {@link Regex#matchesKV(String, int, int)} can be used to see if a portion of a line matches a
+ * Regex and output certain groups as a {@link KV}.
+ *
+ * <p>Lines that do not match the Regex will not be output.
*/
public class Regex {
private Regex() {
@@ -47,159 +42,135 @@ public class Regex {
}
/**
- * Returns a {@link Regex.Matches} {@link PTransform} that checks if
- * the entire line matches the Regex. Returns the entire line (group 0) as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
+ * Returns a {@link Regex.Matches} {@link PTransform} that checks if the entire line matches the
+ * Regex. Returns the entire line (group 0) as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
*/
public static Matches matches(String regex) {
return matches(regex, 0);
}
/**
- * Returns a {@link Regex.Matches} {@link PTransform} that checks if
- * the entire line matches the Regex. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param group
- * The Regex group to return as a PCollection
+ * Returns a {@link Regex.Matches} {@link PTransform} that checks if the entire line matches the
+ * Regex. Returns the group as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param group The Regex group to return as a PCollection
*/
public static Matches matches(String regex, int group) {
return new Matches(regex, group);
}
/**
- * Returns a {@link Regex.MatchesKV} {@link PTransform} that checks
- * if the entire line matches the Regex. Returns the specified groups as the
- * key and value as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param keyGroup
- * The Regex group to use as the key
- * @param valueGroup
- * The Regex group to use the value
+ * Returns a {@link Regex.MatchesKV} {@link PTransform} that checks if the entire line matches the
+ * Regex. Returns the specified groups as the key and value as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param keyGroup The Regex group to use as the key
+ * @param valueGroup The Regex group to use the value
*/
- public static MatchesKV matchesKV(String regex, int keyGroup,
- int valueGroup) {
+ public static MatchesKV matchesKV(String regex, int keyGroup, int valueGroup) {
return new MatchesKV(regex, keyGroup, valueGroup);
}
/**
- * Returns a {@link Regex.Find} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the entire line (group 0) as
- * a {@link PCollection}.
- * @param regex
- * The regular expression to run
+ * Returns a {@link Regex.Find} {@link PTransform} that checks if a portion of the line matches
+ * the Regex. Returns the entire line (group 0) as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
*/
public static Find find(String regex) {
return find(regex, 0);
}
/**
- * Returns a {@link Regex.Find} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param group
- * The Regex group to return as a PCollection
+ * Returns a {@link Regex.Find} {@link PTransform} that checks if a portion of the line matches
+ * the Regex. Returns the group as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param group The Regex group to return as a PCollection
*/
public static Find find(String regex, int group) {
return new Find(regex, group);
}
/**
- * Returns a {@link Regex.FindKV} {@link PTransform} that checks if a
- * portion of the line matches the Regex. Returns the specified groups as the
- * key and value as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param keyGroup
- * The Regex group to use as the key
- * @param valueGroup
- * The Regex group to use the value
+ * Returns a {@link Regex.FindKV} {@link PTransform} that checks if a portion of the line matches
+ * the Regex. Returns the specified groups as the key and value as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param keyGroup The Regex group to use as the key
+ * @param valueGroup The Regex group to use the value
*/
public static FindKV findKV(String regex, int keyGroup, int valueGroup) {
return new FindKV(regex, keyGroup, valueGroup);
}
/**
- * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a
- * portion of the line matches the Regex and replaces all matches with the replacement
- * String. Returns the group as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param replacement
- * The string to be substituted for each match
+ * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a portion of the line
+ * matches the Regex and replaces all matches with the replacement String. Returns the group as a
+ * {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param replacement The string to be substituted for each match
*/
public static ReplaceAll replaceAll(String regex, String replacement) {
return new ReplaceAll(regex, replacement);
}
/**
- * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a
- * portion of the line matches the Regex and replaces the first match with the replacement
- * String. Returns the group as a {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param replacement
- * The string to be substituted for each match
+ * Returns a {@link Regex.ReplaceAll} {@link PTransform} that checks if a portion of the line
+ * matches the Regex and replaces the first match with the replacement String. Returns the group
+ * as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param replacement The string to be substituted for each match
*/
public static ReplaceFirst replaceFirst(String regex, String replacement) {
return new ReplaceFirst(regex, replacement);
}
- /**
- * Returns a {@link Regex.Split} {@link PTransform} that splits a string
- * on the regular expression and then outputs each item. It will not output empty
- * items. Returns the group as a {@link PCollection}.
- * a {@link PCollection}.
- * @param regex
- * The regular expression to run
+ /**
+ * Returns a {@link Regex.Split} {@link PTransform} that splits a string on the regular expression
+ * and then outputs each item. It will not output empty items. Returns the group as a {@link
+ * PCollection}. a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
*/
public static Split split(String regex) {
return split(regex, false);
}
/**
- * Returns a {@link Regex.Split} {@link PTransform} that splits a string
- * on the regular expression and then outputs each item. Returns the group as a
- * {@link PCollection}.
- * @param regex
- * The regular expression to run
- * @param outputEmpty
- * Should empty be output. True to output empties and false if not.
+ * Returns a {@link Regex.Split} {@link PTransform} that splits a string on the regular expression
+ * and then outputs each item. Returns the group as a {@link PCollection}.
+ *
+ * @param regex The regular expression to run
+ * @param outputEmpty Should empty be output. True to output empties and false if not.
*/
public static Split split(String regex, boolean outputEmpty) {
return new Split(regex, outputEmpty);
}
/**
- * {@code Regex.Matches<String>} takes a {@code PCollection<String>}
- * and returns a {@code PCollection<String>} representing the value
- * extracted from the Regex groups of the input {@code PCollection}
- * to the number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If the entire line
- * does not match the Regex, the line will not be output. If it does match the
- * entire line, the group in the Regex will be used. The output will be the
- * Regex group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.Matches<String>} takes a {@code PCollection<String>} and returns a {@code
+ * PCollection<String>} representing the value extracted from the Regex groups of the input {@code
+ * PCollection} to the number of times that element occurs in the input.
+ *
+ * <p>This transform runs a Regex on the entire input line. If the entire line does not match the
+ * Regex, the line will not be output. If it does match the entire line, the group in the Regex
+ * will be used. The output will be the Regex group.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<String> values =
* words.apply(Regex.matches("myregex (mygroup)", 1));
- * }
- * </pre>
+ * }</pre>
*/
- public static class Matches
- extends PTransform<PCollection<String>, PCollection<String>> {
+ public static class Matches extends PTransform<PCollection<String>, PCollection<String>> {
Pattern pattern;
int group;
@@ -209,42 +180,38 @@ public class Regex {
}
public PCollection<String> apply(PCollection<String> in) {
- return in
- .apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.matches()) {
- c.output(m.group(group));
- }
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.matches()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
}
}
/**
- * {@code Regex.MatchesKV<KV<String, String>>} takes a
- * {@code PCollection<String>} and returns a
- * {@code PCollection<KV<String, String>>} representing the key and value
- * extracted from the Regex groups of the input {@code PCollection} to the
- * number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If the entire line
- * does not match the Regex, the line will not be output. If it does match the
- * entire line, the groups in the Regex will be used. The key will be the
- * key's group and the value will be the value's group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.MatchesKV<KV<String, String>>} takes a {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value extracted from the Regex
+ * groups of the input {@code PCollection} to the number of times that element occurs in the
+ * input.
+ *
+ * <p>This transform runs a Regex on the entire input line. If the entire line does not match the
+ * Regex, the line will not be output. If it does match the entire line, the groups in the Regex
+ * will be used. The key will be the key's group and the value will be the value's group.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<KV<String, String>> keysAndValues =
* words.apply(Regex.matchesKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
- * }
- * </pre>
+ * }</pre>
*/
public static class MatchesKV
extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
@@ -258,44 +225,39 @@ public class Regex {
}
public PCollection<KV<String, String>> apply(PCollection<String> in) {
- return in.apply(ParDo
- .of(new DoFn<String, KV<String, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
- }
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
}
}
/**
- * {@code Regex.Find<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} representing the value extracted
- * from the Regex groups of the input {@code PCollection} to
- * the number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will not be output. If it does
- * match a portion of the line, the group in the Regex will be used. The
- * output will be the Regex group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.Find<String>} takes a {@code PCollection<String>} and returns a {@code
+ * PCollection<String>} representing the value extracted from the Regex groups of the input {@code
+ * PCollection} to the number of times that element occurs in the input.
+ *
+ * <p>This transform runs a Regex on the entire input line. If a portion of the line does not
+ * match the Regex, the line will not be output. If it does match a portion of the line, the group
+ * in the Regex will be used. The output will be the Regex group.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<String> values =
* words.apply(Regex.find("myregex (mygroup)", 1));
- * }
- * </pre>
+ * }</pre>
*/
- public static class Find
- extends PTransform<PCollection<String>, PCollection<String>> {
+ public static class Find extends PTransform<PCollection<String>, PCollection<String>> {
Pattern pattern;
int group;
@@ -305,41 +267,39 @@ public class Regex {
}
public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(m.group(group));
- }
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(m.group(group));
+ }
+ }
+ }));
}
}
/**
- * {@code Regex.MatchesKV<KV<String, String>>} takes a
- * {@code PCollection<String>} and returns a
- * {@code PCollection<KV<String, String>>} representing the key and value
- * extracted from the Regex groups of the input {@code PCollection} to the
- * number of times that element occurs in the input.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will not be output. If it does
- * match a portion of the line, the groups in the Regex will be used. The key
- * will be the key's group and the value will be the value's group.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.MatchesKV<KV<String, String>>} takes a {@code PCollection<String>} and returns a
+ * {@code PCollection<KV<String, String>>} representing the key and value extracted from the Regex
+ * groups of the input {@code PCollection} to the number of times that element occurs in the
+ * input.
+ *
+ * <p>This transform runs a Regex on the entire input line. If a portion of the line does not
+ * match the Regex, the line will not be output. If it does match a portion of the line, the
+ * groups in the Regex will be used. The key will be the key's group and the value will be the
+ * value's group.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<KV<String, String>> keysAndValues =
* words.apply(Regex.findKV("myregex (mykeygroup) (myvaluegroup)", 1, 2));
- * }
- * </pre>
+ * }</pre>
*/
public static class FindKV
extends PTransform<PCollection<String>, PCollection<KV<String, String>>> {
@@ -354,42 +314,38 @@ public class Regex {
public PCollection<KV<String, String>> apply(PCollection<String> in) {
return in.apply(
- ParDo.of(new DoFn<String, KV<String, String>>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
-
- if (m.find()) {
- c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
- }
- }
- }));
+ ParDo.of(
+ new DoFn<String, KV<String, String>>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+
+ if (m.find()) {
+ c.output(KV.of(m.group(keyGroup), m.group(valueGroup)));
+ }
+ }
+ }));
}
}
/**
- * {@code Regex.ReplaceAll<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with all Strings that matched the
- * Regex being replaced with the replacement string.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will be output without changes. If it does
- * match a portion of the line, all portions matching the Regex will be replaced
- * with the replacement String.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.ReplaceAll<String>} takes a {@code PCollection<String>} and returns a {@code
+ * PCollection<String>} with all Strings that matched the Regex being replaced with the
+ * replacement string.
+ *
+ * <p>This transform runs a Regex on the entire input line. If a portion of the line does not
+ * match the Regex, the line will be output without changes. If it does match a portion of the
+ * line, all portions matching the Regex will be replaced with the replacement String.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<String> values =
* words.apply(Regex.replaceAll("myregex", "myreplacement"));
- * }
- * </pre>
+ * }</pre>
*/
- public static class ReplaceAll
- extends PTransform<PCollection<String>, PCollection<String>> {
+ public static class ReplaceAll extends PTransform<PCollection<String>, PCollection<String>> {
Pattern pattern;
String replacement;
@@ -399,39 +355,36 @@ public class Regex {
}
public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
- c.output(m.replaceAll(replacement));
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceAll(replacement));
+ }
+ }));
}
}
/**
- * {@code Regex.ReplaceFirst<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with the first Strings that matched the
- * Regex being replaced with the replacement string.
- *
- * <p>
- * This transform runs a Regex on the entire input line. If a portion of the
- * line does not match the Regex, the line will be output without changes. If it does
- * match a portion of the line, the first portion matching the Regex will be replaced
- * with the replacement String.
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.ReplaceFirst<String>} takes a {@code PCollection<String>} and returns a {@code
+ * PCollection<String>} with the first Strings that matched the Regex being replaced with the
+ * replacement string.
+ *
+ * <p>This transform runs a Regex on the entire input line. If a portion of the line does not
+ * match the Regex, the line will be output without changes. If it does match a portion of the
+ * line, the first portion matching the Regex will be replaced with the replacement String.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<String> values =
* words.apply(Regex.replaceFirst("myregex", "myreplacement"));
- * }
- * </pre>
+ * }</pre>
*/
- public static class ReplaceFirst
- extends PTransform<PCollection<String>, PCollection<String>> {
+ public static class ReplaceFirst extends PTransform<PCollection<String>, PCollection<String>> {
Pattern pattern;
String replacement;
@@ -441,44 +394,38 @@ public class Regex {
}
public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- Matcher m = pattern.matcher((String) c.element());
- c.output(m.replaceFirst(replacement));
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ Matcher m = pattern.matcher((String) c.element());
+ c.output(m.replaceFirst(replacement));
+ }
+ }));
}
}
/**
- * {@code Regex.Split<String>} takes a {@code PCollection<String>} and
- * returns a {@code PCollection<String>} with the input string split into
- * individual items in a list. Each item is then output as a separate string.
- *
- * <p>
- * This transform runs a Regex as part of a splint the entire input line. The split
- * gives back an array of items. Each item is output as a separate item in the
- * {@code PCollection<String>}.
- * </p>
- *
- * <p>
- * Depending on the Regex, a split can be an empty or
- * "" string. You can pass in a parameter if you want empty strings or not.
- * </p>
- *
- * <p>
- * Example of use:
- * <pre>
- * {@code
+ * {@code Regex.Split<String>} takes a {@code PCollection<String>} and returns a {@code
+ * PCollection<String>} with the input string split into individual items in a list. Each item is
+ * then output as a separate string.
+ *
+ * <p>This transform runs a Regex as part of a splint the entire input line. The split gives back
+ * an array of items. Each item is output as a separate item in the {@code PCollection<String>}.
+ *
+ * <p>Depending on the Regex, a split can be an empty or "" string. You can pass in a parameter if
+ * you want empty strings or not.
+ *
+ * <p>Example of use:
+ *
+ * <pre>{@code
* PCollection<String> words = ...;
* PCollection<String> values =
* words.apply(Regex.split("\W*"));
- * }
- * </pre>
+ * }</pre>
*/
- public static class Split
- extends PTransform<PCollection<String>, PCollection<String>> {
+ public static class Split extends PTransform<PCollection<String>, PCollection<String>> {
Pattern pattern;
boolean outputEmpty;
@@ -488,18 +435,20 @@ public class Regex {
}
public PCollection<String> apply(PCollection<String> in) {
- return in.apply(ParDo.of(new DoFn<String, String>() {
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- String[] items = pattern.split(c.element());
-
- for (String item : items) {
- if (outputEmpty || !item.isEmpty()) {
- c.output(item);
- }
- }
- }
- }));
+ return in.apply(
+ ParDo.of(
+ new DoFn<String, String>() {
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ String[] items = pattern.split(c.element());
+
+ for (String item : items) {
+ if (outputEmpty || !item.isEmpty()) {
+ c.output(item);
+ }
+ }
+ }
+ }));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/79b04551/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
index 71f080e..6e196b4 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/RegexTest.java
@@ -30,9 +30,7 @@ import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/**
- * Tests for {@link Regex}.
- */
+/** Tests for {@link Regex}. */
@RunWith(JUnit4.class)
public class RegexTest implements Serializable {
@Test
@@ -40,9 +38,8 @@ public class RegexTest implements Serializable {
public void testFind() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("aj", "xj", "yj", "zj"))
- .apply(Regex.find("[xyz]"));
+ PCollection<String> output =
+ p.apply(Create.of("aj", "xj", "yj", "zj")).apply(Regex.find("[xyz]"));
PAssert.that(output).containsInAnyOrder("x", "y", "z");
p.run();
@@ -53,9 +50,8 @@ public class RegexTest implements Serializable {
public void testFindGroup() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("aj", "xj", "yj", "zj"))
- .apply(Regex.find("([xyz])", 1));
+ PCollection<String> output =
+ p.apply(Create.of("aj", "xj", "yj", "zj")).apply(Regex.find("([xyz])", 1));
PAssert.that(output).containsInAnyOrder("x", "y", "z");
p.run();
@@ -66,9 +62,7 @@ public class RegexTest implements Serializable {
public void testFindNone() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("a", "b", "c", "d"))
- .apply(Regex.find("[xyz]"));
+ PCollection<String> output = p.apply(Create.of("a", "b", "c", "d")).apply(Regex.find("[xyz]"));
PAssert.that(output).empty();
p.run();
@@ -79,9 +73,8 @@ public class RegexTest implements Serializable {
public void testKVFind() {
TestPipeline p = TestPipeline.create();
- PCollection<KV<String, String>> output = p
- .apply(Create.of("a b c"))
- .apply(Regex.findKV("a (b) (c)", 1, 2));
+ PCollection<KV<String, String>> output =
+ p.apply(Create.of("a b c")).apply(Regex.findKV("a (b) (c)", 1, 2));
PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
p.run();
@@ -92,9 +85,8 @@ public class RegexTest implements Serializable {
public void testKVFindNone() {
TestPipeline p = TestPipeline.create();
- PCollection<KV<String, String>> output = p
- .apply(Create.of("x y z"))
- .apply(Regex.findKV("a (b) (c)", 1, 2));
+ PCollection<KV<String, String>> output =
+ p.apply(Create.of("x y z")).apply(Regex.findKV("a (b) (c)", 1, 2));
PAssert.that(output).empty();
p.run();
@@ -105,9 +97,8 @@ public class RegexTest implements Serializable {
public void testMatches() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("a", "x", "y", "z"))
- .apply(Regex.matches("[xyz]"));
+ PCollection<String> output =
+ p.apply(Create.of("a", "x", "y", "z")).apply(Regex.matches("[xyz]"));
PAssert.that(output).containsInAnyOrder("x", "y", "z");
p.run();
@@ -118,9 +109,8 @@ public class RegexTest implements Serializable {
public void testMatchesNone() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("a", "b", "c", "d"))
- .apply(Regex.matches("[xyz]"));
+ PCollection<String> output =
+ p.apply(Create.of("a", "b", "c", "d")).apply(Regex.matches("[xyz]"));
PAssert.that(output).empty();
p.run();
@@ -131,9 +121,8 @@ public class RegexTest implements Serializable {
public void testMatchesGroup() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("a", "x xxx", "x yyy", "x zzz"))
- .apply(Regex.matches("x ([xyz]*)", 1));
+ PCollection<String> output =
+ p.apply(Create.of("a", "x xxx", "x yyy", "x zzz")).apply(Regex.matches("x ([xyz]*)", 1));
PAssert.that(output).containsInAnyOrder("xxx", "yyy", "zzz");
p.run();
@@ -144,9 +133,8 @@ public class RegexTest implements Serializable {
public void testKVMatches() {
TestPipeline p = TestPipeline.create();
- PCollection<KV<String, String>> output = p
- .apply(Create.of("a b c"))
- .apply(Regex.matchesKV("a (b) (c)", 1, 2));
+ PCollection<KV<String, String>> output =
+ p.apply(Create.of("a b c")).apply(Regex.matchesKV("a (b) (c)", 1, 2));
PAssert.that(output).containsInAnyOrder(KV.of("b", "c"));
p.run();
@@ -157,9 +145,8 @@ public class RegexTest implements Serializable {
public void testKVMatchesNone() {
TestPipeline p = TestPipeline.create();
- PCollection<KV<String, String>> output = p
- .apply(Create.of("x y z"))
- .apply(Regex.matchesKV("a (b) (c)", 1, 2));
+ PCollection<KV<String, String>> output =
+ p.apply(Create.of("x y z")).apply(Regex.matchesKV("a (b) (c)", 1, 2));
PAssert.that(output).empty();
p.run();
}
@@ -169,9 +156,8 @@ public class RegexTest implements Serializable {
public void testReplaceAll() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("xj", "yj", "zj"))
- .apply(Regex.replaceAll("[xyz]", "new"));
+ PCollection<String> output =
+ p.apply(Create.of("xj", "yj", "zj")).apply(Regex.replaceAll("[xyz]", "new"));
PAssert.that(output).containsInAnyOrder("newj", "newj", "newj");
p.run();
@@ -182,9 +168,8 @@ public class RegexTest implements Serializable {
public void testReplaceAllMixed() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("abc", "xj", "yj", "zj", "def"))
- .apply(Regex.replaceAll("[xyz]", "new"));
+ PCollection<String> output =
+ p.apply(Create.of("abc", "xj", "yj", "zj", "def")).apply(Regex.replaceAll("[xyz]", "new"));
PAssert.that(output).containsInAnyOrder("abc", "newj", "newj", "newj", "def");
p.run();
@@ -195,9 +180,8 @@ public class RegexTest implements Serializable {
public void testReplaceFirst() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("xjx", "yjy", "zjz"))
- .apply(Regex.replaceFirst("[xyz]", "new"));
+ PCollection<String> output =
+ p.apply(Create.of("xjx", "yjy", "zjz")).apply(Regex.replaceFirst("[xyz]", "new"));
PAssert.that(output).containsInAnyOrder("newjx", "newjy", "newjz");
p.run();
@@ -208,9 +192,9 @@ public class RegexTest implements Serializable {
public void testReplaceFirstMixed() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("abc", "xjx", "yjy", "zjz", "def"))
- .apply(Regex.replaceFirst("[xyz]", "new"));
+ PCollection<String> output =
+ p.apply(Create.of("abc", "xjx", "yjy", "zjz", "def"))
+ .apply(Regex.replaceFirst("[xyz]", "new"));
PAssert.that(output).containsInAnyOrder("abc", "newjx", "newjy", "newjz", "def");
p.run();
@@ -221,12 +205,12 @@ public class RegexTest implements Serializable {
public void testSplits() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(Regex.split("\\W+"));
+ PCollection<String> output =
+ p.apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\W+"));
- PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
- "fox", "jumps", "over", "the", "lazy", "dog");
+ PAssert.that(output)
+ .containsInAnyOrder("The", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog");
p.run();
}
@@ -235,14 +219,16 @@ public class RegexTest implements Serializable {
public void testSplitsWithEmpty() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(Regex.split("\\s", true));
+ PCollection<String> output =
+ p.apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\s", true));
String[] outputStr = "The quick brown fox jumps over the lazy dog".split("\\s");
- PAssert.that(output).containsInAnyOrder("The", "", "quick", "brown", "", "",
- "fox", "jumps", "over", "", "", "", "the", "lazy", "dog");
+ PAssert.that(output)
+ .containsInAnyOrder(
+ "The", "", "quick", "brown", "", "", "fox", "jumps", "over", "", "", "", "the", "lazy",
+ "dog");
p.run();
}
@@ -251,12 +237,12 @@ public class RegexTest implements Serializable {
public void testSplitsWithoutEmpty() {
TestPipeline p = TestPipeline.create();
- PCollection<String> output = p
- .apply(Create.of("The quick brown fox jumps over the lazy dog"))
- .apply(Regex.split("\\s", false));
+ PCollection<String> output =
+ p.apply(Create.of("The quick brown fox jumps over the lazy dog"))
+ .apply(Regex.split("\\s", false));
- PAssert.that(output).containsInAnyOrder("The", "quick", "brown",
- "fox", "jumps", "over", "the", "lazy", "dog");
+ PAssert.that(output)
+ .containsInAnyOrder("The", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog");
p.run();
}
}
[06/50] incubator-beam git commit: This closes #1276
Posted by ke...@apache.org.
This closes #1276
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/99062d10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/99062d10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/99062d10
Branch: refs/heads/gearpump-runner
Commit: 99062d103c96e47ea78fc866984195df12de7770
Parents: 6e1e57b 5dca267
Author: Thomas Groh <tg...@google.com>
Authored: Fri Nov 4 13:05:21 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Fri Nov 4 13:05:21 2016 -0700
----------------------------------------------------------------------
.../beam/runners/direct/EvaluationContext.java | 6 +-
.../direct/ExecutorServiceParallelExecutor.java | 41 ++++----
.../beam/runners/direct/WatermarkManager.java | 79 ++++++++------
.../runners/direct/EvaluationContextTest.java | 23 ++---
.../runners/direct/WatermarkManagerTest.java | 102 ++++++-------------
5 files changed, 109 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
[24/50] incubator-beam git commit: Closes #1295
Posted by ke...@apache.org.
Closes #1295
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b5f84735
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b5f84735
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b5f84735
Branch: refs/heads/gearpump-runner
Commit: b5f847355f2961b992abc3f8b4df71f0d96bb025
Parents: 1102455 e1291ee
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 11:40:38 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 11:40:38 2016 -0800
----------------------------------------------------------------------
examples/java/pom.xml | 13 +++++++++++++
examples/java8/pom.xml | 13 +++++++++++++
examples/pom.xml | 5 +++++
runners/core-java/pom.xml | 12 ++++++++++++
runners/direct-java/pom.xml | 13 +++++++++++++
runners/flink/pom.xml | 9 +++++++++
runners/google-cloud-dataflow-java/pom.xml | 13 +++++++++++++
runners/pom.xml | 5 +++++
runners/spark/pom.xml | 9 +++++++++
sdks/java/core/pom.xml | 5 -----
.../apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 ++--
sdks/java/io/jms/pom.xml | 13 +++++++++++++
sdks/java/io/kafka/pom.xml | 13 +++++++++++++
sdks/java/io/kinesis/pom.xml | 13 +++++++++++++
sdks/java/io/mongodb/pom.xml | 13 +++++++++++++
sdks/java/microbenchmarks/pom.xml | 13 +++++++++++++
sdks/pom.xml | 5 +++++
17 files changed, 164 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[43/50] incubator-beam git commit: This closes #1287
Posted by ke...@apache.org.
This closes #1287
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c6d9bf29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c6d9bf29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c6d9bf29
Branch: refs/heads/gearpump-runner
Commit: c6d9bf29700d6f13a33423183201a18525040e05
Parents: 9de9ce6 a58f1eb
Author: Thomas Groh <tg...@google.com>
Authored: Mon Nov 7 15:47:02 2016 -0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 15:47:02 2016 -0800
----------------------------------------------------------------------
.../beam/runners/direct/DirectRunner.java | 5 +
.../direct/ImmutableListBundleFactory.java | 21 ++-
.../beam/runners/direct/WatermarkManager.java | 153 +++++++++++++------
.../direct/ImmutableListBundleFactoryTest.java | 15 +-
4 files changed, 141 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
[02/50] incubator-beam git commit: checkstyle: move from individual
modules to root poms
Posted by ke...@apache.org.
checkstyle: move from individual modules to root poms
Checkstyle config is no longer needed in individual modules;
instead, move it only to sdks/ , runners/, and examples/ poms.
Individual modules like sdks/java/core may still reference checkstyle
to configure their own options.
The only sketchy bit is that I had to link sdks/java/build-tools to
beam-parent rather than sdks/java so that I didn't create a build
dependency loop.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f50b2142
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f50b2142
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f50b2142
Branch: refs/heads/gearpump-runner
Commit: f50b2142b641c6d2aca186903139cff995f7d49e
Parents: faf55c7
Author: Dan Halperin <dh...@google.com>
Authored: Thu Nov 3 17:24:52 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Nov 4 10:51:28 2016 -0700
----------------------------------------------------------------------
examples/java/pom.xml | 5 -----
examples/java8/pom.xml | 5 -----
examples/pom.xml | 13 ++++++++++++-
pom.xml | 3 +++
runners/core-java/pom.xml | 5 -----
runners/direct-java/pom.xml | 5 -----
runners/flink/examples/pom.xml | 5 -----
runners/flink/runner/pom.xml | 5 -----
runners/google-cloud-dataflow-java/pom.xml | 7 -------
runners/pom.xml | 11 +++++++++++
runners/spark/pom.xml | 4 ----
sdks/java/build-tools/pom.xml | 4 ++--
.../src/main/resources/beam/checkstyle.xml | 2 +-
sdks/java/core/pom.xml | 18 +++++++++---------
sdks/java/extensions/join-library/pom.xml | 4 ----
sdks/java/extensions/sorter/pom.xml | 4 ----
sdks/java/io/google-cloud-platform/pom.xml | 4 ----
sdks/java/io/hdfs/pom.xml | 4 ----
sdks/java/io/jdbc/pom.xml | 6 +-----
sdks/java/io/jms/pom.xml | 4 ----
sdks/java/io/kafka/pom.xml | 4 ----
sdks/java/io/kinesis/pom.xml | 4 ----
sdks/java/io/mongodb/pom.xml | 4 ----
sdks/java/java8tests/pom.xml | 5 -----
sdks/java/microbenchmarks/pom.xml | 5 -----
sdks/java/pom.xml | 4 +++-
sdks/pom.xml | 16 +++++++++++++++-
27 files changed, 57 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index fc82ed4..12a114f 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -277,11 +277,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
<!-- Source plugin for generating source and test-source JARs. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index e6408dc..18257d9 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -148,11 +148,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
<!-- Source plugin for generating source and test-source JARs. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 2820473..c6f9cb3 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -45,7 +45,18 @@
<modules>
<module>java8</module>
</modules>
- </profile>
+ </profile>
+ <profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ea7d4ae..28845a5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,6 +140,9 @@
<packaging>pom</packaging>
<modules>
+ <!-- sdks/java/build-tools has project-wide configuration. To make these available
+ in all modules, link it directly to the parent pom.xml. -->
+ <module>sdks/java/build-tools</module>
<module>sdks</module>
<module>runners</module>
<!-- sdks/java/maven-archetypes has several dependencies on the DataflowPipelineRunner.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index e3e4fed..fa64660 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -54,11 +54,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 84190b2..b781c6e 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -48,11 +48,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/flink/examples/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/examples/pom.xml b/runners/flink/examples/pom.xml
index a0cf676..409d77b 100644
--- a/runners/flink/examples/pom.xml
+++ b/runners/flink/examples/pom.xml
@@ -111,11 +111,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<additionalparam>-Xdoclint:missing</additionalparam>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/flink/runner/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/runner/pom.xml b/runners/flink/runner/pom.xml
index 283d060..928889e 100644
--- a/runners/flink/runner/pom.xml
+++ b/runners/flink/runner/pom.xml
@@ -244,11 +244,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<additionalparam>-Xdoclint:missing</additionalparam>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index 750f520..d8f1b82 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -72,13 +72,6 @@
</executions>
</plugin>
- <!-- Run CheckStyle pass on transforms, as they are release in
- source form. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/pom.xml
----------------------------------------------------------------------
diff --git a/runners/pom.xml b/runners/pom.xml
index b2b613f..9c821cc 100644
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@ -41,6 +41,17 @@
</modules>
<profiles>
+ <profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<!-- A profile that adds an integration test phase if and only if
the runnableOnServicePipelineOptions maven property has been set.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 71a3ac2..0edafe8 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -411,10 +411,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/build-tools/pom.xml b/sdks/java/build-tools/pom.xml
index cc27bea..b911328 100644
--- a/sdks/java/build-tools/pom.xml
+++ b/sdks/java/build-tools/pom.xml
@@ -21,9 +21,9 @@
<parent>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-parent</artifactId>
+ <artifactId>beam-parent</artifactId>
<version>0.4.0-incubating-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>beam-sdks-java-build-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
----------------------------------------------------------------------
diff --git a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
index e57cfec..ebbaa7d 100644
--- a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
@@ -42,7 +42,7 @@ page at http://checkstyle.sourceforge.net/config.html -->
</module>
<module name="RegexpSingleline">
- <property name="format" value="[ \t]+$"/>
+ <property name="format" value="\s+$"/>
<property name="message" value="Trailing whitespace"/>
<property name="severity" value="error"/>
</module>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 29aaaab..40a511c 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -106,6 +106,15 @@
</offlineLinks>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <!-- Set testSourceDirectory in order to exclude generated-test-sources -->
+ <testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
@@ -130,15 +139,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration>
- <!-- Set testSourceDirectory in order to exclude generated-test-sources -->
- <testSourceDirectory>${project.basedir}/src/test/</testSourceDirectory>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/extensions/join-library/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/join-library/pom.xml b/sdks/java/extensions/join-library/pom.xml
index 4687554..16326b0 100644
--- a/sdks/java/extensions/join-library/pom.xml
+++ b/sdks/java/extensions/join-library/pom.xml
@@ -46,10 +46,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/extensions/sorter/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/pom.xml b/sdks/java/extensions/sorter/pom.xml
index e77d5ad..1a994de 100644
--- a/sdks/java/extensions/sorter/pom.xml
+++ b/sdks/java/extensions/sorter/pom.xml
@@ -50,10 +50,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/google-cloud-platform/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/pom.xml b/sdks/java/io/google-cloud-platform/pom.xml
index 054ee21..fa51072 100644
--- a/sdks/java/io/google-cloud-platform/pom.xml
+++ b/sdks/java/io/google-cloud-platform/pom.xml
@@ -58,10 +58,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
<!-- Integration Tests -->
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hdfs/pom.xml b/sdks/java/io/hdfs/pom.xml
index 22c3187..e79996f 100644
--- a/sdks/java/io/hdfs/pom.xml
+++ b/sdks/java/io/hdfs/pom.xml
@@ -53,10 +53,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index 3f513ca..a65ed46 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -50,10 +50,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
</plugin>
</plugins>
@@ -135,4 +131,4 @@
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/jms/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jms/pom.xml b/sdks/java/io/jms/pom.xml
index 5a74b34..0b46154 100644
--- a/sdks/java/io/jms/pom.xml
+++ b/sdks/java/io/jms/pom.xml
@@ -50,10 +50,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/kafka/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index f2b8326..184c1c9 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -54,10 +54,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<additionalparam>-Xdoclint:missing</additionalparam>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index 0f65d67..76df329 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -54,10 +54,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<additionalparam>-Xdoclint:missing</additionalparam>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/io/mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index ad0609d..d674e63 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -50,10 +50,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/java8tests/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/java8tests/pom.xml b/sdks/java/java8tests/pom.xml
index 13f5fce..989fba2 100644
--- a/sdks/java/java8tests/pom.xml
+++ b/sdks/java/java8tests/pom.xml
@@ -61,11 +61,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
<!-- Source plugin for generating source and test-source JARs. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/microbenchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/microbenchmarks/pom.xml b/sdks/java/microbenchmarks/pom.xml
index 5c6cf4d..05f06aa 100644
--- a/sdks/java/microbenchmarks/pom.xml
+++ b/sdks/java/microbenchmarks/pom.xml
@@ -50,11 +50,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/java/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index eae5a58..2866cb3 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -33,7 +33,9 @@
<name>Apache Beam :: SDKs :: Java</name>
<modules>
- <module>build-tools</module>
+ <!-- build-tools inherits from the Beam root pom to enable checkstyle
+ and other project configuration to be used in all modules.
+ <module>build-tools</module> -->
<module>core</module>
<module>io</module>
<!-- sdks/java/maven-archtypes has several dependencies on the
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f50b2142/sdks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/pom.xml b/sdks/pom.xml
index 235d102..6347fe1 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -36,6 +36,20 @@
<module>java</module>
</modules>
+ <profiles>
+ <profile>
+ <id>release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<build>
<pluginManagement>
<plugins>
@@ -57,4 +71,4 @@
</pluginManagement>
</build>
-</project>
\ No newline at end of file
+</project>
[34/50] incubator-beam git commit: This closes #1300
Posted by ke...@apache.org.
This closes #1300
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ac252a7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ac252a7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ac252a7e
Branch: refs/heads/gearpump-runner
Commit: ac252a7e151f4287bc349f9d657eae15c0b3e0fc
Parents: 912500f ff7fe07
Author: Thomas Groh <tg...@google.com>
Authored: Mon Nov 7 15:08:44 2016 -0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 15:08:44 2016 -0800
----------------------------------------------------------------------
.../direct/UnboundedReadEvaluatorFactory.java | 6 ++++--
.../UnboundedReadEvaluatorFactoryTest.java | 19 +++++++++++++------
2 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[22/50] incubator-beam git commit: Disable all broken findbugs builds
and link to JIRA issues
Posted by ke...@apache.org.
Disable all broken findbugs builds and link to JIRA issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e1291eea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e1291eea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e1291eea
Branch: refs/heads/gearpump-runner
Commit: e1291eea9e21dce4df44245d53a78664003c264a
Parents: fae52a3
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 09:54:04 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 10:43:11 2016 -0800
----------------------------------------------------------------------
examples/java/pom.xml | 13 +++++++++++++
examples/java8/pom.xml | 13 +++++++++++++
runners/core-java/pom.xml | 12 ++++++++++++
runners/direct-java/pom.xml | 13 +++++++++++++
runners/flink/pom.xml | 9 +++++++++
runners/google-cloud-dataflow-java/pom.xml | 13 +++++++++++++
runners/spark/pom.xml | 9 +++++++++
.../apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 ++--
sdks/java/io/jms/pom.xml | 13 +++++++++++++
sdks/java/io/kafka/pom.xml | 13 +++++++++++++
sdks/java/io/kinesis/pom.xml | 13 +++++++++++++
sdks/java/io/mongodb/pom.xml | 13 +++++++++++++
sdks/java/microbenchmarks/pom.xml | 13 +++++++++++++
13 files changed, 149 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/examples/java/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java/pom.xml b/examples/java/pom.xml
index 5d69bfe..6d18a0f 100644
--- a/examples/java/pom.xml
+++ b/examples/java/pom.xml
@@ -260,6 +260,19 @@
</profiles>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-933 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/examples/java8/pom.xml
----------------------------------------------------------------------
diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml
index 72f24ca..8591955 100644
--- a/examples/java8/pom.xml
+++ b/examples/java8/pom.xml
@@ -118,6 +118,19 @@
</profiles>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-934 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/runners/core-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/core-java/pom.xml b/runners/core-java/pom.xml
index 40ebf58..aa5f145 100644
--- a/runners/core-java/pom.xml
+++ b/runners/core-java/pom.xml
@@ -33,6 +33,18 @@
<packaging>jar</packaging>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-922 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/runners/direct-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/direct-java/pom.xml b/runners/direct-java/pom.xml
index 8983b1c..43cf3c0 100644
--- a/runners/direct-java/pom.xml
+++ b/runners/direct-java/pom.xml
@@ -40,6 +40,19 @@
</resource>
</resources>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-924 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/runners/flink/pom.xml
----------------------------------------------------------------------
diff --git a/runners/flink/pom.xml b/runners/flink/pom.xml
index e012c4b..1b73922 100644
--- a/runners/flink/pom.xml
+++ b/runners/flink/pom.xml
@@ -95,6 +95,15 @@
</execution>
</executions>
</plugin>
+
+ <!-- BEAM-931 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
</pluginManagement>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/runners/google-cloud-dataflow-java/pom.xml
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/pom.xml b/runners/google-cloud-dataflow-java/pom.xml
index a5ffe0e..59b1465 100644
--- a/runners/google-cloud-dataflow-java/pom.xml
+++ b/runners/google-cloud-dataflow-java/pom.xml
@@ -45,6 +45,19 @@
</resource>
</resources>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-925 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 5e94848..1e4a720 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -318,6 +318,15 @@
<build>
<pluginManagement>
<plugins>
+ <!-- BEAM-931 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 50c5ae9..f30825f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -1840,8 +1840,8 @@ public class BigQueryIO {
return PDone.in(input.getPipeline());
}
- private class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
- private TableRowWriter writer = null;
+ private static class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
+ private transient TableRowWriter writer = null;
private final String tempFilePrefix;
WriteBundles(String tempFilePrefix) {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/io/jms/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/jms/pom.xml b/sdks/java/io/jms/pom.xml
index 7063ce9..40bc219 100644
--- a/sdks/java/io/jms/pom.xml
+++ b/sdks/java/io/jms/pom.xml
@@ -37,6 +37,19 @@
</properties>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-927 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/io/kafka/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kafka/pom.xml b/sdks/java/io/kafka/pom.xml
index 1ad31aa..69329a4 100644
--- a/sdks/java/io/kafka/pom.xml
+++ b/sdks/java/io/kafka/pom.xml
@@ -30,6 +30,19 @@
<description>Library to read Kafka topics.</description>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-928 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/io/kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/kinesis/pom.xml b/sdks/java/io/kinesis/pom.xml
index f874c21..e0b57db 100644
--- a/sdks/java/io/kinesis/pom.xml
+++ b/sdks/java/io/kinesis/pom.xml
@@ -30,6 +30,19 @@
<description>Library to read Kinesis streams.</description>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-929 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/io/mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/mongodb/pom.xml b/sdks/java/io/mongodb/pom.xml
index 4b100a9..17dc6e7 100644
--- a/sdks/java/io/mongodb/pom.xml
+++ b/sdks/java/io/mongodb/pom.xml
@@ -31,6 +31,19 @@
<description>IO to read and write on MongoDB.</description>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-930 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e1291eea/sdks/java/microbenchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/microbenchmarks/pom.xml b/sdks/java/microbenchmarks/pom.xml
index c95309c..06bc4df 100644
--- a/sdks/java/microbenchmarks/pom.xml
+++ b/sdks/java/microbenchmarks/pom.xml
@@ -32,6 +32,19 @@
<packaging>jar</packaging>
<build>
+ <pluginManagement>
+ <plugins>
+ <!-- BEAM-926 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
[41/50] incubator-beam git commit: Incrementally update Pending
elements when work completes
Posted by ke...@apache.org.
Incrementally update Pending elements when work completes
This reduces the amount of single-threaded updates the monitor thread
performs before firing timers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/317b5e65
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/317b5e65
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/317b5e65
Branch: refs/heads/gearpump-runner
Commit: 317b5e6577a623fa8fddeac90e6a3c9510a250e5
Parents: 9de9ce6
Author: Thomas Groh <tg...@google.com>
Authored: Fri Nov 4 11:28:03 2016 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 15:47:02 2016 -0800
----------------------------------------------------------------------
.../beam/runners/direct/WatermarkManager.java | 109 ++++++++++++++-----
1 file changed, 83 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/317b5e65/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
----------------------------------------------------------------------
diff --git a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
index f01c13c..2228cd5 100644
--- a/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
+++ b/runners/direct-java/src/main/java/org/apache/beam/runners/direct/WatermarkManager.java
@@ -43,7 +43,10 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
import org.apache.beam.runners.direct.DirectRunner.CommittedBundle;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.transforms.AppliedPTransform;
@@ -682,10 +685,16 @@ public class WatermarkManager {
private final ConcurrentLinkedQueue<PendingWatermarkUpdate> pendingUpdates;
/**
+ * A lock used to control concurrency for updating pending values.
+ */
+ private final Lock refreshLock;
+
+ /**
* A queue of pending {@link AppliedPTransform AppliedPTransforms} that have potentially
* stale data.
*/
- private final ConcurrentLinkedQueue<AppliedPTransform<?, ?, ?>> pendingRefreshes;
+ @GuardedBy("refreshLock")
+ private final Set<AppliedPTransform<?, ?, ?>> pendingRefreshes;
/**
* Creates a new {@link WatermarkManager}. All watermarks within the newly created
@@ -710,7 +719,9 @@ public class WatermarkManager {
this.clock = clock;
this.consumers = consumers;
this.pendingUpdates = new ConcurrentLinkedQueue<>();
- this.pendingRefreshes = new ConcurrentLinkedQueue<>();
+
+ this.refreshLock = new ReentrantLock();
+ this.pendingRefreshes = new HashSet<>();
transformToWatermarks = new HashMap<>();
@@ -795,13 +806,18 @@ public class WatermarkManager {
public void initialize(
Map<AppliedPTransform<?, ?, ?>, ? extends Iterable<CommittedBundle<?>>> initialBundles) {
- for (Map.Entry<AppliedPTransform<?, ?, ?>, ? extends Iterable<CommittedBundle<?>>> rootEntry :
- initialBundles.entrySet()) {
- TransformWatermarks rootWms = transformToWatermarks.get(rootEntry.getKey());
- for (CommittedBundle<?> initialBundle : rootEntry.getValue()) {
- rootWms.addPending(initialBundle);
+ refreshLock.lock();
+ try {
+ for (Map.Entry<AppliedPTransform<?, ?, ?>, ? extends Iterable<CommittedBundle<?>>> rootEntry :
+ initialBundles.entrySet()) {
+ TransformWatermarks rootWms = transformToWatermarks.get(rootEntry.getKey());
+ for (CommittedBundle<?> initialBundle : rootEntry.getValue()) {
+ rootWms.addPending(initialBundle);
+ }
+ pendingRefreshes.add(rootEntry.getKey());
}
- pendingRefreshes.offer(rootEntry.getKey());
+ } finally {
+ refreshLock.unlock();
}
}
@@ -834,6 +850,17 @@ public class WatermarkManager {
timerUpdate,
result,
earliestHold));
+ tryApplyPendingUpdates();
+ }
+
+ private void tryApplyPendingUpdates() {
+ if (refreshLock.tryLock()) {
+ try {
+ applyNUpdates(10);
+ } finally {
+ refreshLock.unlock();
+ }
+ }
}
/**
@@ -841,14 +868,24 @@ public class WatermarkManager {
* of all {@link TransformWatermarks} to be advanced as far as possible.
*/
private void applyPendingUpdates() {
- Set<AppliedPTransform<?, ?, ?>> updatedTransforms = new HashSet<>();
- PendingWatermarkUpdate pending = pendingUpdates.poll();
- while (pending != null) {
+ refreshLock.lock();
+ try {
+ applyNUpdates(-1);
+ } finally {
+ refreshLock.unlock();
+ }
+ }
+
+ @GuardedBy("refreshLock")
+ /**
+ * Applies up to {@code numUpdates}, or all available updates if numUpdates is non-positive.
+ */
+ private void applyNUpdates(int numUpdates) {
+ for (int i = 0; !pendingUpdates.isEmpty() && (i < numUpdates || numUpdates <= 0); i++) {
+ PendingWatermarkUpdate pending = pendingUpdates.poll();
applyPendingUpdate(pending);
- updatedTransforms.add(pending.getTransform());
- pending = pendingUpdates.poll();
+ pendingRefreshes.add(pending.getTransform());
}
- pendingRefreshes.addAll(updatedTransforms);
}
private void applyPendingUpdate(PendingWatermarkUpdate pending) {
@@ -905,22 +942,37 @@ public class WatermarkManager {
* watermarks to be advanced as far as possible.
*/
synchronized void refreshAll() {
- applyPendingUpdates();
- while (!pendingRefreshes.isEmpty()) {
- refreshWatermarks(pendingRefreshes.poll());
+ refreshLock.lock();
+ try {
+ applyPendingUpdates();
+ Set<AppliedPTransform<?, ?, ?>> toRefresh = pendingRefreshes;
+ while (!toRefresh.isEmpty()) {
+ toRefresh = refreshAllOf(toRefresh);
+ }
+ } finally {
+ refreshLock.unlock();
}
}
- private void refreshWatermarks(AppliedPTransform<?, ?, ?> toRefresh) {
+ private Set<AppliedPTransform<?, ?, ?>> refreshAllOf(Set<AppliedPTransform<?, ?, ?>> toRefresh) {
+ Set<AppliedPTransform<?, ?, ?>> newRefreshes = new HashSet<>();
+ for (AppliedPTransform<?, ?, ?> transform : toRefresh) {
+ newRefreshes.addAll(refreshWatermarks(transform));
+ }
+ return newRefreshes;
+ }
+
+ private Set<AppliedPTransform<?, ?, ?>> refreshWatermarks(AppliedPTransform<?, ?, ?> toRefresh) {
TransformWatermarks myWatermarks = transformToWatermarks.get(toRefresh);
WatermarkUpdate updateResult = myWatermarks.refresh();
- Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
if (updateResult.isAdvanced()) {
+ Set<AppliedPTransform<?, ?, ?>> additionalRefreshes = new HashSet<>();
for (PValue outputPValue : toRefresh.getOutput().expand()) {
additionalRefreshes.addAll(consumers.get(outputPValue));
}
+ return additionalRefreshes;
}
- pendingRefreshes.addAll(additionalRefreshes);
+ return Collections.emptySet();
}
/**
@@ -929,12 +981,17 @@ public class WatermarkManager {
*/
public Collection<FiredTimers> extractFiredTimers() {
Collection<FiredTimers> allTimers = new ArrayList<>();
- for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> watermarksEntry :
- transformToWatermarks.entrySet()) {
- Collection<FiredTimers> firedTimers = watermarksEntry.getValue().extractFiredTimers();
- allTimers.addAll(firedTimers);
+ refreshLock.lock();
+ try {
+ for (Map.Entry<AppliedPTransform<?, ?, ?>, TransformWatermarks> watermarksEntry
+ : transformToWatermarks.entrySet()) {
+ Collection<FiredTimers> firedTimers = watermarksEntry.getValue().extractFiredTimers();
+ allTimers.addAll(firedTimers);
+ }
+ return allTimers;
+ } finally {
+ refreshLock.unlock();
}
- return allTimers;
}
/**
@@ -1032,7 +1089,7 @@ public class WatermarkManager {
* Removes the hold of the provided key.
*/
public void removeHold(Object key) {
- KeyedHold oldHold = keyedHolds.get(key);
+ KeyedHold oldHold = keyedHolds.remove(key);
if (oldHold != null) {
allHolds.remove(oldHold);
}
[35/50] incubator-beam git commit: Switch DoFnInvokersTest to use a
mock ExtraContextFactory
Posted by ke...@apache.org.
Switch DoFnInvokersTest to use a mock ExtraContextFactory
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c052d2a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c052d2a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c052d2a1
Branch: refs/heads/gearpump-runner
Commit: c052d2a1a1fd36f2c21e63427a9e4a50addf85f7
Parents: 40ff9d4
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Nov 3 19:26:36 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:03 2016 -0800
----------------------------------------------------------------------
.../transforms/reflect/DoFnInvokersTest.java | 35 ++++----------------
1 file changed, 6 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c052d2a1/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index 0bfe2be..dbb7955 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -39,6 +39,7 @@ import org.apache.beam.sdk.coders.CoderRegistry;
import org.apache.beam.sdk.coders.CustomCoder;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFn.GetInitialRestriction;
+import org.apache.beam.sdk.transforms.DoFn.ExtraContextFactory;
import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.OldDoFn;
import org.apache.beam.sdk.transforms.reflect.testhelper.DoFnInvokersTestHelper;
@@ -67,41 +68,17 @@ public class DoFnInvokersTest {
@Mock private DoFn.InputProvider<String> mockInputProvider;
@Mock private DoFn.OutputReceiver<String> mockOutputReceiver;
@Mock private WindowingInternals<String, String> mockWindowingInternals;
+ @Mock private ExtraContextFactory<String, String> extraContextFactory;
@Mock private OldDoFn<String, String> mockOldDoFn;
- private DoFn.ExtraContextFactory<String, String> extraContextFactory;
-
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- this.extraContextFactory =
- new DoFn.ExtraContextFactory<String, String>() {
- @Override
- public BoundedWindow window() {
- return mockWindow;
- }
-
- @Override
- public DoFn.InputProvider<String> inputProvider() {
- return mockInputProvider;
- }
-
- @Override
- public DoFn.OutputReceiver<String> outputReceiver() {
- return mockOutputReceiver;
- }
-
- @Override
- public WindowingInternals<String, String> windowingInternals() {
- return mockWindowingInternals;
- }
-
- @Override
- public <RestrictionT> RestrictionTracker<RestrictionT> restrictionTracker() {
- return null;
- }
- };
+ when(extraContextFactory.window()).thenReturn(mockWindow);
+ when(extraContextFactory.inputProvider()).thenReturn(mockInputProvider);
+ when(extraContextFactory.outputReceiver()).thenReturn(mockOutputReceiver);
+ when(extraContextFactory.windowingInternals()).thenReturn(mockWindowingInternals);
}
private ProcessContinuation invokeProcessElement(DoFn<String, String> fn) {
[23/50] incubator-beam git commit: move findbugs execution to release
profile, enable in all modules
Posted by ke...@apache.org.
move findbugs execution to release profile, enable in all modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/fae52a3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/fae52a3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/fae52a3f
Branch: refs/heads/gearpump-runner
Commit: fae52a3f8f4963524c7883450aa740aefaf4a9c6
Parents: 1102455
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 08:50:07 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 10:43:11 2016 -0800
----------------------------------------------------------------------
examples/pom.xml | 5 +++++
runners/pom.xml | 5 +++++
sdks/java/core/pom.xml | 5 -----
sdks/pom.xml | 5 +++++
4 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fae52a3f/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index c6f9cb3..eb42861 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -54,6 +54,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fae52a3f/runners/pom.xml
----------------------------------------------------------------------
diff --git a/runners/pom.xml b/runners/pom.xml
index 9c821cc..8084d0b 100644
--- a/runners/pom.xml
+++ b/runners/pom.xml
@@ -49,6 +49,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fae52a3f/sdks/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/core/pom.xml b/sdks/java/core/pom.xml
index 7906afb..77a3309 100644
--- a/sdks/java/core/pom.xml
+++ b/sdks/java/core/pom.xml
@@ -147,11 +147,6 @@
</executions>
</plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
-
<!-- Coverage analysis for unit tests. -->
<plugin>
<groupId>org.jacoco</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/fae52a3f/sdks/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/pom.xml b/sdks/pom.xml
index 6347fe1..29ccd37 100644
--- a/sdks/pom.xml
+++ b/sdks/pom.xml
@@ -45,6 +45,11 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ </plugin>
</plugins>
</build>
</profile>
[47/50] incubator-beam git commit: This closes #1303
Posted by ke...@apache.org.
This closes #1303
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/99505e12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/99505e12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/99505e12
Branch: refs/heads/gearpump-runner
Commit: 99505e1256082824aebab3da26128a1e52fd7c17
Parents: 339dee9 e2856fb
Author: Thomas Groh <tg...@google.com>
Authored: Mon Nov 7 18:04:51 2016 -0800
Committer: Thomas Groh <tg...@google.com>
Committed: Mon Nov 7 18:04:51 2016 -0800
----------------------------------------------------------------------
.../direct/UnboundedReadEvaluatorFactory.java | 6 ++----
.../UnboundedReadEvaluatorFactoryTest.java | 19 ++++++-------------
2 files changed, 8 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[32/50] incubator-beam git commit: [BEAM-898] Fix Jenkins Build
Failure - IT Options Conflicts
Posted by ke...@apache.org.
[BEAM-898] Fix Jenkins Build Failure - IT Options Conflicts
This closes #1301
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/912500f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/912500f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/912500f1
Branch: refs/heads/gearpump-runner
Commit: 912500f13a1c5d6cc18752567bddd137c54795d1
Parents: 3e84a5f 1927968
Author: Luke Cwik <lc...@google.com>
Authored: Mon Nov 7 13:46:14 2016 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 13:46:14 2016 -0800
----------------------------------------------------------------------
.../apache/beam/examples/WindowedWordCountIT.java | 16 +++++++---------
.../java/org/apache/beam/examples/WordCountIT.java | 17 +++++++----------
.../examples/cookbook/BigQueryTornadoesIT.java | 16 +++++++---------
3 files changed, 21 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[30/50] incubator-beam git commit: Closes #1299
Posted by ke...@apache.org.
Closes #1299
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/3e84a5f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/3e84a5f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/3e84a5f3
Branch: refs/heads/gearpump-runner
Commit: 3e84a5f3c35a64de5e7be642ff3b7b4dcfd541b0
Parents: 9b47228 367fcac
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 13:33:05 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 13:33:05 2016 -0800
----------------------------------------------------------------------
sdks/java/io/kinesis/pom.xml | 21 ++++++--------------
.../beam/sdk/io/kinesis/CustomOptional.java | 21 ++++++++++++++++----
.../beam/sdk/io/kinesis/KinesisRecord.java | 3 +--
.../beam/sdk/io/kinesis/CustomOptionalTest.java | 10 ++++++++++
4 files changed, 34 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
[04/50] incubator-beam git commit: pom: move maven-archetypes to its
proper spot in dep hierarchy
Posted by ke...@apache.org.
pom: move maven-archetypes to its proper spot in dep hierarchy
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f19a25da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f19a25da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f19a25da
Branch: refs/heads/gearpump-runner
Commit: f19a25da225b65ecdf43342ec48389a8348e68ce
Parents: f06deac
Author: Dan Halperin <dh...@google.com>
Authored: Thu Nov 3 17:30:58 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Nov 4 10:56:39 2016 -0700
----------------------------------------------------------------------
pom.xml | 4 ----
sdks/java/pom.xml | 5 +----
2 files changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f19a25da/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 28845a5..14c38fd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,10 +145,6 @@
<module>sdks/java/build-tools</module>
<module>sdks</module>
<module>runners</module>
- <!-- sdks/java/maven-archetypes has several dependencies on the DataflowPipelineRunner.
- Until these are refactored out or a released artifact exists, we need to modify the
- build order. -->
- <module>sdks/java/maven-archetypes</module>
<module>examples</module>
</modules>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f19a25da/sdks/java/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml
index 2866cb3..fb07d25 100644
--- a/sdks/java/pom.xml
+++ b/sdks/java/pom.xml
@@ -38,10 +38,7 @@
<module>build-tools</module> -->
<module>core</module>
<module>io</module>
- <!-- sdks/java/maven-archtypes has several dependencies on the
- DataflowRunner. Until these are refactored out or
- a released artifact exists, we need to modify the build order.
- <module>maven-archetypes</module> -->
+ <module>maven-archetypes</module>
<module>extensions</module>
<module>microbenchmarks</module>
</modules>
[45/50] incubator-beam git commit: [BEAM-917] ExpectedLogs: clear
saved records after each test.
Posted by ke...@apache.org.
[BEAM-917] ExpectedLogs: clear saved records after each test.
This closes #1289
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/339dee95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/339dee95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/339dee95
Branch: refs/heads/gearpump-runner
Commit: 339dee9542497d845873dbd939c7868bdd9c0835
Parents: c6d9bf2 6bf729e
Author: Luke Cwik <lc...@google.com>
Authored: Mon Nov 7 17:38:06 2016 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Nov 7 17:38:06 2016 -0800
----------------------------------------------------------------------
.../apache/beam/sdk/testing/ExpectedLogs.java | 15 ++++++----
.../beam/sdk/testing/ExpectedLogsTest.java | 30 +++++++++++++++++++-
2 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[37/50] incubator-beam git commit: Rename BoundedWindowParameter ->
WindowParameter
Posted by ke...@apache.org.
Rename BoundedWindowParameter -> WindowParameter
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/85b908be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/85b908be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/85b908be
Branch: refs/heads/gearpump-runner
Commit: 85b908be5b5858ff9dfb5d7f04c0e3ca28dbaa05
Parents: e2db826
Author: Kenneth Knowles <kl...@google.com>
Authored: Thu Nov 3 21:27:24 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Nov 7 15:25:03 2016 -0800
----------------------------------------------------------------------
.../sdk/transforms/reflect/DoFnInvokers.java | 4 ++--
.../sdk/transforms/reflect/DoFnSignature.java | 24 ++++++++++----------
.../transforms/reflect/DoFnSignaturesTest.java | 3 ++-
3 files changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/85b908be/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
index b7f75ed..ba95f98 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokers.java
@@ -68,13 +68,13 @@ import org.apache.beam.sdk.transforms.DoFn.ExtraContextFactory;
import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
import org.apache.beam.sdk.transforms.DoFnAdapters;
import org.apache.beam.sdk.transforms.OldDoFn;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BoundedWindowParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.Cases;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.InputProviderParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.OutputReceiverParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.util.Timer;
import org.apache.beam.sdk.util.UserCodeException;
@@ -537,7 +537,7 @@ public class DoFnInvokers {
new Cases<StackManipulation>() {
@Override
- public StackManipulation dispatch(BoundedWindowParameter p) {
+ public StackManipulation dispatch(WindowParameter p) {
return simpleExtraContextParameter("window", pushExtraContextFactory);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/85b908be/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
index a189bd5..befc10b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/reflect/DoFnSignature.java
@@ -33,9 +33,9 @@ import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.DoFn.StateId;
import org.apache.beam.sdk.transforms.DoFn.TimerId;
-import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.BoundedWindowParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.RestrictionTrackerParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.Timer;
@@ -159,8 +159,8 @@ public abstract class DoFnSignature {
public <ResultT> ResultT match(Cases<ResultT> cases) {
// This could be done with reflection, but since the number of cases is small and known,
// they are simply inlined.
- if (this instanceof BoundedWindowParameter) {
- return cases.dispatch((BoundedWindowParameter) this);
+ if (this instanceof WindowParameter) {
+ return cases.dispatch((WindowParameter) this);
} else if (this instanceof RestrictionTrackerParameter) {
return cases.dispatch((RestrictionTrackerParameter) this);
} else if (this instanceof InputProviderParameter) {
@@ -182,7 +182,7 @@ public abstract class DoFnSignature {
* An interface for destructuring a {@link Parameter}.
*/
public interface Cases<ResultT> {
- ResultT dispatch(BoundedWindowParameter p);
+ ResultT dispatch(WindowParameter p);
ResultT dispatch(InputProviderParameter p);
ResultT dispatch(OutputReceiverParameter p);
ResultT dispatch(RestrictionTrackerParameter p);
@@ -197,7 +197,7 @@ public abstract class DoFnSignature {
protected abstract ResultT dispatchDefault(Parameter p);
@Override
- public ResultT dispatch(BoundedWindowParameter p) {
+ public ResultT dispatch(WindowParameter p) {
return dispatchDefault(p);
}
@@ -229,17 +229,17 @@ public abstract class DoFnSignature {
}
// These parameter descriptors are constant
- private static final BoundedWindowParameter BOUNDED_WINDOW_PARAMETER =
- new AutoValue_DoFnSignature_Parameter_BoundedWindowParameter();
+ private static final WindowParameter BOUNDED_WINDOW_PARAMETER =
+ new AutoValue_DoFnSignature_Parameter_WindowParameter();
private static final InputProviderParameter INPUT_PROVIDER_PARAMETER =
new AutoValue_DoFnSignature_Parameter_InputProviderParameter();
private static final OutputReceiverParameter OUTPUT_RECEIVER_PARAMETER =
new AutoValue_DoFnSignature_Parameter_OutputReceiverParameter();
/**
- * Returns a {@link BoundedWindowParameter}.
+ * Returns a {@link WindowParameter}.
*/
- public static BoundedWindowParameter boundedWindow() {
+ public static WindowParameter boundedWindow() {
return BOUNDED_WINDOW_PARAMETER;
}
@@ -281,8 +281,8 @@ public abstract class DoFnSignature {
* <p>All such descriptors are equal.
*/
@AutoValue
- public abstract static class BoundedWindowParameter extends Parameter {
- BoundedWindowParameter() {}
+ public abstract static class WindowParameter extends Parameter {
+ WindowParameter() {}
}
/**
@@ -380,7 +380,7 @@ public abstract class DoFnSignature {
return Iterables.any(
extraParameters(),
Predicates.or(
- Predicates.instanceOf(BoundedWindowParameter.class),
+ Predicates.instanceOf(WindowParameter.class),
Predicates.instanceOf(StateParameter.class)));
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/85b908be/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
index 4187e0a..fe78749 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnSignaturesTest.java
@@ -33,6 +33,7 @@ import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.StateParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.TimerParameter;
+import org.apache.beam.sdk.transforms.reflect.DoFnSignature.Parameter.WindowParameter;
import org.apache.beam.sdk.transforms.reflect.DoFnSignaturesTestUtils.FakeDoFn;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.TimeDomain;
@@ -310,7 +311,7 @@ public class DoFnSignaturesTest {
assertThat(sig.onTimerMethods().get(timerId).extraParameters().size(), equalTo(1));
assertThat(
sig.onTimerMethods().get(timerId).extraParameters().get(0),
- instanceOf(DoFnSignature.Parameter.BoundedWindowParameter.class));
+ instanceOf(WindowParameter.class));
}
@Test
[28/50] incubator-beam git commit: Closes #1298
Posted by ke...@apache.org.
Closes #1298
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9b47228c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9b47228c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9b47228c
Branch: refs/heads/gearpump-runner
Commit: 9b47228c3046ac2e1f45d307f44bce4ab1128c1c
Parents: baa7fb0 0604d2b
Author: Dan Halperin <dh...@google.com>
Authored: Mon Nov 7 13:31:48 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Nov 7 13:31:48 2016 -0800
----------------------------------------------------------------------
sdks/java/extensions/sorter/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------