You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/06/13 18:04:40 UTC
[1/3] incubator-beam git commit: Changed Word Counts to use
TypeDescriptors.
Repository: incubator-beam
Updated Branches:
refs/heads/master a3feeefa2 -> dbddb9f3c
Changed Word Counts to use TypeDescriptors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f0e266bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f0e266bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f0e266bb
Branch: refs/heads/master
Commit: f0e266bbd1f5781f5280703876b9fd76c7836090
Parents: 8291219
Author: Jesse Anderson <je...@smokinghand.com>
Authored: Mon May 16 13:58:08 2016 -0700
Committer: Jesse Anderson <je...@smokinghand.com>
Committed: Sat Jun 11 08:02:54 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/examples/MinimalWordCountJava8.java | 6 +++---
.../org/apache/beam/examples/MinimalWordCountJava8Test.java | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f0e266bb/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
index 398d517..d491741 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
@@ -27,7 +27,7 @@ import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.FlatMapElements;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import java.util.Arrays;
@@ -54,12 +54,12 @@ public class MinimalWordCountJava8 {
p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
.apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
- .withOutputType(new TypeDescriptor<String>() {}))
+ .withOutputType(TypeDescriptors.strings()))
.apply(Filter.byPredicate((String word) -> !word.isEmpty()))
.apply(Count.<String>perElement())
.apply(MapElements
.via((KV<String, Long> wordCount) -> wordCount.getKey() + ": " + wordCount.getValue())
- .withOutputType(new TypeDescriptor<String>() {}))
+ .withOutputType(TypeDescriptors.strings()))
// CHANGE 3 of 3: The Google Cloud Storage path is required for outputting the results to.
.apply(TextIO.Write.to("gs://YOUR_OUTPUT_BUCKET/AND_OUTPUT_PREFIX"));
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f0e266bb/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
index ae92559..f73250f 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
@@ -28,7 +28,7 @@ import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
import org.apache.beam.sdk.values.KV;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import com.google.common.collect.ImmutableList;
@@ -65,12 +65,12 @@ public class MinimalWordCountJava8Test implements Serializable {
p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
.apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
- .withOutputType(new TypeDescriptor<String>() {}))
+ .withOutputType(TypeDescriptors.strings()))
.apply(Filter.byPredicate((String word) -> !word.isEmpty()))
.apply(Count.<String>perElement())
.apply(MapElements
.via((KV<String, Long> wordCount) -> wordCount.getKey() + ": " + wordCount.getValue())
- .withOutputType(new TypeDescriptor<String>() {}))
+ .withOutputType(TypeDescriptors.strings()))
.apply(TextIO.Write.to("gs://YOUR_OUTPUT_BUCKET/AND_OUTPUT_PREFIX"));
}
[2/3] incubator-beam git commit: Updated complete examples to use
TypeDescriptors.
Posted by ke...@apache.org.
Updated complete examples to use TypeDescriptors.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/50c0ed04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/50c0ed04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/50c0ed04
Branch: refs/heads/master
Commit: 50c0ed04784cff0454d89fcacab24e2a4f051edb
Parents: f0e266b
Author: Jesse Anderson <je...@smokinghand.com>
Authored: Mon May 16 14:09:18 2016 -0700
Committer: Jesse Anderson <je...@smokinghand.com>
Committed: Sat Jun 11 08:02:55 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/examples/complete/game/GameStats.java | 5 +++--
.../java/org/apache/beam/examples/complete/game/UserScore.java | 5 +++--
.../apache/beam/examples/complete/game/HourlyTeamScoreTest.java | 5 +++--
.../org/apache/beam/examples/complete/game/UserScoreTest.java | 5 +++--
4 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/50c0ed04/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
index 7814eb1..c557913 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java
@@ -44,7 +44,7 @@ import org.apache.beam.sdk.transforms.windowing.Window;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionView;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
@@ -255,7 +255,8 @@ public class GameStats extends LeaderBoard {
PCollection<KV<String, Integer>> userEvents =
rawEvents.apply("ExtractUserScore",
MapElements.via((GameActionInfo gInfo) -> KV.of(gInfo.getUser(), gInfo.getScore()))
- .withOutputType(new TypeDescriptor<KV<String, Integer>>() {}));
+ .withOutputType(
+ TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())));
// Calculate the total score per user over fixed windows, and
// cumulative updates for late data.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/50c0ed04/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 866adef..de049e8 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -35,7 +35,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.avro.reflect.Nullable;
import org.slf4j.Logger;
@@ -168,7 +168,8 @@ public class UserScore {
return gameInfo
.apply(MapElements
.via((GameActionInfo gInfo) -> KV.of(gInfo.getKey(field), gInfo.getScore()))
- .withOutputType(new TypeDescriptor<KV<String, Integer>>() {}))
+ .withOutputType(
+ TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())))
.apply(Sum.<String>integersPerKey());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/50c0ed04/examples/java8/src/test/java/org/apache/beam/examples/complete/game/HourlyTeamScoreTest.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/HourlyTeamScoreTest.java b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/HourlyTeamScoreTest.java
index aa11c6c..5ff615a 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/HourlyTeamScoreTest.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/HourlyTeamScoreTest.java
@@ -30,7 +30,7 @@ import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.joda.time.Instant;
import org.junit.Test;
@@ -102,7 +102,8 @@ public class HourlyTeamScoreTest implements Serializable {
// run a map to access the fields in the result.
.apply(MapElements
.via((GameActionInfo gInfo) -> KV.of(gInfo.getUser(), gInfo.getScore()))
- .withOutputType(new TypeDescriptor<KV<String, Integer>>() {}));
+ .withOutputType(
+ TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())));
PAssert.that(output).containsInAnyOrder(FILTERED_EVENTS);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/50c0ed04/examples/java8/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java
index 842ea30..6f58114 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/complete/game/UserScoreTest.java
@@ -31,7 +31,7 @@ import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
-import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.junit.Assert;
import org.junit.Test;
@@ -146,7 +146,8 @@ public class UserScoreTest implements Serializable {
.apply(ParDo.of(new ParseEventFn()))
.apply(
MapElements.via((GameActionInfo gInfo) -> KV.of(gInfo.getUser(), gInfo.getScore()))
- .withOutputType(new TypeDescriptor<KV<String, Integer>>() {}));
+ .withOutputType(
+ TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())));
PAssert.that(extract).empty();
[3/3] incubator-beam git commit: This closes #393
Posted by ke...@apache.org.
This closes #393
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/dbddb9f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/dbddb9f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/dbddb9f3
Branch: refs/heads/master
Commit: dbddb9f3cda5b8052e5a0e5117a1d57508b229f9
Parents: a3feeef 50c0ed0
Author: Kenneth Knowles <kl...@google.com>
Authored: Mon Jun 13 11:04:25 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Mon Jun 13 11:04:25 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/examples/MinimalWordCountJava8.java | 6 +++---
.../java/org/apache/beam/examples/complete/game/GameStats.java | 5 +++--
.../java/org/apache/beam/examples/complete/game/UserScore.java | 5 +++--
.../org/apache/beam/examples/MinimalWordCountJava8Test.java | 6 +++---
.../beam/examples/complete/game/HourlyTeamScoreTest.java | 5 +++--
.../org/apache/beam/examples/complete/game/UserScoreTest.java | 5 +++--
6 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------