You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/25 20:03:22 UTC
[1/2] incubator-beam git commit: Move the samples data to
gs://apache-beam-samples/
Repository: incubator-beam
Updated Branches:
refs/heads/master f7384e1a6 -> 10b3fee34
Move the samples data to gs://apache-beam-samples/
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/83ccf082
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/83ccf082
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/83ccf082
Branch: refs/heads/master
Commit: 83ccf082ceb129f536e8a01b755c15ec0f5c2495
Parents: f7384e1
Author: Pei He <pe...@google.com>
Authored: Wed Aug 24 17:26:50 2016 -0700
Committer: Pei He <pe...@google.com>
Committed: Wed Aug 24 17:57:07 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/beam/examples/DebuggingWordCount.java | 4 ++--
.../main/java/org/apache/beam/examples/MinimalWordCount.java | 2 +-
.../main/java/org/apache/beam/examples/WindowedWordCount.java | 2 +-
.../java/src/main/java/org/apache/beam/examples/WordCount.java | 4 ++--
.../org/apache/beam/examples/complete/StreamingWordExtract.java | 2 +-
.../src/main/java/org/apache/beam/examples/complete/TfIdf.java | 4 ++--
.../org/apache/beam/examples/complete/TopWikipediaSessions.java | 5 +++--
.../org/apache/beam/examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../java/org/apache/beam/examples/complete/TrafficRoutes.java | 2 +-
.../org/apache/beam/examples/cookbook/DatastoreWordCount.java | 2 +-
.../java/org/apache/beam/examples/cookbook/DeDupExample.java | 4 ++--
.../java/org/apache/beam/examples/cookbook/TriggerExample.java | 2 +-
.../java/org/apache/beam/examples/MinimalWordCountJava8.java | 2 +-
.../org/apache/beam/examples/complete/game/HourlyTeamScore.java | 2 +-
.../java/org/apache/beam/examples/complete/game/UserScore.java | 2 +-
.../org/apache/beam/examples/MinimalWordCountJava8Test.java | 2 +-
16 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
index 5a0930c..be3aa41 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java
@@ -100,8 +100,8 @@ import org.slf4j.LoggerFactory;
* that changing the default worker log level to TRACE or DEBUG will significantly increase
* the amount of logs output.
*
- * <p>The input file defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt} and can be
- * overridden with {@code --inputFile}.
+ * <p>The input file defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}
+ * and can be overridden with {@code --inputFile}.
*/
public class DebuggingWordCount {
/** A DoFn that filters for a specific key based upon a regular expression. */
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
index df725e3..f28a20c 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java
@@ -87,7 +87,7 @@ public class MinimalWordCount {
// Concept #1: Apply a root transform to the pipeline; in this case, TextIO.Read to read a set
// of input text files. TextIO.Read returns a PCollection where each element is one line from
// the input text (a set of Shakespeare's texts).
- p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+ p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
// Concept #2: Apply a ParDo transform to our PCollection of text lines. This ParDo invokes a
// DoFn (defined in-line) on each element that tokenizes the text line into individual words.
// The ParDo returns a PCollection<String>, where each element is an individual word in
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
index 5f60524..7af354c 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java
@@ -82,7 +82,7 @@ import org.joda.time.Instant;
*
* <p>Optionally specify the input file path via:
* {@code --inputFile=gs://INPUT_PATH},
- * which defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt}.
+ * which defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}.
*
* <p>Specify an output BigQuery dataset and optionally, a table for the output. If you don't
* specify the table, one will be created for you using the job name. If you don't specify the
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
index d42d621..0275651 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java
@@ -89,8 +89,8 @@ import org.apache.beam.sdk.values.PCollection;
* --output=gs://YOUR_OUTPUT_PREFIX
* }</pre>
*
- * <p>The input file defaults to {@code gs://dataflow-samples/shakespeare/kinglear.txt} and can be
- * overridden with {@code --inputFile}.
+ * <p>The input file defaults to {@code gs://apache-beam-samples/shakespeare/kinglear.txt}
+ * and can be overridden with {@code --inputFile}.
*/
public class WordCount {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java b/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
index 348bab8..869ea69 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/StreamingWordExtract.java
@@ -104,7 +104,7 @@ public class StreamingWordExtract {
private interface StreamingWordExtractOptions
extends ExampleOptions, ExampleBigQueryTableOptions, StreamingOptions {
@Description("Path of the file to read from")
- @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+ @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
String getInputFile();
void setInputFile(String value);
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
index a5a9392..87023ed 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java
@@ -83,7 +83,7 @@ import org.slf4j.LoggerFactory;
* --output=gs://YOUR_OUTPUT_PREFIX
* }</pre>
*
- * <p>The default input is {@code gs://dataflow-samples/shakespeare/} and can be overridden with
+ * <p>The default input is {@code gs://apache-beam-samples/shakespeare/} and can be overridden with
* {@code --input}.
*/
public class TfIdf {
@@ -94,7 +94,7 @@ public class TfIdf {
*/
private static interface Options extends PipelineOptions {
@Description("Path to the directory or GCS prefix containing files to read from")
- @Default.String("gs://dataflow-samples/shakespeare/")
+ @Default.String("gs://apache-beam-samples/shakespeare/")
String getInput();
void setInput(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
index 1b2064a..d597258 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java
@@ -64,7 +64,7 @@ import org.joda.time.Instant;
* --output=gs://YOUR_OUTPUT_PREFIX
* }</pre>
*
- * <p>The default input is {@code gs://dataflow-samples/wikipedia_edits/*.json} and can be
+ * <p>The default input is {@code gs://apache-beam-samples/wikipedia_edits/*.json} and can be
* overridden with {@code --input}.
*
* <p>The input for this example is large enough that it's a good place to enable (experimental)
@@ -77,7 +77,8 @@ import org.joda.time.Instant;
* This will automatically scale the number of workers up over time until the job completes.
*/
public class TopWikipediaSessions {
- private static final String EXPORTED_WIKI_TABLE = "gs://dataflow-samples/wikipedia_edits/*.json";
+ private static final String EXPORTED_WIKI_TABLE =
+ "gs://apache-beam-samples/wikipedia_edits/*.json";
/**
* Extracts user and timestamp from a TableRow representing a Wikipedia edit.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
index 1b27e65..e456960 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficMaxLaneFlow.java
@@ -303,7 +303,7 @@ public class TrafficMaxLaneFlow {
*/
private interface TrafficMaxLaneFlowOptions extends ExampleOptions, ExampleBigQueryTableOptions {
@Description("Path of the file to read from")
- @Default.String("gs://dataflow-samples/traffic_sensor/"
+ @Default.String("gs://apache-beam-samples/traffic_sensor/"
+ "Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv")
String getInputFile();
void setInputFile(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
index f3c2d39..95336c6 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TrafficRoutes.java
@@ -313,7 +313,7 @@ public class TrafficRoutes {
*/
private interface TrafficRoutesOptions extends ExampleOptions, ExampleBigQueryTableOptions {
@Description("Path of the file to read from")
- @Default.String("gs://dataflow-samples/traffic_sensor/"
+ @Default.String("gs://apache-beam-samples/traffic_sensor/"
+ "Freeways-5Minaa2010-01-01_to_2010-02-15_test2.csv")
String getInputFile();
void setInputFile(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
index 9a9e799..eb2165f 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java
@@ -151,7 +151,7 @@ public class DatastoreWordCount {
*/
public static interface Options extends PipelineOptions {
@Description("Path of the file to read from and store to Datastore")
- @Default.String("gs://dataflow-samples/shakespeare/kinglear.txt")
+ @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
String getInput();
void setInput(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
index d573bcd..5791710 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java
@@ -47,7 +47,7 @@ import org.apache.beam.sdk.util.gcsfs.GcsPath;
* and an output prefix on GCS:
* --output=gs://YOUR_OUTPUT_PREFIX
*
- * <p>The input defaults to {@code gs://dataflow-samples/shakespeare/*} and can be
+ * <p>The input defaults to {@code gs://apache-beam-samples/shakespeare/*} and can be
* overridden with {@code --input}.
*/
public class DeDupExample {
@@ -59,7 +59,7 @@ public class DeDupExample {
*/
private static interface Options extends PipelineOptions {
@Description("Path to the directory or GCS prefix containing files to read from")
- @Default.String("gs://dataflow-samples/shakespeare/*")
+ @Default.String("gs://apache-beam-samples/shakespeare/*")
String getInput();
void setInput(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
----------------------------------------------------------------------
diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
index db59435..2630541 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java
@@ -420,7 +420,7 @@ public class TriggerExample {
extends ExampleOptions, ExampleBigQueryTableOptions, StreamingOptions {
@Description("Input file to read from")
- @Default.String("gs://dataflow-samples/traffic_sensor/"
+ @Default.String("gs://apache-beam-samples/traffic_sensor/"
+ "Freeways-5Minaa2010-01-01_to_2010-02-15.csv")
String getInput();
void setInput(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
index a49da7b..24dd6f9 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java
@@ -55,7 +55,7 @@ public class MinimalWordCountJava8 {
Pipeline p = Pipeline.create(options);
- p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+ p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
.apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
.withOutputType(TypeDescriptors.strings()))
.apply(Filter.by((String word) -> !word.isEmpty()))
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index d408e21..cf13899 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -70,7 +70,7 @@ import org.joda.time.format.DateTimeFormatter;
* timestamped after 23:59 PST on 2015-10-18 should not be included in the analysis.
* To indicate a time before which data should be filtered out, include the {@code --startMin} arg.
* If you're using the default input specified in {@link UserScore},
- * "gs://dataflow-samples/game/gaming_data*.csv", then
+ * "gs://apache-beam-samples/game/gaming_data*.csv", then
* {@code --startMin=2015-11-16-16-10 --stopMin=2015-11-17-16-10} are good values.
*/
public class HourlyTeamScore extends UserScore {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index 65036ce..f05879f 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -182,7 +182,7 @@ public class UserScore {
@Description("Path to the data file(s) containing game data.")
// The default maps to two large Google Cloud Storage files (each ~12GB) holding two subsequent
// day's worth (roughly) of data.
- @Default.String("gs://dataflow-samples/game/gaming_data*.csv")
+ @Default.String("gs://apache-beam-samples/game/gaming_data*.csv")
String getInput();
void setInput(String value);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/83ccf082/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
----------------------------------------------------------------------
diff --git a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
index 85841a7..1819219 100644
--- a/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
+++ b/examples/java8/src/test/java/org/apache/beam/examples/MinimalWordCountJava8Test.java
@@ -60,7 +60,7 @@ public class MinimalWordCountJava8Test implements Serializable {
Pipeline p = TestPipeline.create();
p.getOptions().as(GcsOptions.class).setGcsUtil(buildMockGcsUtil());
- p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
+ p.apply(TextIO.Read.from("gs://apache-beam-samples/shakespeare/*"))
.apply(FlatMapElements.via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))
.withOutputType(TypeDescriptors.strings()))
.apply(Filter.by((String word) -> !word.isEmpty()))
[2/2] incubator-beam git commit: Closes #882
Posted by dh...@apache.org.
Closes #882
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/10b3fee3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/10b3fee3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/10b3fee3
Branch: refs/heads/master
Commit: 10b3fee342878c1b5efac3e2ae956017d70db4a2
Parents: f7384e1 83ccf08
Author: Dan Halperin <dh...@google.com>
Authored: Thu Aug 25 13:02:59 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Thu Aug 25 13:02:59 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/beam/examples/DebuggingWordCount.java | 4 ++--
.../main/java/org/apache/beam/examples/MinimalWordCount.java | 2 +-
.../main/java/org/apache/beam/examples/WindowedWordCount.java | 2 +-
.../java/src/main/java/org/apache/beam/examples/WordCount.java | 4 ++--
.../org/apache/beam/examples/complete/StreamingWordExtract.java | 2 +-
.../src/main/java/org/apache/beam/examples/complete/TfIdf.java | 4 ++--
.../org/apache/beam/examples/complete/TopWikipediaSessions.java | 5 +++--
.../org/apache/beam/examples/complete/TrafficMaxLaneFlow.java | 2 +-
.../java/org/apache/beam/examples/complete/TrafficRoutes.java | 2 +-
.../org/apache/beam/examples/cookbook/DatastoreWordCount.java | 2 +-
.../java/org/apache/beam/examples/cookbook/DeDupExample.java | 4 ++--
.../java/org/apache/beam/examples/cookbook/TriggerExample.java | 2 +-
.../java/org/apache/beam/examples/MinimalWordCountJava8.java | 2 +-
.../org/apache/beam/examples/complete/game/HourlyTeamScore.java | 2 +-
.../java/org/apache/beam/examples/complete/game/UserScore.java | 2 +-
.../org/apache/beam/examples/MinimalWordCountJava8Test.java | 2 +-
16 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------