You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/28 21:13:39 UTC
[1/3] beam git commit: Use getFileName in NumberedShardedFile
Repository: beam
Updated Branches:
refs/heads/master 2bd668fdb -> dbd44faf3
Use getFileName in NumberedShardedFile
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0c269cc2
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0c269cc2
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0c269cc2
Branch: refs/heads/master
Commit: 0c269cc21674b6f1d863e3ddb8ab233303086212
Parents: 4e6dbf8
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Fri Apr 28 12:14:24 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 14:13:28 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/beam/sdk/util/NumberedShardedFile.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/0c269cc2/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
index fa22586..e904382 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/NumberedShardedFile.java
@@ -195,8 +195,7 @@ public class NumberedShardedFile implements ShardedFile {
@VisibleForTesting
boolean checkTotalNumOfFiles(Collection<Metadata> files) {
for (Metadata fileMedadata : files) {
- String fileName = fileMedadata.resourceId().toString().substring(
- fileMedadata.resourceId().getCurrentDirectory().toString().length());
+ String fileName = fileMedadata.resourceId().getFilename();
if (fileName == null) {
// this path has zero elements
[2/3] beam git commit: Remove IOChannelUtils from a few ITs
Posted by dh...@apache.org.
Remove IOChannelUtils from a few ITs
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4e6dbf8e
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4e6dbf8e
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4e6dbf8e
Branch: refs/heads/master
Commit: 4e6dbf8e1adb81147202212e3220e763075b1e26
Parents: 2bd668f
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Fri Apr 28 12:12:43 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 14:13:28 2017 -0700
----------------------------------------------------------------------
.../org/apache/beam/examples/WindowedWordCountIT.java | 13 +++++++------
.../java/org/apache/beam/examples/WordCountIT.java | 14 +++++++-------
2 files changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4e6dbf8e/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
index a53a151..b5eddb5 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WindowedWordCountIT.java
@@ -34,6 +34,8 @@ import java.util.concurrent.ThreadLocalRandom;
import org.apache.beam.examples.common.ExampleUtils;
import org.apache.beam.examples.common.WriteOneFilePerWindow.PerWindowFiles;
import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.testing.FileChecksumMatcher;
@@ -44,7 +46,6 @@ import org.apache.beam.sdk.testing.TestPipelineOptions;
import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.util.ExplicitShardedFile;
import org.apache.beam.sdk.util.FluentBackoff;
-import org.apache.beam.sdk.util.IOChannelUtils;
import org.apache.beam.sdk.util.NumberedShardedFile;
import org.apache.beam.sdk.util.ShardedFile;
import org.hamcrest.Description;
@@ -107,13 +108,13 @@ public class WindowedWordCountIT {
options.setWindowSize(10);
options.setOutput(
- IOChannelUtils.resolve(
- options.getTempRoot(),
- String.format(
+ FileSystems.matchNewResource(options.getTempRoot(), true)
+ .resolve(String.format(
"WindowedWordCountIT.%s-%tFT%<tH:%<tM:%<tS.%<tL+%s",
testName.getMethodName(), new Date(), ThreadLocalRandom.current().nextInt()),
- "output",
- "results"));
+ StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("results", StandardResolveOptions.RESOLVE_FILE).toString());
return options;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4e6dbf8e/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
----------------------------------------------------------------------
diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
index 01438de..1660b61 100644
--- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
+++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java
@@ -20,11 +20,12 @@ package org.apache.beam.examples;
import java.util.Date;
import org.apache.beam.examples.WordCount.WordCountOptions;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.FileChecksumMatcher;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.testing.TestPipelineOptions;
-import org.apache.beam.sdk.util.IOChannelUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -35,7 +36,6 @@ import org.junit.runners.JUnit4;
*/
@RunWith(JUnit4.class)
public class WordCountIT {
-
private static final String DEFAULT_INPUT =
"gs://apache-beam-samples/shakespeare/winterstale-personae";
private static final String DEFAULT_OUTPUT_CHECKSUM = "508517575eba8d8d5a54f7f0080a00951cfe84ca";
@@ -59,11 +59,11 @@ public class WordCountIT {
WordCountITOptions options = TestPipeline.testingPipelineOptions().as(WordCountITOptions.class);
options.setInputFile(DEFAULT_INPUT);
- options.setOutput(IOChannelUtils.resolve(
- options.getTempRoot(),
- String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
- "output",
- "results"));
+ options.setOutput(FileSystems.matchNewResource(options.getTempRoot(), true)
+ .resolve(String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
+ StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("results", StandardResolveOptions.RESOLVE_FILE).toString());
options.setOnSuccessMatcher(
new FileChecksumMatcher(DEFAULT_OUTPUT_CHECKSUM, options.getOutput() + "*-of-*"));
[3/3] beam git commit: This closes #2768
Posted by dh...@apache.org.
This closes #2768
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/dbd44faf
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/dbd44faf
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/dbd44faf
Branch: refs/heads/master
Commit: dbd44faf32601be6f04c351a0aa3828c2e4f07c9
Parents: 2bd668f 0c269cc
Author: Dan Halperin <dh...@google.com>
Authored: Fri Apr 28 14:13:32 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 14:13:32 2017 -0700
----------------------------------------------------------------------
.../org/apache/beam/examples/WindowedWordCountIT.java | 13 +++++++------
.../java/org/apache/beam/examples/WordCountIT.java | 14 +++++++-------
.../org/apache/beam/sdk/util/NumberedShardedFile.java | 3 +--
3 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------