You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2017/04/21 20:08:46 UTC
[1/2] beam git commit: Validates that input and output GCS paths
specify a bucket
Repository: beam
Updated Branches:
refs/heads/master f3806f64f -> 0527f6b66
Validates that input and output GCS paths specify a bucket
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4efd7eb8
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4efd7eb8
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4efd7eb8
Branch: refs/heads/master
Commit: 4efd7eb8326066027d585dc7e49b3329500045ae
Parents: f3806f6
Author: Eugene Kirpichov <ki...@google.com>
Authored: Wed Apr 19 16:10:45 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Apr 21 13:07:27 2017 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/util/GcsPathValidator.java | 3 +++
.../beam/sdk/util/GcsPathValidatorTest.java | 18 ++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4efd7eb8/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
index a5b951d..f2e7a83 100644
--- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
+++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/GcsPathValidator.java
@@ -67,6 +67,9 @@ public class GcsPathValidator implements PathValidator {
public String verifyPath(String path) {
GcsPath gcsPath = getGcsPath(path);
checkArgument(gcsPath.isAbsolute(), "Must provide absolute paths for Dataflow");
+ checkArgument(!gcsPath.getObject().isEmpty(),
+ "Missing object or bucket in path: '%s', did you mean: 'gs://some-bucket/%s'?",
+ gcsPath, gcsPath.getBucket());
checkArgument(!gcsPath.getObject().contains("//"),
"Dataflow Service does not allow objects with consecutive slashes");
return gcsPath.toResourceName();
http://git-wip-us.apache.org/repos/asf/beam/blob/4efd7eb8/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
index dc36319..d4c804a 100644
--- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
+++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/util/GcsPathValidatorTest.java
@@ -64,6 +64,15 @@ public class GcsPathValidatorTest {
}
@Test
+ public void testFilePatternMissingBucket() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Missing object or bucket in path: 'gs://input/', "
+ + "did you mean: 'gs://some-bucket/input'?");
+ validator.validateInputFilePatternSupported("gs://input");
+ }
+
+ @Test
public void testWhenBucketDoesNotExist() throws Exception {
when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(false);
expectedException.expect(IllegalArgumentException.class);
@@ -84,4 +93,13 @@ public class GcsPathValidatorTest {
"Expected a valid 'gs://' path but was given '/local/path'");
validator.validateOutputFilePrefixSupported("/local/path");
}
+
+ @Test
+ public void testOutputPrefixMissingBucket() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage(
+ "Missing object or bucket in path: 'gs://output/', "
+ + "did you mean: 'gs://some-bucket/output'?");
+ validator.validateOutputFilePrefixSupported("gs://output");
+ }
}
[2/2] beam git commit: Validates that input and output GCS paths
specify a bucket
Posted by lc...@apache.org.
Validates that input and output GCS paths specify a bucket
This closes #2602
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0527f6b6
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0527f6b6
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0527f6b6
Branch: refs/heads/master
Commit: 0527f6b665d74ba32b7393ad6fdceefefc4431b3
Parents: f3806f6 4efd7eb
Author: Luke Cwik <lc...@google.com>
Authored: Fri Apr 21 13:08:29 2017 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Fri Apr 21 13:08:29 2017 -0700
----------------------------------------------------------------------
.../apache/beam/sdk/util/GcsPathValidator.java | 3 +++
.../beam/sdk/util/GcsPathValidatorTest.java | 18 ++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------