You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/28 18:44:58 UTC
[1/2] beam git commit: ResourceId: Add #isDirectory
Repository: beam
Updated Branches:
refs/heads/master 1a265e34f -> 6b97fb1c2
ResourceId: Add #isDirectory
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4041519c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4041519c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4041519c
Branch: refs/heads/master
Commit: 4041519c6a4bc232e0956190880fdad7c80ba0f9
Parents: 1a265e3
Author: Dan Halperin <dh...@google.com>
Authored: Fri Apr 28 09:13:47 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 11:44:36 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/LocalResourceId.java | 5 +++++
.../main/java/org/apache/beam/sdk/io/fs/ResourceId.java | 5 +++++
.../java/org/apache/beam/sdk/io/LocalResourceIdTest.java | 9 +++++++++
.../apache/beam/sdk/io/gcp/storage/GcsResourceId.java | 3 ++-
.../beam/sdk/io/gcp/storage/GcsResourceIdTest.java | 11 +++++++++++
.../org/apache/beam/sdk/io/hdfs/HadoopResourceId.java | 4 ++++
6 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
index 0a9b458..4e61473 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
@@ -123,6 +123,11 @@ class LocalResourceId implements ResourceId {
return "file";
}
+ @Override
+ public boolean isDirectory() {
+ return isDirectory;
+ }
+
Path getPath() {
if (cachedPath == null) {
cachedPath = Paths.get(pathString);
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
index 9b86689..dfe771f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
@@ -112,6 +112,11 @@ public interface ResourceId extends Serializable {
@Nullable String getFilename();
/**
+ * Returns {@code true} if this {@link ResourceId} represents a directory, false otherwise.
+ */
+ boolean isDirectory();
+
+ /**
* Returns the string representation of this {@link ResourceId}.
*
* <p>The corresponding {@link FileSystem#match} is required to accept this string representation.
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
index 7d57b84..7ea85cf 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
@@ -22,8 +22,10 @@ import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import java.io.File;
import java.nio.file.Paths;
@@ -224,6 +226,13 @@ public class LocalResourceIdTest {
}
@Test
+ public void testIsDirectory() throws Exception {
+ assertTrue(toResourceIdentifier("/").isDirectory());
+ assertTrue(toResourceIdentifier("/root/tmp/").isDirectory());
+ assertFalse(toResourceIdentifier("/root").isDirectory());
+ }
+
+ @Test
public void testToString() throws Exception {
File someFile = tmpFolder.newFile("somefile");
LocalResourceId fileResource =
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
index b049277..29215e7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
@@ -83,7 +83,8 @@ public class GcsResourceId implements ResourceId {
}
}
- private boolean isDirectory() {
+ @Override
+ public boolean isDirectory() {
return gcsPath.endsWith("/");
}
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
index 9cdf5a8..702e754 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
@@ -18,7 +18,9 @@
package org.apache.beam.sdk.io.gcp.storage;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -115,6 +117,15 @@ public class GcsResourceIdTest {
}
@Test
+ public void testIsDirectory() throws Exception {
+ assertTrue(toResourceIdentifier("gs://my_bucket/tmp dir/").isDirectory());
+ assertTrue(toResourceIdentifier("gs://my_bucket/").isDirectory());
+ assertTrue(toResourceIdentifier("gs://my_bucket").isDirectory());
+
+ assertFalse(toResourceIdentifier("gs://my_bucket/file").isDirectory());
+ }
+
+ @Test
public void testInvalidGcsPath() throws Exception {
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Invalid GCS URI: gs://");
http://git-wip-us.apache.org/repos/asf/beam/blob/4041519c/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
index d3068e2..8e0b58c 100644
--- a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
+++ b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
@@ -44,4 +44,8 @@ public class HadoopResourceId implements ResourceId {
public String getFilename() {
throw new UnsupportedOperationException();
}
+
+ public boolean isDirectory() {
+ throw new UnsupportedOperationException();
+ }
}
[2/2] beam git commit: This closes #2761
Posted by dh...@apache.org.
This closes #2761
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6b97fb1c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6b97fb1c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6b97fb1c
Branch: refs/heads/master
Commit: 6b97fb1c224cd996d47fcfb6f6c5ae6e05b0caf0
Parents: 1a265e3 4041519
Author: Dan Halperin <dh...@google.com>
Authored: Fri Apr 28 11:44:48 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 11:44:48 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/LocalResourceId.java | 5 +++++
.../main/java/org/apache/beam/sdk/io/fs/ResourceId.java | 5 +++++
.../java/org/apache/beam/sdk/io/LocalResourceIdTest.java | 9 +++++++++
.../apache/beam/sdk/io/gcp/storage/GcsResourceId.java | 3 ++-
.../beam/sdk/io/gcp/storage/GcsResourceIdTest.java | 11 +++++++++++
.../org/apache/beam/sdk/io/hdfs/HadoopResourceId.java | 4 ++++
6 files changed, 36 insertions(+), 1 deletion(-)
----------------------------------------------------------------------