You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/04/28 17:03:23 UTC
[1/2] beam git commit: Add getFilename method to ResourceId
Repository: beam
Updated Branches:
refs/heads/master ccfff5f82 -> a8a04a290
Add getFilename method to ResourceId
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c1ceab7c
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c1ceab7c
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c1ceab7c
Branch: refs/heads/master
Commit: c1ceab7c1957f2c3144307aa324cd4ec1bc49524
Parents: ccfff5f
Author: Vikas Kedigehalli <vi...@google.com>
Authored: Thu Apr 27 15:58:26 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 10:03:09 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/LocalResourceId.java | 7 +++++++
.../main/java/org/apache/beam/sdk/io/fs/ResourceId.java | 11 +++++++++++
.../java/org/apache/beam/sdk/io/LocalResourceIdTest.java | 11 +++++++++++
.../apache/beam/sdk/io/gcp/storage/GcsResourceId.java | 11 +++++++++++
.../beam/sdk/io/gcp/storage/GcsResourceIdTest.java | 11 +++++++++++
.../org/apache/beam/sdk/io/hdfs/HadoopResourceId.java | 5 +++++
6 files changed, 56 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
index b67ec46..0a9b458 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/LocalResourceId.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
import java.util.UUID;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.io.fs.ResolveOptions;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -92,6 +93,12 @@ class LocalResourceId implements ResourceId {
}
}
+ @Override
+ @Nullable public String getFilename() {
+ Path fileName = getPath().getFileName();
+ return fileName == null ? null : fileName.toString();
+ }
+
private LocalResourceId resolveLocalPath(String other, ResolveOptions resolveOptions) {
return new LocalResourceId(
getPath().resolve(other),
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
index 26a21bb..9b86689 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/fs/ResourceId.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.fs;
import java.io.Serializable;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.io.FileSystem;
import org.apache.beam.sdk.io.FileSystems;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
@@ -100,6 +101,16 @@ public interface ResourceId extends Serializable {
*/
String getScheme();
+
+ /**
+ * Returns the name of the file or directory denoted by this {@code ResourceId}. The file name
+ * is the farthest element from the root in the directory hierarchy.
+ *
+ * @return a string representing the name of file or directory, or null if there are zero
+ * components.
+ */
+ @Nullable String getFilename();
+
/**
* Returns the string representation of this {@link ResourceId}.
*
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
index 5adfae6..7d57b84 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalResourceIdTest.java
@@ -239,6 +239,17 @@ public class LocalResourceIdTest {
assertThat(dirResource.toString(), startsWith(tmpFolder.getRoot().getAbsolutePath()));
}
+ @Test
+ public void testGetFilename() throws Exception {
+ assertEquals(toResourceIdentifier("/").getFilename(), null);
+ assertEquals(toResourceIdentifier("/root/tmp").getFilename(),
+ "tmp");
+ assertEquals(toResourceIdentifier("/root/tmp/").getFilename(),
+ "tmp");
+ assertEquals(toResourceIdentifier("/root/tmp/xyz.txt").getFilename(),
+ "xyz.txt");
+ }
+
private LocalResourceId toResourceIdentifier(String str) throws Exception {
boolean isDirectory;
if (SystemUtils.IS_OS_WINDOWS) {
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
index 38dcaf5..b049277 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceId.java
@@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import javax.annotation.Nullable;
import org.apache.beam.sdk.io.fs.ResolveOptions;
import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
@@ -91,6 +92,16 @@ public class GcsResourceId implements ResourceId {
return "gs";
}
+ @Override
+ @Nullable public String getFilename() {
+ if (gcsPath.getNameCount() <= 1) {
+ return null;
+ } else {
+ GcsPath gcsFilename = gcsPath.getFileName();
+ return gcsFilename == null ? null : gcsFilename.toString();
+ }
+ }
+
GcsPath getGcsPath() {
return gcsPath;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
index db28265..9cdf5a8 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsResourceIdTest.java
@@ -141,6 +141,17 @@ public class GcsResourceIdTest {
toResourceIdentifier("gs://my_bucket/tmp/"));
}
+ @Test
+ public void testGetFilename() throws Exception {
+ assertEquals(toResourceIdentifier("gs://my_bucket/").getFilename(), null);
+ assertEquals(toResourceIdentifier("gs://my_bucket/abc").getFilename(),
+ "abc");
+ assertEquals(toResourceIdentifier("gs://my_bucket/abc/").getFilename(),
+ "abc");
+ assertEquals(toResourceIdentifier("gs://my_bucket/abc/xyz.txt").getFilename(),
+ "xyz.txt");
+ }
+
private GcsResourceId toResourceIdentifier(String str) throws Exception {
return GcsResourceId.fromGcsPath(GcsPath.fromUri(str));
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c1ceab7c/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
index 2a29bb9..d3068e2 100644
--- a/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
+++ b/sdks/java/io/hdfs/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopResourceId.java
@@ -39,4 +39,9 @@ public class HadoopResourceId implements ResourceId {
public String getScheme() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public String getFilename() {
+ throw new UnsupportedOperationException();
+ }
}
[2/2] beam git commit: This closes #2754
Posted by dh...@apache.org.
This closes #2754
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a8a04a29
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a8a04a29
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a8a04a29
Branch: refs/heads/master
Commit: a8a04a2905e1ff639f403be7d9132acb77365e84
Parents: ccfff5f c1ceab7
Author: Dan Halperin <dh...@google.com>
Authored: Fri Apr 28 10:03:15 2017 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Fri Apr 28 10:03:15 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/LocalResourceId.java | 7 +++++++
.../main/java/org/apache/beam/sdk/io/fs/ResourceId.java | 11 +++++++++++
.../java/org/apache/beam/sdk/io/LocalResourceIdTest.java | 11 +++++++++++
.../apache/beam/sdk/io/gcp/storage/GcsResourceId.java | 11 +++++++++++
.../beam/sdk/io/gcp/storage/GcsResourceIdTest.java | 11 +++++++++++
.../org/apache/beam/sdk/io/hdfs/HadoopResourceId.java | 5 +++++
6 files changed, 56 insertions(+)
----------------------------------------------------------------------