You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/11/27 18:55:55 UTC
[2/5] incubator-flink git commit: [FLINK-1270] [APIs] FS.get()
supports relative paths
[FLINK-1270] [APIs] FS.get() supports relative paths
This closes #224
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/bbe54ac0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/bbe54ac0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/bbe54ac0
Branch: refs/heads/master
Commit: bbe54ac09154dd3e60870b0d26085e2e2ee564af
Parents: 3d242fd
Author: zentol <s....@web.de>
Authored: Wed Nov 26 13:16:18 2014 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Thu Nov 27 18:18:25 2014 +0100
----------------------------------------------------------------------
.../org/apache/flink/core/fs/FileSystem.java | 10 ++-
.../apache/flink/core/fs/FileSystemTest.java | 48 +++++++++++
.../java/org/apache/flink/core/fs/PathTest.java | 87 ++++++++++++++++++++
3 files changed, 142 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bbe54ac0/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
index 9483290..eabe830 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
@@ -25,6 +25,7 @@
package org.apache.flink.core.fs;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@@ -197,9 +198,12 @@ public abstract class FileSystem {
uri = new URI("file", null, uri.getPath(), null);
}
catch (URISyntaxException e) {
- // we tried to repair it, but could not. report the scheme error
- throw new IOException("The file URI '" + uri.toString() + "' is not valid. "
- + " File URIs need to specify aboslute file paths.");
+ try {
+ uri = new URI("file", null, new Path(new File(uri.getPath()).getAbsolutePath()).toUri().getPath(), null);
+ } catch (URISyntaxException ex) {
+ // we tried to repair it, but could not. report the scheme error
+ throw new IOException("The file URI '" + uri.toString() + "' is not valid.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bbe54ac0/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
new file mode 100644
index 0000000..04ebc0e
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/FileSystemTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.core.fs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import org.apache.flink.core.fs.local.LocalFileSystem;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class FileSystemTest {
+ @Test
+ public void testGet() throws URISyntaxException, IOException {
+ String scheme = "file";
+
+ assertTrue(FileSystem.get(new URI(scheme + ":///test/test")) instanceof LocalFileSystem);
+
+ try {
+ FileSystem.get(new URI(scheme + "://test/test"));
+ } catch (IOException ioe) {
+ assertTrue(ioe.getMessage().startsWith("Found local file path with authority '"));
+ }
+
+ assertTrue(FileSystem.get(new URI(scheme + ":/test/test")) instanceof LocalFileSystem);
+
+ assertTrue(FileSystem.get(new URI(scheme + ":test/test")) instanceof LocalFileSystem);
+
+ assertTrue(FileSystem.get(new URI("/test/test")) instanceof LocalFileSystem);
+
+ assertTrue(FileSystem.get(new URI("test/test")) instanceof LocalFileSystem);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/bbe54ac0/flink-core/src/test/java/org/apache/flink/core/fs/PathTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/core/fs/PathTest.java b/flink-core/src/test/java/org/apache/flink/core/fs/PathTest.java
new file mode 100644
index 0000000..7f04fa6
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/core/fs/PathTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.core.fs;
+
+import java.io.IOException;
+import java.net.URI;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class PathTest {
+ @Test
+ public void testParsing() {
+ URI u;
+ String scheme = "hdfs";
+ String authority = "localhost:8000";
+ String path = "/test/test";
+
+ // correct usage
+ // hdfs://localhost:8000/test/test
+ u = new Path(scheme + "://" + authority + path).toUri();
+ assertEquals(scheme, u.getScheme());
+ assertEquals(authority, u.getAuthority());
+ assertEquals(path, u.getPath());
+ // hdfs:///test/test
+ u = new Path(scheme + "://" + path).toUri();
+ assertEquals(scheme, u.getScheme());
+ assertEquals(null, u.getAuthority());
+ assertEquals(path, u.getPath());
+ // hdfs:/test/test
+ u = new Path(scheme + ":" + path).toUri();
+ assertEquals(scheme, u.getScheme());
+ assertEquals(null, u.getAuthority());
+ assertEquals(path, u.getPath());
+
+ // incorrect usage
+ // hdfs://test/test
+ u = new Path(scheme + ":/" + path).toUri();
+ assertEquals(scheme, u.getScheme());
+ assertEquals("test", u.getAuthority());
+ assertEquals("/test", u.getPath());
+ // hdfs:////test/test
+ u = new Path(scheme + ":///" + path).toUri();
+ assertEquals("hdfs", u.getScheme());
+ assertEquals(null, u.getAuthority());
+ assertEquals(path, u.getPath());
+ }
+
+ @Test
+ public void testMakeQualified() throws IOException {
+ String path;
+ Path p;
+ URI u;
+
+ path = "test/test";
+ p = new Path(path);
+ u = p.toUri();
+ p = p.makeQualified(FileSystem.get(u));
+ u = p.toUri();
+ assertEquals("file", u.getScheme());
+ assertEquals(null, u.getAuthority());
+ assertEquals(FileSystem.getLocalFileSystem().getWorkingDirectory().toUri().getPath() + "/" + path, u.getPath());
+
+ path = "/test/test";
+ p = new Path(path);
+ u = p.toUri();
+ p = p.makeQualified(FileSystem.get(u));
+ u = p.toUri();
+ assertEquals("file", u.getScheme());
+ assertEquals(null, u.getAuthority());
+ assertEquals(path, u.getPath());
+ }
+}