You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by gg...@apache.org on 2020/09/14 16:41:27 UTC
[commons-vfs] branch master updated: VFS-570 Add HDFS write support
(#114)
This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-vfs.git
The following commit(s) were added to refs/heads/master by this push:
new 423c08d VFS-570 Add HDFS write support (#114)
423c08d is described below
commit 423c08d2d2812563411053320bbd95b69afb5154
Author: garpinc <ga...@hotmail.com>
AuthorDate: Mon Sep 14 12:41:20 2020 -0400
VFS-570 Add HDFS write support (#114)
* VFS-570 Add HDFS write support
* VFS-570 fix is Writable assertion
* VFS-570 address reviewer concerns
Co-authored-by: Garry Boyce <ga...@cambridgesemantics.com>
---
.../commons/vfs2/provider/hdfs/HdfsFileObject.java | 70 +++++++++++++++++++++-
.../vfs2/provider/hdfs/HdfsFileProvider.java | 8 ++-
.../commons/vfs2/provider/hdfs/HdfsFileSystem.java | 3 +-
.../provider/hdfs/test/HdfsFileProviderTest.java | 4 +-
4 files changed, 78 insertions(+), 7 deletions(-)
diff --git a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileObject.java b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileObject.java
index cac98a0..4d41858 100644
--- a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileObject.java
+++ b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileObject.java
@@ -17,7 +17,9 @@
package org.apache.commons.vfs2.provider.hdfs;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -30,6 +32,7 @@ import org.apache.commons.vfs2.RandomAccessContent;
import org.apache.commons.vfs2.provider.AbstractFileName;
import org.apache.commons.vfs2.provider.AbstractFileObject;
import org.apache.commons.vfs2.util.RandomAccessMode;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -67,7 +70,25 @@ public class HdfsFileObject extends AbstractFileObject<HdfsFileSystem> {
*/
@Override
public boolean canRenameTo(final FileObject newfile) {
- throw new UnsupportedOperationException();
+ if (!super.canRenameTo(newfile)) {
+ return false;
+ }
+
+ FileStatus newfileStat = null;
+ try {
+ newfileStat = this.hdfs.getFileStatus(new Path(newfile.getName().getPath()));
+ } catch (final FileNotFoundException e) {
+ // do nothing
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ if (newfileStat == null) {
+ return true;
+ } else if (newfileStat.isDirectory() || newfileStat.isFile()) {
+ return false;
+ }
+
+ return false;
}
/**
@@ -119,10 +140,48 @@ public class HdfsFileObject extends AbstractFileObject<HdfsFileSystem> {
}
/**
+ * @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetOutputStream()
+ */
+ @Override
+ protected OutputStream doGetOutputStream(final boolean append) throws Exception {
+ if (append) {
+ throw new FileSystemException("vfs.provider/write-append-not-supported.error", this.path.getName());
+ } else {
+ FSDataOutputStream out = hdfs.create(this.path);
+ return out;
+ }
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileObject#doDelete()
+ */
+ @Override
+ protected void doDelete() throws Exception {
+ hdfs.delete(this.path, true);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileObject#doCreateFolder()
+ */
+ @Override
+ protected void doCreateFolder() throws Exception {
+ hdfs.mkdirs(this.path);
+ }
+
+ /**
+ * @see org.apache.commons.vfs2.provider.AbstractFileObject#doRename(FileObject)
+ */
+ @Override
+ protected void doRename(FileObject newfile) throws Exception {
+ hdfs.rename(this.path, new Path(newfile.getName().getPath()));
+ }
+
+ /**
* @see org.apache.commons.vfs2.provider.AbstractFileObject#doGetLastModifiedTime()
*/
@Override
protected long doGetLastModifiedTime() throws Exception {
+ doAttach();
if (null != this.stat) {
return this.stat.getModificationTime();
}
@@ -181,7 +240,7 @@ public class HdfsFileObject extends AbstractFileObject<HdfsFileSystem> {
*/
@Override
protected boolean doIsWriteable() throws Exception {
- return false;
+ return true;
}
/**
@@ -235,7 +294,12 @@ public class HdfsFileObject extends AbstractFileObject<HdfsFileSystem> {
*/
@Override
protected boolean doSetLastModifiedTime(final long modtime) throws Exception {
- throw new UnsupportedOperationException();
+ try {
+ hdfs.setTimes(this.path, modtime, System.currentTimeMillis());
+ } catch (IOException ioe) {
+ throw new FileSystemException(ioe);
+ }
+ return true;
}
/**
diff --git a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileProvider.java b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileProvider.java
index 983647a..d9376ec 100644
--- a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileProvider.java
+++ b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileProvider.java
@@ -37,7 +37,13 @@ import org.apache.commons.vfs2.provider.http.HttpFileNameParser;
public class HdfsFileProvider extends AbstractOriginatingFileProvider {
static final Collection<Capability> CAPABILITIES = Collections
- .unmodifiableCollection(Arrays.asList(Capability.GET_TYPE, Capability.READ_CONTENT, Capability.URI, Capability.GET_LAST_MODIFIED,
+ .unmodifiableCollection(Arrays.asList(Capability.GET_TYPE, Capability.READ_CONTENT,
+ Capability.CREATE,
+ Capability.DELETE,
+ Capability.RENAME,
+ Capability.WRITE_CONTENT,
+ Capability.URI, Capability.GET_LAST_MODIFIED,
+ Capability.SET_LAST_MODIFIED_FILE,
Capability.ATTRIBUTES, Capability.RANDOM_ACCESS_READ, Capability.DIRECTORY_READ_CONTENT,
Capability.LIST_CHILDREN));
diff --git a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
index 19e89b5..72b60b4 100644
--- a/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
+++ b/commons-vfs2/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
@@ -86,7 +86,7 @@ public class HdfsFileSystem extends AbstractFileSystem {
*/
@Override
protected FileObject createFile(final AbstractFileName name) throws Exception {
- throw new FileSystemException("Operation not supported");
+ return resolveFile(name);
}
/**
@@ -166,6 +166,7 @@ public class HdfsFileSystem extends AbstractFileSystem {
}
final Path filePath = new Path(path);
file = new HdfsFileObject((AbstractFileName) name, this, fs, filePath);
+ file = decorateFileObject(file);
if (useCache) {
this.putFileToCache(file);
}
diff --git a/commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/hdfs/test/HdfsFileProviderTest.java b/commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/hdfs/test/HdfsFileProviderTest.java
index ee1f789..c0c5466 100644
--- a/commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/hdfs/test/HdfsFileProviderTest.java
+++ b/commons-vfs2/src/test/java/org/apache/commons/vfs2/provider/hdfs/test/HdfsFileProviderTest.java
@@ -159,7 +159,7 @@ public class HdfsFileProviderTest {
return f;
}
- @Test(expected = UnsupportedOperationException.class)
+ @Test
public void testCanRenameTo() throws Exception {
final FileObject fo = createTestFile(hdfs);
Assert.assertNotNull(fo);
@@ -286,7 +286,7 @@ public class HdfsFileProviderTest {
// Create the test file
final FileObject file = createTestFile(hdfs);
Assert.assertTrue(fo.exists());
- Assert.assertFalse(file.isWriteable());
+ Assert.assertTrue(file.isWriteable());
}
@Test