You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/06/02 21:52:39 UTC
svn commit: r662537 - in /hadoop/core/trunk: ./
src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/ftp/ src/java/org/apache/hadoop/fs/kfs/
src/java/org/apache/hadoop/fs/s3/ src/test/org/apache/hadoop/fs/
Author: cdouglas
Date: Mon Jun 2 12:52:38 2008
New Revision: 662537
URL: http://svn.apache.org/viewvc?rev=662537&view=rev
Log:
HADOOP-3250. Extend FileSystem API to allow appending to files.
Contributed by Tsz Wo (Nicholas), SZE.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Jun 2 12:52:38 2008
@@ -114,6 +114,9 @@
HADOOP-3246. Add FTPFileSystem. (Ankur Goel via cutting)
+ HADOOP-3250. Extend FileSystem API to allow appending to files.
+ (Tsz Wo (Nicholas), SZE via cdouglas)
+
IMPROVEMENTS
HADOOP-2928. Remove deprecated FileSystem.getContentLength().
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Mon Jun 2 12:52:38 2008
@@ -128,6 +128,12 @@
dfs.open(getPathName(f), bufferSize, verifyChecksum, statistics));
}
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite,
int bufferSize, short replication, long blockSize,
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java Mon Jun 2 12:52:38 2008
@@ -228,6 +228,12 @@
@Override
public void setWorkingDirectory(Path f) { }
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
@Override
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, int bufferSize,
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Mon Jun 2 12:52:38 2008
@@ -522,6 +522,36 @@
}
/**
+ * Append to an existing file (optional operation).
+ * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null)
+ * @param f the existing file to be appended.
+ * @throws IOException
+ */
+ public FSDataOutputStream append(Path f) throws IOException {
+ return append(f, getConf().getInt("io.file.buffer.size", 4096), null);
+ }
+ /**
+ * Append to an existing file (optional operation).
+ * Same as append(f, bufferSize, null).
+ * @param f the existing file to be appended.
+ * @param bufferSize the size of the buffer to be used.
+ * @throws IOException
+ */
+ public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
+ return append(f, bufferSize, null);
+ }
+
+ /**
+ * Append to an existing file (optional operation).
+ * @param f the existing file to be appended.
+ * @param bufferSize the size of the buffer to be used.
+ * @param progress for reporting progress if it is not null.
+ * @throws IOException
+ */
+ public abstract FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException;
+
+ /**
* Get replication.
*
* @deprecated Use getFileStatus() instead
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Mon Jun 2 12:52:38 2008
@@ -104,7 +104,13 @@
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
return fs.open(f, bufferSize);
}
-
+
+ /** {@inheritDoc} */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ return fs.append(f, bufferSize, progress);
+ }
+
/** {@inheritDoc} */
@Override
public FSDataOutputStream create(Path f, FsPermission permission,
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java Mon Jun 2 12:52:38 2008
@@ -199,6 +199,12 @@
}
}
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
/**
* @param permission Currently ignored.
*/
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Mon Jun 2 12:52:38 2008
@@ -183,8 +183,8 @@
class LocalFSFileOutputStream extends OutputStream {
FileOutputStream fos;
- public LocalFSFileOutputStream(Path f) throws IOException {
- this.fos = new FileOutputStream(pathToFile(f));
+ private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
+ this.fos = new FileOutputStream(pathToFile(f), append);
}
/*
@@ -209,6 +209,20 @@
}
}
+ /** {@inheritDoc} */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ if (!exists(f)) {
+ throw new FileNotFoundException("File " + f + " not found.");
+ }
+ if (getFileStatus(f).isDir()) {
+ throw new IOException("Cannot append to a diretory (=" + f + " ).");
+ }
+ return new FSDataOutputStream(new BufferedOutputStream(
+ new LocalFSFileOutputStream(f, true), bufferSize), statistics);
+ }
+
+ /** {@inheritDoc} */
public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
short replication, long blockSize, Progressable progress)
throws IOException {
@@ -219,9 +233,8 @@
if (parent != null && !mkdirs(parent)) {
throw new IOException("Mkdirs failed to create " + parent.toString());
}
- return new FSDataOutputStream(
- new BufferedOutputStream(new LocalFSFileOutputStream(f), bufferSize),
- statistics);
+ return new FSDataOutputStream(new BufferedOutputStream(
+ new LocalFSFileOutputStream(f, false), bufferSize), statistics);
}
/** {@inheritDoc} */
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java Mon Jun 2 12:52:38 2008
@@ -243,6 +243,12 @@
return fos;
}
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
/**
* Convenience method, so that we don't open a new connection when using this
* method from within another method. Otherwise every API invocation incurs
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Mon Jun 2 12:52:38 2008
@@ -183,6 +183,12 @@
}
}
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
public FSDataOutputStream create(Path file, FsPermission permission,
boolean overwrite, int bufferSize,
short replication, long blockSize, Progressable progress)
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Mon Jun 2 12:52:38 2008
@@ -179,6 +179,12 @@
return ret.toArray(new FileStatus[0]);
}
+ /** This optional operation is not yet supported. */
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Not supported");
+ }
+
/**
* @param permission Currently ignored.
*/
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=662537&r1=662536&r2=662537&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestLocalFileSystem.java Mon Jun 2 12:52:38 2008
@@ -29,12 +29,26 @@
= System.getProperty("test.build.data","build/test/data/work-dir/localfs");
- private void writeFile(FileSystem fs, Path name) throws IOException {
+ static void writeFile(FileSystem fs, Path name) throws IOException {
FSDataOutputStream stm = fs.create(name);
stm.writeBytes("42\n");
stm.close();
}
+ static String readFile(FileSystem fs, Path name) throws IOException {
+ byte[] b = new byte[1024];
+ int offset = 0;
+ FSDataInputStream in = fs.open(name);
+ for(int remaining, n;
+ (remaining = b.length - offset) > 0 && (n = in.read(b, offset, remaining)) != -1;
+ offset += n);
+ in.close();
+
+ String s = new String(b, 0, offset);
+ System.out.println("s=" + s);
+ return s;
+ }
+
private void cleanupFile(FileSystem fs, Path name) throws IOException {
assertTrue(fs.exists(name));
fs.delete(name, true);
@@ -140,4 +154,54 @@
cleanupFile(fs, path);
}
+ public void testAppend() throws IOException {
+ Configuration conf = new Configuration();
+ final String dir = TEST_ROOT_DIR + "/append";
+ LocalFileSystem fs = FileSystem.getLocal(conf);
+
+ //normal case
+ {
+ Path f = new Path(dir, "f");
+ FSDataOutputStream out = fs.create(f);
+ out.writeBytes("something");
+ out.close();
+ assertEquals("something", readFile(fs, f));
+
+ out = fs.append(f);
+ out.writeBytes(" more");
+ out.close();
+ assertEquals("something more", readFile(fs, f));
+
+ out = fs.append(f);
+ out.writeBytes(" and more");
+ out.close();
+ assertEquals("something more and more", readFile(fs, f));
+
+ cleanupFile(fs, f);
+ }
+
+ //file not found case
+ {
+ Path f = new Path(dir, "fileNotFound");
+ try {
+ fs.append(f);
+ assertTrue(false);
+ }
+ catch(FileNotFoundException e) {
+ System.out.println("good: " + e);
+ }
+ }
+
+ //append to dir case
+ {
+ Path f = new Path(dir);
+ try {
+ fs.append(f);
+ assertTrue(false);
+ }
+ catch(IOException e) {
+ System.out.println("good: " + e);
+ }
+ }
+ }
}