You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/06/26 08:06:33 UTC
[36/50] [abbrv] hadoop git commit: HADOOP-14396. Add builder
interface to FileContext. Contributed by Lei (Eddy) Xu.
HADOOP-14396. Add builder interface to FileContext.
Contributed by Lei (Eddy) Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba4e623
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba4e623
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba4e623
Branch: refs/heads/HDDS-4
Commit: 1ba4e62304a70d53f1a4f76995b6e1fac3107922
Parents: 440140c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 25 14:38:33 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 25 14:38:57 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/FSDataOutputStreamBuilder.java | 22 +++++++
.../java/org/apache/hadoop/fs/FileContext.java | 66 ++++++++++++++++++++
.../main/java/org/apache/hadoop/fs/Options.java | 3 +
.../fs/FileContextMainOperationsBaseTest.java | 44 ++++++++++++-
4 files changed, 134 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 86c284a..d431293 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -116,6 +116,27 @@ public abstract class FSDataOutputStreamBuilder
protected abstract B getThisBuilder();
/**
+ * Construct from a {@link FileContext}.
+ *
+ * @param fc FileContext
+ * @param p path.
+ * @throws IOException
+ */
+ FSDataOutputStreamBuilder(@Nonnull FileContext fc,
+ @Nonnull Path p) throws IOException {
+ Preconditions.checkNotNull(fc);
+ Preconditions.checkNotNull(p);
+ this.fs = null;
+ this.path = p;
+
+ AbstractFileSystem afs = fc.getFSofPath(p);
+ FsServerDefaults defaults = afs.getServerDefaults(p);
+ bufferSize = defaults.getFileBufferSize();
+ replication = defaults.getReplication();
+ blockSize = defaults.getBlockSize();
+ }
+
+ /**
* Constructor.
*/
protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
@@ -131,6 +152,7 @@ public abstract class FSDataOutputStreamBuilder
}
protected FileSystem getFS() {
+ Preconditions.checkNotNull(fs);
return fs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 6ea69d0..5215c3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@@ -35,6 +36,8 @@ import java.util.Stack;
import java.util.TreeSet;
import java.util.Map.Entry;
+import javax.annotation.Nonnull;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -695,6 +698,69 @@ public class FileContext {
}
/**
+ * {@link FSDataOutputStreamBuilder} for {@liink FileContext}.
+ */
+ private static final class FCDataOutputStreamBuilder extends
+ FSDataOutputStreamBuilder<
+ FSDataOutputStream, FCDataOutputStreamBuilder> {
+ private final FileContext fc;
+
+ private FCDataOutputStreamBuilder(
+ @Nonnull FileContext fc, @Nonnull Path p) throws IOException {
+ super(fc, p);
+ this.fc = fc;
+ Preconditions.checkNotNull(fc);
+ }
+
+ @Override
+ protected FCDataOutputStreamBuilder getThisBuilder() {
+ return this;
+ }
+
+ @Override
+ public FSDataOutputStream build() throws IOException {
+ final EnumSet<CreateFlag> flags = getFlags();
+ List<CreateOpts> createOpts = new ArrayList<>(Arrays.asList(
+ CreateOpts.blockSize(getBlockSize()),
+ CreateOpts.bufferSize(getBufferSize()),
+ CreateOpts.repFac(getReplication()),
+ CreateOpts.perms(getPermission())
+ ));
+ if (getChecksumOpt() != null) {
+ createOpts.add(CreateOpts.checksumParam(getChecksumOpt()));
+ }
+ if (getProgress() != null) {
+ createOpts.add(CreateOpts.progress(getProgress()));
+ }
+ if (isRecursive()) {
+ createOpts.add(CreateOpts.createParent());
+ }
+ return fc.create(getPath(), flags,
+ createOpts.toArray(new CreateOpts[0]));
+ }
+ }
+
+ /**
+ * Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ * a file on indicated path.
+ *
+ * @param f the file path to create builder for.
+ * @return {@link FSDataOutputStreamBuilder} to build a
+ * {@link FSDataOutputStream}.
+ *
+ * Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ * builder parameters will be verified by {@link FileContext} and
+ * {@link AbstractFileSystem#create}. And filesystem states will be modified.
+ *
+ * Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ * same exceptions as create(Path, EnumSet, CreateOpts...).
+ */
+ public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
+ throws IOException {
+ return new FCDataOutputStreamBuilder(this, f).create();
+ }
+
+ /**
* Make(create) a directory and all the non-existent parents.
*
* @param dir - the dir to make
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 126e754..5e93286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -55,6 +55,9 @@ public final class Options {
ChecksumOpt csumOpt) {
return new ChecksumParam(csumOpt);
}
+ public static Progress progress(Progressable prog) {
+ return new Progress(prog);
+ }
public static Perms perms(FsPermission perm) {
return new Perms(perm);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 35ec4ff..62ecd9f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -810,7 +810,49 @@ public abstract class FileContextMainOperationsBaseTest {
fc.create(p, EnumSet.of(CREATE, APPEND, OVERWRITE));
Assert.fail("Excepted exception not thrown");
}
-
+
+ @Test
+ public void testBuilderCreateNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateNonExistingFile");
+ FSDataOutputStream out = fc.create(p).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).overwrite(true).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendNonExistingFile");
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, 2 * data.length);
+ }
+
+ @Test
+ public void testBuilderCreateRecursive() throws IOException {
+ Path p = getTestRootPath(fc, "test/parent/no/exist/file1");
+ try (FSDataOutputStream out = fc.create(p).build()) {
+ fail("Should throw FileNotFoundException on non-exist directory");
+ } catch (FileNotFoundException e) {
+ }
+
+ FSDataOutputStream out = fc.create(p).recursive().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
private static void writeData(FileContext fc, Path p, FSDataOutputStream out,
byte[] data, long expectedLen) throws IOException {
out.write(data, 0, data.length);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org