You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2015/08/19 02:41:25 UTC
[1/2] hadoop git commit: HDFS-8435. Support CreateFlag in WebHDFS.
Contributed by Jakob Homan
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d84e4a90d -> 79af15fd3
refs/heads/trunk 71aedfabf -> 30e342a5d
HDFS-8435. Support CreateFlag in WebHDFS. Contributed by Jakob Homan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30e342a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30e342a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30e342a5
Branch: refs/heads/trunk
Commit: 30e342a5d32be5efffeb472cce76d4ed43642608
Parents: 71aedfa
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Aug 18 17:32:48 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Aug 18 17:32:48 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 +++
.../hdfs/web/resources/CreateFlagParam.java | 48 ++++++
.../hdfs/web/resources/CreateParentParam.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../datanode/web/webhdfs/ParameterParser.java | 14 ++
.../datanode/web/webhdfs/WebHdfsHandler.java | 23 ++-
.../web/resources/NamenodeWebHdfsMethods.java | 20 ++-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 21 ++-
.../apache/hadoop/hdfs/TestFileCreation.java | 154 ++++++++++---------
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 25 +++
11 files changed, 239 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index e008ecc..539b511 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* partial block.</li>
* </ol>
*
- * Following combination is not valid and will result in
+ * Following combinations are not valid and will result in
* {@link HadoopIllegalArgumentException}:
* <ol>
* <li> APPEND|OVERWRITE</li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index b661d07..ee5238a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.DelegationTokenRenewer;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1167,6 +1168,25 @@ public class WebHdfsFileSystem extends FileSystem
}
@Override
+ @SuppressWarnings("deprecation")
+ public FSDataOutputStream createNonRecursive(final Path f,
+ final FsPermission permission, final EnumSet<CreateFlag> flag,
+ final int bufferSize, final short replication, final long blockSize,
+ final Progressable progress) throws IOException {
+ statistics.incrementWriteOps(1);
+
+ final HttpOpParam.Op op = PutOpParam.Op.CREATE;
+ return new FsPathOutputStreamRunner(op, f, bufferSize,
+ new PermissionParam(applyUMask(permission)),
+ new CreateFlagParam(flag),
+ new CreateParentParam(false),
+ new BufferSizeParam(bufferSize),
+ new ReplicationParam(replication),
+ new BlockSizeParam(blockSize)
+ ).run();
+ }
+
+ @Override
public FSDataOutputStream append(final Path f, final int bufferSize,
final Progressable progress) throws IOException {
statistics.incrementWriteOps(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
new file mode 100644
index 0000000..91dbf11
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.CreateFlag;
+
+import java.util.EnumSet;
+
+/**
+ * CreateFlag enum.
+ */
+public class CreateFlagParam extends EnumSetParam<CreateFlag> {
+ public static final String NAME = "createflag";
+
+ public static final String DEFAULT = "";
+
+ private static final Domain<CreateFlag> DOMAIN = new Domain<CreateFlag>(
+ NAME, CreateFlag.class);
+
+ public CreateFlagParam(final EnumSet<CreateFlag> createFlags) {
+ super(DOMAIN, createFlags);
+ }
+
+ public CreateFlagParam(final String str) {
+ super(DOMAIN, DOMAIN.parse(str));
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
index 8152515..c701bb8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
@@ -22,7 +22,7 @@ public class CreateParentParam extends BooleanParam {
/** Parameter name. */
public static final String NAME = "createparent";
/** Default parameter value. */
- public static final String DEFAULT = FALSE;
+ public static final String DEFAULT = TRUE;
private static final Domain DOMAIN = new Domain(NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3a7b816..9092fcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -806,6 +806,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8278. When computing max-size-to-move in Balancer, count only the
storage with remaining >= default block size. (szetszwo)
+ HDFS-8435. Support CreateFlag in WebHDFS. (Jakob Homan via cdouglas)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index c6f8489..39b547c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -20,11 +20,14 @@ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
import io.netty.handler.codec.http.QueryStringDecoder;
import org.apache.commons.io.Charsets;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HAUtilClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.CreateFlagParam;
+import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
@@ -41,6 +44,7 @@ import org.apache.hadoop.security.token.Token;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -122,6 +126,16 @@ class ParameterParser {
return token;
}
+ public boolean createParent() {
+ return new CreateParentParam(param(CreateParentParam.NAME)).getValue();
+ }
+
+ public EnumSet<CreateFlag> createFlag() {
+ String cf = decodeComponent(param(CreateFlagParam.NAME), Charsets.UTF_8);
+
+ return new CreateFlagParam(cf).getValue();
+ }
+
Configuration conf() {
return conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index 9eefb66..dffe34d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -59,6 +59,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@@ -84,6 +85,9 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
public static final String APPLICATION_JSON_UTF8 =
"application/json; charset=utf-8";
+ public static final EnumSet<CreateFlag> EMPTY_CREATE_FLAG =
+ EnumSet.noneOf(CreateFlag.class);
+
private final Configuration conf;
private final Configuration confForCreate;
@@ -155,15 +159,24 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
final short replication = params.replication();
final long blockSize = params.blockSize();
final FsPermission permission = params.permission();
+ final boolean createParent = params.createParent();
- EnumSet<CreateFlag> flags = params.overwrite() ?
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
- : EnumSet.of(CreateFlag.CREATE);
+ EnumSet<CreateFlag> flags = params.createFlag();
+ if (flags.equals(EMPTY_CREATE_FLAG)) {
+ flags = params.overwrite() ?
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
+ : EnumSet.of(CreateFlag.CREATE);
+ } else {
+ if(params.overwrite()) {
+ flags.add(CreateFlag.OVERWRITE);
+ }
+ }
final DFSClient dfsClient = newDfsClient(nnId, confForCreate);
OutputStream out = dfsClient.createWrappedOutputStream(dfsClient.create(
- path, permission, flags, replication,
- blockSize, null, bufferSize, null), null);
+ path, permission, flags, createParent, replication, blockSize, null,
+ bufferSize, null), null);
+
DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, CREATED);
final URI uri = new URI(HDFS_URI_SCHEME, nnId, path, null, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 6e880f0..b03e074 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -356,13 +356,15 @@ public class NamenodeWebHdfsMethods {
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(CreateFlagParam.NAME) @DefaultValue(CreateFlagParam.DEFAULT)
+ final CreateFlagParam createFlagParam
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
owner, group, permission, overwrite, bufferSize, replication,
blockSize, modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes, createFlagParam);
}
/** Handle HTTP PUT request. */
@@ -420,14 +422,16 @@ public class NamenodeWebHdfsMethods {
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(CreateFlagParam.NAME) @DefaultValue(CreateFlagParam.DEFAULT)
+ final CreateFlagParam createFlagParam
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
group, permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName, excludeDatanodes);
+ oldSnapshotName, excludeDatanodes, createFlagParam);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -438,7 +442,8 @@ public class NamenodeWebHdfsMethods {
permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ createFlagParam);
} finally {
reset();
}
@@ -472,7 +477,8 @@ public class NamenodeWebHdfsMethods {
final XAttrSetFlagParam xattrSetFlag,
final SnapshotNameParam snapshotName,
final OldSnapshotNameParam oldSnapshotName,
- final ExcludeDatanodesParam exclDatanodes
+ final ExcludeDatanodesParam exclDatanodes,
+ final CreateFlagParam createFlagParam
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
@@ -485,7 +491,7 @@ public class NamenodeWebHdfsMethods {
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
exclDatanodes.getValue(), permission, overwrite, bufferSize,
- replication, blockSize);
+ replication, blockSize, createParent, createFlagParam);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case MKDIRS:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index f4f79b3..20b9d73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -97,6 +97,7 @@ WebHDFS REST API
* [Access Time](#Access_Time)
* [Block Size](#Block_Size)
* [Buffer Size](#Buffer_Size)
+ * [Create Flag](#Create_Flag)
* [Create Parent](#Create_Parent)
* [Delegation](#Delegation)
* [Destination](#Destination)
@@ -1632,14 +1633,30 @@ See also: [`CREATE`](#Create_and_Write_to_a_File)
See also: [`CREATE`](#Create_and_Write_to_a_File), [`APPEND`](#Append_to_a_File), [`OPEN`](#Open_and_Read_a_File)
+### Create Flag
+
+| Name | `createflag` |
+|:---- |:---- |
+| Description | Enum of possible flags to process while creating a file |
+| Type | enumerated strings |
+| Default Value | \<empty\> |
+| Valid Values | Legal combinations of create, overwrite, append and sync_block |
+| Syntax | See note below |
+
+The following combinations are not valid:
+* append,create
+* create,append,overwrite
+
+See also: [`CREATE`](#Create_and_Write_to_a_File)
+
### Create Parent
| Name | `createparent` |
|:---- |:---- |
| Description | If the parent directories do not exist, should they be created? |
| Type | boolean |
-| Default Value | false |
-| Valid Values | true |
+| Default Value | true |
+| Valid Values | true, false |
| Syntax | true |
See also: [`CREATESYMLINK`](#Create_a_Symbolic_Link)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index 525d84e..24ec988 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -33,6 +33,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
@@ -794,93 +795,94 @@ public class TestFileCreation {
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
- final Path path = new Path("/" + Time.now()
- + "-testFileCreationNonRecursive");
- FSDataOutputStream out = null;
try {
- IOException expectedException = null;
- final String nonExistDir = "/non-exist-" + Time.now();
-
- fs.delete(new Path(nonExistDir), true);
- EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
- // Create a new file in root dir, should succeed
- out = createNonRecursive(fs, path, 1, createFlag);
- out.close();
- // Create a file when parent dir exists as file, should fail
- try {
- createNonRecursive(fs, new Path(path, "Create"), 1, createFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Create a file when parent directory exists as a file"
- + " should throw ParentNotDirectoryException ",
- expectedException != null
- && expectedException instanceof ParentNotDirectoryException);
- fs.delete(path, true);
- // Create a file in a non-exist directory, should fail
- final Path path2 = new Path(nonExistDir + "/testCreateNonRecursive");
- expectedException = null;
- try {
- createNonRecursive(fs, path2, 1, createFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Create a file in a non-exist dir using"
- + " createNonRecursive() should throw FileNotFoundException ",
- expectedException != null
- && expectedException instanceof FileNotFoundException);
-
- EnumSet<CreateFlag> overwriteFlag =
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
- // Overwrite a file in root dir, should succeed
- out = createNonRecursive(fs, path, 1, overwriteFlag);
- out.close();
- // Overwrite a file when parent dir exists as file, should fail
- expectedException = null;
- try {
- createNonRecursive(fs, new Path(path, "Overwrite"), 1, overwriteFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Overwrite a file when parent directory exists as a file"
- + " should throw ParentNotDirectoryException ",
- expectedException != null
- && expectedException instanceof ParentNotDirectoryException);
- fs.delete(path, true);
- // Overwrite a file in a non-exist directory, should fail
- final Path path3 = new Path(nonExistDir + "/testOverwriteNonRecursive");
- expectedException = null;
- try {
- createNonRecursive(fs, path3, 1, overwriteFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Overwrite a file in a non-exist dir using"
- + " createNonRecursive() should throw FileNotFoundException ",
- expectedException != null
- && expectedException instanceof FileNotFoundException);
+ testFileCreationNonRecursive(fs);
} finally {
fs.close();
cluster.shutdown();
}
}
- // creates a file using DistributedFileSystem.createNonRecursive()
- static FSDataOutputStream createNonRecursive(FileSystem fs, Path name,
+ // Worker method for testing non-recursive. Extracted to allow other
+ // FileSystem implementations to re-use the tests
+ public static void testFileCreationNonRecursive(FileSystem fs) throws IOException {
+ final Path path = new Path("/" + Time.now()
+ + "-testFileCreationNonRecursive");
+ FSDataOutputStream out = null;
+ IOException expectedException = null;
+ final String nonExistDir = "/non-exist-" + Time.now();
+
+ fs.delete(new Path(nonExistDir), true);
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ // Create a new file in root dir, should succeed
+ assertNull(createNonRecursive(fs, path, 1, createFlag));
+
+ // Create a file when parent dir exists as file, should fail
+ expectedException = createNonRecursive(fs, new Path(path, "Create"), 1, createFlag);
+
+ assertTrue("Create a file when parent directory exists as a file"
+ + " should throw ParentNotDirectoryException ",
+ expectedException != null
+ && expectedException instanceof ParentNotDirectoryException);
+ fs.delete(path, true);
+ // Create a file in a non-exist directory, should fail
+ final Path path2 = new Path(nonExistDir + "/testCreateNonRecursive");
+ expectedException = createNonRecursive(fs, path2, 1, createFlag);
+
+ assertTrue("Create a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+
+ EnumSet<CreateFlag> overwriteFlag =
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
+ // Overwrite a file in root dir, should succeed
+ assertNull(createNonRecursive(fs, path, 1, overwriteFlag));
+
+ // Overwrite a file when parent dir exists as file, should fail
+ expectedException = createNonRecursive(fs, new Path(path, "Overwrite"), 1, overwriteFlag);
+
+ assertTrue("Overwrite a file when parent directory exists as a file"
+ + " should throw ParentNotDirectoryException ",
+ expectedException != null
+ && expectedException instanceof ParentNotDirectoryException);
+ fs.delete(path, true);
+
+ // Overwrite a file in a non-exist directory, should fail
+ final Path path3 = new Path(nonExistDir + "/testOverwriteNonRecursive");
+ expectedException = createNonRecursive(fs, path3, 1, overwriteFlag);
+
+ assertTrue("Overwrite a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+ }
+
+ // Attempts to create and close a file using FileSystem.createNonRecursive(),
+ // catching and returning an exception if one occurs or null
+ // if the operation is successful.
+ @SuppressWarnings("deprecation")
+ static IOException createNonRecursive(FileSystem fs, Path name,
int repl, EnumSet<CreateFlag> flag) throws IOException {
- System.out.println("createNonRecursive: Created " + name + " with " + repl
- + " replica.");
- FSDataOutputStream stm = ((DistributedFileSystem) fs).createNonRecursive(
- name, FsPermission.getDefault(), flag, fs.getConf().getInt(
- CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), (short) repl, blockSize, null);
- return stm;
+ try {
+ System.out.println("createNonRecursive: Attempting to create " + name +
+ " with " + repl + " replica.");
+ int bufferSize = fs.getConf()
+ .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096);
+ FSDataOutputStream stm = fs.createNonRecursive(name,
+ FsPermission.getDefault(), flag, bufferSize, (short) repl, blockSize,
+ null);
+ stm.close();
+ } catch (IOException e) {
+ return e;
+ }
+ return null;
}
-
-/**
- * Test that file data becomes available before file is closed.
- */
+ /**
+ * Test that file data becomes available before file is closed.
+ */
@Test
public void testFileCreationSimulated() throws IOException {
simulatedStorage = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30e342a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 8bba105..4bb3664 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestDFSClientRetries;
+import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
@@ -417,6 +418,30 @@ public class TestWebHDFS {
}
}
+ @Test
+ public void testWebHdfsCreateNonRecursive() throws IOException, URISyntaxException {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ WebHdfsFileSystem webHdfs = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+
+ webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+ TestFileCreation.testFileCreationNonRecursive(webHdfs);
+
+ } finally {
+ if(webHdfs != null) {
+ webHdfs.close();
+ }
+
+ if(cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/**
* Test snapshot rename through WebHdfs
*/
[2/2] hadoop git commit: HDFS-8435. Support CreateFlag in WebHDFS.
Contributed by Jakob Homan
Posted by cd...@apache.org.
HDFS-8435. Support CreateFlag in WebHDFS. Contributed by Jakob Homan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79af15fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79af15fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79af15fd
Branch: refs/heads/branch-2
Commit: 79af15fd37cb2e863ee29db61f4e1430dd2280c1
Parents: d84e4a9
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Aug 18 17:32:48 2015 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Aug 18 17:33:21 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/CreateFlag.java | 2 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 +++
.../hdfs/web/resources/CreateFlagParam.java | 48 ++++++
.../hdfs/web/resources/CreateParentParam.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../datanode/web/webhdfs/ParameterParser.java | 14 ++
.../datanode/web/webhdfs/WebHdfsHandler.java | 23 ++-
.../web/resources/NamenodeWebHdfsMethods.java | 20 ++-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 21 ++-
.../apache/hadoop/hdfs/TestFileCreation.java | 154 ++++++++++---------
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 25 +++
11 files changed, 239 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index e008ecc..539b511 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.classification.InterfaceStability;
* partial block.</li>
* </ol>
*
- * Following combination is not valid and will result in
+ * Following combinations are not valid and will result in
* {@link HadoopIllegalArgumentException}:
* <ol>
* <li> APPEND|OVERWRITE</li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index bf4c186..1035691 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.DelegationTokenRenewer;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1176,6 +1177,25 @@ public class WebHdfsFileSystem extends FileSystem
}
@Override
+ @SuppressWarnings("deprecation")
+ public FSDataOutputStream createNonRecursive(final Path f,
+ final FsPermission permission, final EnumSet<CreateFlag> flag,
+ final int bufferSize, final short replication, final long blockSize,
+ final Progressable progress) throws IOException {
+ statistics.incrementWriteOps(1);
+
+ final HttpOpParam.Op op = PutOpParam.Op.CREATE;
+ return new FsPathOutputStreamRunner(op, f, bufferSize,
+ new PermissionParam(applyUMask(permission)),
+ new CreateFlagParam(flag),
+ new CreateParentParam(false),
+ new BufferSizeParam(bufferSize),
+ new ReplicationParam(replication),
+ new BlockSizeParam(blockSize)
+ ).run();
+ }
+
+ @Override
public FSDataOutputStream append(final Path f, final int bufferSize,
final Progressable progress) throws IOException {
statistics.incrementWriteOps(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
new file mode 100644
index 0000000..91dbf11
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateFlagParam.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.CreateFlag;
+
+import java.util.EnumSet;
+
+/**
+ * CreateFlag enum.
+ */
+public class CreateFlagParam extends EnumSetParam<CreateFlag> {
+ public static final String NAME = "createflag";
+
+ public static final String DEFAULT = "";
+
+ private static final Domain<CreateFlag> DOMAIN = new Domain<CreateFlag>(
+ NAME, CreateFlag.class);
+
+ public CreateFlagParam(final EnumSet<CreateFlag> createFlags) {
+ super(DOMAIN, createFlags);
+ }
+
+ public CreateFlagParam(final String str) {
+ super(DOMAIN, DOMAIN.parse(str));
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
index 8152515..c701bb8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
@@ -22,7 +22,7 @@ public class CreateParentParam extends BooleanParam {
/** Parameter name. */
public static final String NAME = "createparent";
/** Default parameter value. */
- public static final String DEFAULT = FALSE;
+ public static final String DEFAULT = TRUE;
private static final Domain DOMAIN = new Domain(NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b1a959a..ce4305c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -461,6 +461,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8278. When computing max-size-to-move in Balancer, count only the
storage with remaining >= default block size. (szetszwo)
+ HDFS-8435. Support CreateFlag in WebHDFS. (Jakob Homan via cdouglas)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index c6f8489..39b547c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -20,11 +20,14 @@ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
import io.netty.handler.codec.http.QueryStringDecoder;
import org.apache.commons.io.Charsets;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HAUtilClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.CreateFlagParam;
+import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
@@ -41,6 +44,7 @@ import org.apache.hadoop.security.token.Token;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -122,6 +126,16 @@ class ParameterParser {
return token;
}
+ public boolean createParent() {
+ return new CreateParentParam(param(CreateParentParam.NAME)).getValue();
+ }
+
+ public EnumSet<CreateFlag> createFlag() {
+ String cf = decodeComponent(param(CreateFlagParam.NAME), Charsets.UTF_8);
+
+ return new CreateFlagParam(cf).getValue();
+ }
+
Configuration conf() {
return conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index 9eefb66..dffe34d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -59,6 +59,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@@ -84,6 +85,9 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
public static final String APPLICATION_JSON_UTF8 =
"application/json; charset=utf-8";
+ public static final EnumSet<CreateFlag> EMPTY_CREATE_FLAG =
+ EnumSet.noneOf(CreateFlag.class);
+
private final Configuration conf;
private final Configuration confForCreate;
@@ -155,15 +159,24 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
final short replication = params.replication();
final long blockSize = params.blockSize();
final FsPermission permission = params.permission();
+ final boolean createParent = params.createParent();
- EnumSet<CreateFlag> flags = params.overwrite() ?
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
- : EnumSet.of(CreateFlag.CREATE);
+ EnumSet<CreateFlag> flags = params.createFlag();
+ if (flags.equals(EMPTY_CREATE_FLAG)) {
+ flags = params.overwrite() ?
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
+ : EnumSet.of(CreateFlag.CREATE);
+ } else {
+ if(params.overwrite()) {
+ flags.add(CreateFlag.OVERWRITE);
+ }
+ }
final DFSClient dfsClient = newDfsClient(nnId, confForCreate);
OutputStream out = dfsClient.createWrappedOutputStream(dfsClient.create(
- path, permission, flags, replication,
- blockSize, null, bufferSize, null), null);
+ path, permission, flags, createParent, replication, blockSize, null,
+ bufferSize, null), null);
+
DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, CREATED);
final URI uri = new URI(HDFS_URI_SCHEME, nnId, path, null, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 6e880f0..b03e074 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -356,13 +356,15 @@ public class NamenodeWebHdfsMethods {
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(CreateFlagParam.NAME) @DefaultValue(CreateFlagParam.DEFAULT)
+ final CreateFlagParam createFlagParam
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
owner, group, permission, overwrite, bufferSize, replication,
blockSize, modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes, createFlagParam);
}
/** Handle HTTP PUT request. */
@@ -420,14 +422,16 @@ public class NamenodeWebHdfsMethods {
@QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT)
final OldSnapshotNameParam oldSnapshotName,
@QueryParam(ExcludeDatanodesParam.NAME) @DefaultValue(ExcludeDatanodesParam.DEFAULT)
- final ExcludeDatanodesParam excludeDatanodes
+ final ExcludeDatanodesParam excludeDatanodes,
+ @QueryParam(CreateFlagParam.NAME) @DefaultValue(CreateFlagParam.DEFAULT)
+ final CreateFlagParam createFlagParam
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
group, permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName, excludeDatanodes);
+ oldSnapshotName, excludeDatanodes, createFlagParam);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -438,7 +442,8 @@ public class NamenodeWebHdfsMethods {
permission, overwrite, bufferSize, replication, blockSize,
modificationTime, accessTime, renameOptions, createParent,
delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes);
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ createFlagParam);
} finally {
reset();
}
@@ -472,7 +477,8 @@ public class NamenodeWebHdfsMethods {
final XAttrSetFlagParam xattrSetFlag,
final SnapshotNameParam snapshotName,
final OldSnapshotNameParam oldSnapshotName,
- final ExcludeDatanodesParam exclDatanodes
+ final ExcludeDatanodesParam exclDatanodes,
+ final CreateFlagParam createFlagParam
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
@@ -485,7 +491,7 @@ public class NamenodeWebHdfsMethods {
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
exclDatanodes.getValue(), permission, overwrite, bufferSize,
- replication, blockSize);
+ replication, blockSize, createParent, createFlagParam);
return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
}
case MKDIRS:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index b440469..39d435a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -97,6 +97,7 @@ WebHDFS REST API
* [Access Time](#Access_Time)
* [Block Size](#Block_Size)
* [Buffer Size](#Buffer_Size)
+ * [Create Flag](#Create_Flag)
* [Create Parent](#Create_Parent)
* [Delegation](#Delegation)
* [Destination](#Destination)
@@ -1633,14 +1634,30 @@ See also: [`CREATE`](#Create_and_Write_to_a_File)
See also: [`CREATE`](#Create_and_Write_to_a_File), [`APPEND`](#Append_to_a_File), [`OPEN`](#Open_and_Read_a_File)
+### Create Flag
+
+| Name | `createflag` |
+|:---- |:---- |
+| Description | Enum of possible flags to process while creating a file |
+| Type | enumerated strings |
+| Default Value | \<empty\> |
+| Valid Values | Legal combinations of create, overwrite, append and sync_block |
+| Syntax | See note below |
+
+The following combinations are not valid:
+* append,create
+* create,append,overwrite
+
+See also: [`CREATE`](#Create_and_Write_to_a_File)
+
### Create Parent
| Name | `createparent` |
|:---- |:---- |
| Description | If the parent directories do not exist, should they be created? |
| Type | boolean |
-| Default Value | false |
-| Valid Values | true |
+| Default Value | true |
+| Valid Values | true, false |
| Syntax | true |
See also: [`CREATESYMLINK`](#Create_a_Symbolic_Link)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index 525d84e..24ec988 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -33,6 +33,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
@@ -794,93 +795,94 @@ public class TestFileCreation {
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
- final Path path = new Path("/" + Time.now()
- + "-testFileCreationNonRecursive");
- FSDataOutputStream out = null;
try {
- IOException expectedException = null;
- final String nonExistDir = "/non-exist-" + Time.now();
-
- fs.delete(new Path(nonExistDir), true);
- EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
- // Create a new file in root dir, should succeed
- out = createNonRecursive(fs, path, 1, createFlag);
- out.close();
- // Create a file when parent dir exists as file, should fail
- try {
- createNonRecursive(fs, new Path(path, "Create"), 1, createFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Create a file when parent directory exists as a file"
- + " should throw ParentNotDirectoryException ",
- expectedException != null
- && expectedException instanceof ParentNotDirectoryException);
- fs.delete(path, true);
- // Create a file in a non-exist directory, should fail
- final Path path2 = new Path(nonExistDir + "/testCreateNonRecursive");
- expectedException = null;
- try {
- createNonRecursive(fs, path2, 1, createFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Create a file in a non-exist dir using"
- + " createNonRecursive() should throw FileNotFoundException ",
- expectedException != null
- && expectedException instanceof FileNotFoundException);
-
- EnumSet<CreateFlag> overwriteFlag =
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
- // Overwrite a file in root dir, should succeed
- out = createNonRecursive(fs, path, 1, overwriteFlag);
- out.close();
- // Overwrite a file when parent dir exists as file, should fail
- expectedException = null;
- try {
- createNonRecursive(fs, new Path(path, "Overwrite"), 1, overwriteFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Overwrite a file when parent directory exists as a file"
- + " should throw ParentNotDirectoryException ",
- expectedException != null
- && expectedException instanceof ParentNotDirectoryException);
- fs.delete(path, true);
- // Overwrite a file in a non-exist directory, should fail
- final Path path3 = new Path(nonExistDir + "/testOverwriteNonRecursive");
- expectedException = null;
- try {
- createNonRecursive(fs, path3, 1, overwriteFlag);
- } catch (IOException e) {
- expectedException = e;
- }
- assertTrue("Overwrite a file in a non-exist dir using"
- + " createNonRecursive() should throw FileNotFoundException ",
- expectedException != null
- && expectedException instanceof FileNotFoundException);
+ testFileCreationNonRecursive(fs);
} finally {
fs.close();
cluster.shutdown();
}
}
- // creates a file using DistributedFileSystem.createNonRecursive()
- static FSDataOutputStream createNonRecursive(FileSystem fs, Path name,
+ // Worker method for testing non-recursive. Extracted to allow other
+ // FileSystem implementations to re-use the tests
+ public static void testFileCreationNonRecursive(FileSystem fs) throws IOException {
+ final Path path = new Path("/" + Time.now()
+ + "-testFileCreationNonRecursive");
+ FSDataOutputStream out = null;
+ IOException expectedException = null;
+ final String nonExistDir = "/non-exist-" + Time.now();
+
+ fs.delete(new Path(nonExistDir), true);
+ EnumSet<CreateFlag> createFlag = EnumSet.of(CreateFlag.CREATE);
+ // Create a new file in root dir, should succeed
+ assertNull(createNonRecursive(fs, path, 1, createFlag));
+
+ // Create a file when parent dir exists as file, should fail
+ expectedException = createNonRecursive(fs, new Path(path, "Create"), 1, createFlag);
+
+ assertTrue("Create a file when parent directory exists as a file"
+ + " should throw ParentNotDirectoryException ",
+ expectedException != null
+ && expectedException instanceof ParentNotDirectoryException);
+ fs.delete(path, true);
+ // Create a file in a non-exist directory, should fail
+ final Path path2 = new Path(nonExistDir + "/testCreateNonRecursive");
+ expectedException = createNonRecursive(fs, path2, 1, createFlag);
+
+ assertTrue("Create a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+
+ EnumSet<CreateFlag> overwriteFlag =
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
+ // Overwrite a file in root dir, should succeed
+ assertNull(createNonRecursive(fs, path, 1, overwriteFlag));
+
+ // Overwrite a file when parent dir exists as file, should fail
+ expectedException = createNonRecursive(fs, new Path(path, "Overwrite"), 1, overwriteFlag);
+
+ assertTrue("Overwrite a file when parent directory exists as a file"
+ + " should throw ParentNotDirectoryException ",
+ expectedException != null
+ && expectedException instanceof ParentNotDirectoryException);
+ fs.delete(path, true);
+
+ // Overwrite a file in a non-exist directory, should fail
+ final Path path3 = new Path(nonExistDir + "/testOverwriteNonRecursive");
+ expectedException = createNonRecursive(fs, path3, 1, overwriteFlag);
+
+ assertTrue("Overwrite a file in a non-exist dir using"
+ + " createNonRecursive() should throw FileNotFoundException ",
+ expectedException != null
+ && expectedException instanceof FileNotFoundException);
+ }
+
+ // Attempts to create and close a file using FileSystem.createNonRecursive(),
+ // catching and returning an exception if one occurs or null
+ // if the operation is successful.
+ @SuppressWarnings("deprecation")
+ static IOException createNonRecursive(FileSystem fs, Path name,
int repl, EnumSet<CreateFlag> flag) throws IOException {
- System.out.println("createNonRecursive: Created " + name + " with " + repl
- + " replica.");
- FSDataOutputStream stm = ((DistributedFileSystem) fs).createNonRecursive(
- name, FsPermission.getDefault(), flag, fs.getConf().getInt(
- CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), (short) repl, blockSize, null);
- return stm;
+ try {
+ System.out.println("createNonRecursive: Attempting to create " + name +
+ " with " + repl + " replica.");
+ int bufferSize = fs.getConf()
+ .getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096);
+ FSDataOutputStream stm = fs.createNonRecursive(name,
+ FsPermission.getDefault(), flag, bufferSize, (short) repl, blockSize,
+ null);
+ stm.close();
+ } catch (IOException e) {
+ return e;
+ }
+ return null;
}
-
-/**
- * Test that file data becomes available before file is closed.
- */
+ /**
+ * Test that file data becomes available before file is closed.
+ */
@Test
public void testFileCreationSimulated() throws IOException {
simulatedStorage = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79af15fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index db367c9..3f11c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.TestDFSClientRetries;
+import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
@@ -430,6 +431,30 @@ public class TestWebHDFS {
}
}
+ @Test
+ public void testWebHdfsCreateNonRecursive() throws IOException, URISyntaxException {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ WebHdfsFileSystem webHdfs = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+
+ webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+
+ TestFileCreation.testFileCreationNonRecursive(webHdfs);
+
+ } finally {
+ if(webHdfs != null) {
+ webHdfs.close();
+ }
+
+ if(cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/**
* Test snapshot rename through WebHdfs
*/