You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/11/08 21:06:10 UTC
svn commit: r1199431 - in
/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/
src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/
src/main/java/org/apache/hadoop/hdfs/web/ src/main/java/org/apache/ha...
Author: szetszwo
Date: Tue Nov 8 20:06:10 2011
New Revision: 1199431
URL: http://svn.apache.org/viewvc?rev=1199431&view=rev
Log:
svn merge -c 1199403 from 0.23 for HDFS-2540.
Modified:
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
Propchange: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov 8 20:06:10 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:1196812,1197335,1198905
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360,1189546,1189932,1189982,1190077,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1197329,1198903
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:1196812,1197335,1198905,1199403
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360,1189546,1189932,1189982,1190077,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1197329,1198903,1199396
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1199431&r1=1199430&r2=1199431&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Nov 8 20:06:10 2011
@@ -1188,6 +1188,10 @@ Release 0.23.0 - 2011-11-01
HDFS-2528. Webhdfs: set delegation kind to WEBHDFS and add a HDFS token
when http requests are redirected to datanode. (szetszwo)
+ HDFS-2540. Webhdfs: change "Expect: 100-continue" to two-step write; change
+ "HdfsFileStatus" and "localName" respectively to "FileStatus" and
+ "pathSuffix" in JSON response. (szetszwo)
+
BREAKDOWN OF HDFS-1073 SUBTASKS
HDFS-1521. Persist transaction ID on disk between NN restarts.
Propchange: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Nov 8 20:06:10 2011
@@ -1,5 +1,5 @@
-/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1196812,1197335,1198905
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360,1189546,1189932,1189982,1190077,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1197329,1198903
+/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1196812,1197335,1198905,1199403
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177100,1177161,1177487,1177531,1177757,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360,1189546,1189932,1189982,1190077,1190708,1195575,1195656,1195731,1195754,1196113,1196129,1197329,1198903,1199396
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-1052/src/java:987665-1095512
Modified: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java?rev=1199431&r1=1199430&r2=1199431&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java Tue Nov 8 20:06:10 2011
@@ -48,6 +48,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
@@ -576,8 +577,8 @@ public class NamenodeWebHdfsMethods {
@Override
public void write(final OutputStream outstream) throws IOException {
final PrintStream out = new PrintStream(outstream);
- out.println("{\"" + HdfsFileStatus.class.getSimpleName() + "es\":{\""
- + HdfsFileStatus.class.getSimpleName() + "\":[");
+ out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
+ + FileStatus.class.getSimpleName() + "\":[");
final HdfsFileStatus[] partial = first.getPartialListing();
if (partial.length > 0) {
Modified: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1199431&r1=1199430&r2=1199431&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original)
+++ hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Tue Nov 8 20:06:10 2011
@@ -28,6 +28,7 @@ import java.util.TreeMap;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -149,7 +150,7 @@ public class JsonUtil {
return null;
}
final Map<String, Object> m = new TreeMap<String, Object>();
- m.put("localName", status.getLocalName());
+ m.put("pathSuffix", status.getLocalName());
m.put("type", PathType.valueOf(status));
if (status.isSymlink()) {
m.put("symlink", status.getSymlink());
@@ -163,8 +164,7 @@ public class JsonUtil {
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
m.put("replication", status.getReplication());
- return includeType ? toJsonString(HdfsFileStatus.class, m) :
- JSON.toString(m);
+ return includeType ? toJsonString(FileStatus.class, m): JSON.toString(m);
}
/** Convert a Json map to a HdfsFileStatus object. */
@@ -174,8 +174,8 @@ public class JsonUtil {
}
final Map<?, ?> m = includesType ?
- (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName()) : json;
- final String localName = (String) m.get("localName");
+ (Map<?, ?>)json.get(FileStatus.class.getSimpleName()) : json;
+ final String localName = (String) m.get("pathSuffix");
final PathType type = PathType.valueOf((String) m.get("type"));
final byte[] symlink = type != PathType.SYMLINK? null
: DFSUtil.string2Bytes((String)m.get("symlink"));
Modified: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1199431&r1=1199430&r2=1199431&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Tue Nov 8 20:06:10 2011
@@ -334,14 +334,13 @@ public class WebHdfsFileSystem extends F
final URL url = toUrl(op, fspath, parameters);
//connect and get response
- final HttpURLConnection conn = getHttpUrlConnection(url);
+ HttpURLConnection conn = getHttpUrlConnection(url);
try {
conn.setRequestMethod(op.getType().toString());
- conn.setDoOutput(op.getDoOutput());
if (op.getDoOutput()) {
- conn.setRequestProperty("Expect", "100-Continue");
- conn.setInstanceFollowRedirects(true);
+ conn = twoStepWrite(conn, op);
}
+ conn.setDoOutput(op.getDoOutput());
conn.connect();
return conn;
} catch (IOException e) {
@@ -349,6 +348,35 @@ public class WebHdfsFileSystem extends F
throw e;
}
}
+
+ /**
+ * Two-step Create/Append:
+ * Step 1) Submit a Http request with neither auto-redirect nor data.
+ * Step 2) Submit Http PUT with the URL from the Location header with data.
+ *
+ * The reason of having two-step create/append is for preventing clients to
+ * send out the data before the redirect. This issue is addressed by the
+ * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
+ * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
+ * and Java 6 http client), which do not correctly implement "Expect:
+ * 100-continue". The two-step create/append is a temporary workaround for
+ * the software library bugs.
+ */
+ private static HttpURLConnection twoStepWrite(HttpURLConnection conn,
+ final HttpOpParam.Op op) throws IOException {
+ //Step 1) Submit a Http request with neither auto-redirect nor data.
+ conn.setInstanceFollowRedirects(false);
+ conn.setDoOutput(false);
+ conn.connect();
+ validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn);
+ final String redirect = conn.getHeaderField("Location");
+ conn.disconnect();
+
+ //Step 2) Submit Http PUT with the URL from the Location header with data.
+ conn = (HttpURLConnection)new URL(redirect).openConnection();
+ conn.setRequestMethod(op.getType().toString());
+ return conn;
+ }
/**
* Run a http operation.
@@ -626,8 +654,8 @@ public class WebHdfsFileSystem extends F
final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
final Map<?, ?> json = run(op, f);
- final Map<?, ?> rootmap = (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName() + "es");
- final Object[] array = (Object[])rootmap.get(HdfsFileStatus.class.getSimpleName());
+ final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
+ final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
//convert FileStatus
final FileStatus[] statuses = new FileStatus[array.length];
Modified: hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java?rev=1199431&r1=1199430&r2=1199431&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java (original)
+++ hadoop/common/branches/branch-0.23.0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/HttpOpParam.java Tue Nov 8 20:06:10 2011
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.web.resources;
+import javax.ws.rs.core.Response;
+
+
/** Http operation parameter. */
public abstract class HttpOpParam<E extends Enum<E> & HttpOpParam.Op>
extends EnumParam<E> {
@@ -46,6 +49,49 @@ public abstract class HttpOpParam<E exte
public String toQueryString();
}
+ /** Expects HTTP response 307 "Temporary Redirect". */
+ public static class TemporaryRedirectOp implements Op {
+ static final TemporaryRedirectOp CREATE = new TemporaryRedirectOp(PutOpParam.Op.CREATE);
+ static final TemporaryRedirectOp APPEND = new TemporaryRedirectOp(PostOpParam.Op.APPEND);
+
+ /** Get an object for the given op. */
+ public static TemporaryRedirectOp valueOf(final Op op) {
+ if (op == CREATE.op) {
+ return CREATE;
+ } if (op == APPEND.op) {
+ return APPEND;
+ }
+ throw new IllegalArgumentException(op + " not found.");
+ }
+
+ private final Op op;
+
+ private TemporaryRedirectOp(final Op op) {
+ this.op = op;
+ }
+
+ @Override
+ public Type getType() {
+ return op.getType();
+ }
+
+ @Override
+ public boolean getDoOutput() {
+ return op.getDoOutput();
+ }
+
+ /** Override the original expected response with "Temporary Redirect". */
+ @Override
+ public int getExpectedHttpResponseCode() {
+ return Response.Status.TEMPORARY_REDIRECT.getStatusCode();
+ }
+
+ @Override
+ public String toQueryString() {
+ return op.toQueryString();
+ }
+ }
+
HttpOpParam(final Domain<E> domain, final E value) {
super(domain, value);
}