You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/10/27 00:53:32 UTC
svn commit: r1189520 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
CHANGES.txt src/main/java/
src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
Author: szetszwo
Date: Wed Oct 26 22:53:32 2011
New Revision: 1189520
URL: http://svn.apache.org/viewvc?rev=1189520&view=rev
Log:
svn merge -c 1189360 from trunk for HDFS-2494.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
Propchange: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 26 22:53:32 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177487,1177531,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177487,1177531,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-1052:987665-1095512
/hadoop/hdfs/branches/HDFS-265:796829-820463
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1189520&r1=1189519&r2=1189520&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 26 22:53:32 2011
@@ -1123,6 +1123,9 @@ Release 0.23.0 - Unreleased
HDFS-2411. The the auth to local mappings are not being respected, with
webhdfs enabled. (jitendra)
+ HDFS-2494. Close the streams and DFSClient in DatanodeWebHdfsMethods.
+ (Uma Maheswara Rao G via szetszwo)
+
BREAKDOWN OF HDFS-1073 SUBTASKS
HDFS-1521. Persist transaction ID on disk between NN restarts.
Propchange: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct 26 22:53:32 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177487,1177531,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355
+/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163081,1163490,1163768,1164255,1164301,1164339,1166402,1167383,1167662,1170085,1170379,1170459,1170996,1171136,1171297,1171379,1171611,1172916,1173402,1173468,1175113,1176178,1176550,1176719,1176729,1176733,1177487,1177531,1177859,1177864,1177905,1179169,1179856,1179861,1180757,1183081,1183098,1183175,1183554,1186508,1187140,1189028,1189355,1189360
/hadoop/core/branches/branch-0.19/hdfs/src/java:713112
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-1052/src/java:987665-1095512
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1189520&r1=1189519&r2=1189520&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Wed Oct 26 22:53:32 2011
@@ -152,17 +152,23 @@ public class DatanodeWebHdfsMethods {
{
final Configuration conf = new Configuration(datanode.getConf());
final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
- final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
final int b = bufferSize.getValue(conf);
- final FSDataOutputStream out = new FSDataOutputStream(dfsclient.create(
- fullpath, permission.getFsPermission(),
- overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
- : EnumSet.of(CreateFlag.CREATE),
- replication.getValue(conf), blockSize.getValue(conf), null, b), null);
+ DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
+ FSDataOutputStream out = null;
try {
+ out = new FSDataOutputStream(dfsclient.create(
+ fullpath, permission.getFsPermission(),
+ overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
+ : EnumSet.of(CreateFlag.CREATE),
+ replication.getValue(conf), blockSize.getValue(conf), null, b), null);
IOUtils.copyBytes(in, out, b);
- } finally {
out.close();
+ out = null;
+ dfsclient.close();
+ dfsclient = null;
+ } finally {
+ IOUtils.cleanup(LOG, out);
+ IOUtils.cleanup(LOG, dfsclient);
}
final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
@@ -227,13 +233,19 @@ public class DatanodeWebHdfsMethods {
{
final Configuration conf = new Configuration(datanode.getConf());
final InetSocketAddress nnRpcAddr = NameNode.getAddress(conf);
- final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
final int b = bufferSize.getValue(conf);
- final FSDataOutputStream out = dfsclient.append(fullpath, b, null, null);
+ DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
+ FSDataOutputStream out = null;
try {
+ out = dfsclient.append(fullpath, b, null, null);
IOUtils.copyBytes(in, out, b);
- } finally {
out.close();
+ out = null;
+ dfsclient.close();
+ dfsclient = null;
+ } finally {
+ IOUtils.cleanup(LOG, out);
+ IOUtils.cleanup(LOG, dfsclient);
}
return Response.ok().type(MediaType.APPLICATION_JSON).build();
}
@@ -301,18 +313,36 @@ public class DatanodeWebHdfsMethods {
case OPEN:
{
final int b = bufferSize.getValue(conf);
- final DFSDataInputStream in = new DFSClient.DFSDataInputStream(
- dfsclient.open(fullpath, b, true));
- in.seek(offset.getValue());
-
+ DFSDataInputStream in = null;
+ try {
+ in = new DFSClient.DFSDataInputStream(
+ dfsclient.open(fullpath, b, true));
+ in.seek(offset.getValue());
+ } catch(IOException ioe) {
+ IOUtils.cleanup(LOG, in);
+ IOUtils.cleanup(LOG, dfsclient);
+ throw ioe;
+ }
+ final DFSDataInputStream dis = in;
final StreamingOutput streaming = new StreamingOutput() {
@Override
public void write(final OutputStream out) throws IOException {
final Long n = length.getValue();
- if (n == null) {
- IOUtils.copyBytes(in, out, b);
- } else {
- IOUtils.copyBytes(in, out, n, false);
+ DFSDataInputStream dfsin = dis;
+ DFSClient client = dfsclient;
+ try {
+ if (n == null) {
+ IOUtils.copyBytes(dfsin, out, b);
+ } else {
+ IOUtils.copyBytes(dfsin, out, n, false);
+ }
+ dfsin.close();
+ dfsin = null;
+ dfsclient.close();
+ client = null;
+ } finally {
+ IOUtils.cleanup(LOG, dfsin);
+ IOUtils.cleanup(LOG, client);
}
}
};
@@ -324,7 +354,15 @@ public class DatanodeWebHdfsMethods {
}
case GETFILECHECKSUM:
{
- final MD5MD5CRC32FileChecksum checksum = dfsclient.getFileChecksum(fullpath);
+ MD5MD5CRC32FileChecksum checksum = null;
+ DFSClient client = dfsclient;
+ try {
+ checksum = client.getFileChecksum(fullpath);
+ client.close();
+ client = null;
+ } finally {
+ IOUtils.cleanup(LOG, client);
+ }
final String js = JsonUtil.toJsonString(checksum);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}