You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by br...@apache.org on 2014/03/24 22:05:50 UTC
svn commit: r1581064 - in
/hadoop/common/branches/branch-2.4/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs/
Author: brandonli
Date: Mon Mar 24 21:05:49 2014
New Revision: 1581064
URL: http://svn.apache.org/r1581064
Log:
HDFS-6050. Merging change r1581059 from branch-2
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java Mon Mar 24 21:05:49 2014
@@ -131,7 +131,7 @@ public class RpcProgramMountd extends Rp
handle = new FileHandle(exFileStatus.getFileId());
} catch (IOException e) {
- LOG.error("Can't get handle for export:" + path + ", exception:" + e);
+ LOG.error("Can't get handle for export:" + path, e);
MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_NOENT, out, xid, null);
return out;
}
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/AsyncDataService.java Mon Mar 24 21:05:49 2014
@@ -135,8 +135,7 @@ public class AsyncDataService {
try {
openFileCtx.executeWriteBack();
} catch (Throwable t) {
- LOG.error("Asyn data service got error:"
- + ExceptionUtils.getFullStackTrace(t));
+ LOG.error("Asyn data service got error:", t);
}
}
}
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java Mon Mar 24 21:05:49 2014
@@ -94,7 +94,8 @@ public class Nfs3Utils {
}
public static WccAttr getWccAttr(Nfs3FileAttributes attr) {
- return new WccAttr(attr.getSize(), attr.getMtime(), attr.getCtime());
+ return attr == null ? new WccAttr() : new WccAttr(attr.getSize(),
+ attr.getMtime(), attr.getCtime());
}
// TODO: maybe not efficient
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java Mon Mar 24 21:05:49 2014
@@ -365,7 +365,7 @@ class OpenFileCtx {
}
} catch (Throwable t) {
LOG.info("Dumper get Throwable: " + t + ". dumpFilePath: "
- + OpenFileCtx.this.dumpFilePath);
+ + OpenFileCtx.this.dumpFilePath, t);
}
}
}
@@ -660,8 +660,8 @@ class OpenFileCtx {
// Sync file data and length to avoid partial read failure
fos.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
} catch (ClosedChannelException closedException) {
- LOG.info("The FSDataOutputStream has been closed. " +
- "Continue processing the perfect overwrite.");
+ LOG.info("The FSDataOutputStream has been closed. "
+ + "Continue processing the perfect overwrite.");
} catch (IOException e) {
LOG.info("hsync failed when processing possible perfect overwrite, path="
+ path + " error:" + e);
@@ -680,7 +680,7 @@ class OpenFileCtx {
}
} catch (IOException e) {
LOG.info("Read failed when processing possible perfect overwrite, path="
- + path + " error:" + e);
+ + path, e);
return new WRITE3Response(Nfs3Status.NFS3ERR_IO, wccData, 0, stableHow,
Nfs3Constant.WRITE_COMMIT_VERF);
} finally {
@@ -914,7 +914,7 @@ class OpenFileCtx {
/** Invoked by AsynDataService to write back to HDFS */
void executeWriteBack() {
Preconditions.checkState(asyncStatus,
- "openFileCtx has false asyncStatus, fileId:" + latestAttr.getFileid());
+ "openFileCtx has false asyncStatus, fileId:" + latestAttr.getFileId());
final long startOffset = asyncWriteBackStartOffset;
try {
while (activeState) {
@@ -940,7 +940,7 @@ class OpenFileCtx {
asyncStatus = false;
} else {
LOG.info("Another asyn task is already started before this one"
- + " is finalized. fileId:" + latestAttr.getFileid()
+ + " is finalized. fileId:" + latestAttr.getFileId()
+ " asyncStatus:" + asyncStatus + " original startOffset:"
+ startOffset + " new startOffset:" + asyncWriteBackStartOffset
+ ". Won't change asyncStatus here.");
@@ -969,11 +969,11 @@ class OpenFileCtx {
} catch (ClosedChannelException cce) {
if (!pendingWrites.isEmpty()) {
LOG.error("Can't sync for fileId: " + latestAttr.getFileId()
- + ". Channel closed with writes pending");
+ + ". Channel closed with writes pending.", cce);
}
status = Nfs3Status.NFS3ERR_IO;
} catch (IOException e) {
- LOG.error("Got stream error during data sync:" + e);
+ LOG.error("Got stream error during data sync:", e);
// Do nothing. Stream will be closed eventually by StreamMonitor.
status = Nfs3Status.NFS3ERR_IO;
}
@@ -983,7 +983,7 @@ class OpenFileCtx {
latestAttr = Nfs3Utils.getFileAttr(client,
Nfs3Utils.getFileIdPath(latestAttr.getFileId()), iug);
} catch (IOException e) {
- LOG.error("Can't get new file attr for fileId: " + latestAttr.getFileId());
+ LOG.error("Can't get new file attr, fileId: " + latestAttr.getFileId(), e);
status = Nfs3Status.NFS3ERR_IO;
}
@@ -1006,7 +1006,7 @@ class OpenFileCtx {
new VerifierNone()), commit.getXid());
if (LOG.isDebugEnabled()) {
- LOG.debug("FileId: " + latestAttr.getFileid() + " Service time:"
+ LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:"
+ (System.currentTimeMillis() - commit.getStartTime())
+ "ms. Sent response for commit:" + commit);
}
@@ -1069,7 +1069,7 @@ class OpenFileCtx {
fos.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
}
} catch (IOException e) {
- LOG.error("hsync failed with writeCtx:" + writeCtx + " error:" + e);
+ LOG.error("hsync failed with writeCtx:" + writeCtx, e);
throw e;
}
}
@@ -1101,7 +1101,7 @@ class OpenFileCtx {
}
LOG.info("Clean up open file context for fileId: "
- + latestAttr.getFileid());
+ + latestAttr.getFileId());
cleanup();
}
}
@@ -1128,7 +1128,7 @@ class OpenFileCtx {
fos.close();
}
} catch (IOException e) {
- LOG.info("Can't close stream for fileId:" + latestAttr.getFileid()
+ LOG.info("Can't close stream for fileId:" + latestAttr.getFileId()
+ ", error:" + e);
}
@@ -1156,7 +1156,7 @@ class OpenFileCtx {
try {
dumpOut.close();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to close outputstream of dump file" + dumpFilePath, e);
}
File dumpFile = new File(dumpFilePath);
if (dumpFile.exists() && !dumpFile.delete()) {
@@ -1167,7 +1167,7 @@ class OpenFileCtx {
try {
raf.close();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Got exception when closing input stream of dump file.", e);
}
}
}
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java Mon Mar 24 21:05:49 2014
@@ -268,7 +268,7 @@ public class RpcProgramNfs3 extends RpcP
try {
attrs = writeManager.getFileAttr(dfsClient, handle, iug);
} catch (IOException e) {
- LOG.info("Can't get file attribute, fileId=" + handle.getFileId());
+ LOG.info("Can't get file attribute, fileId=" + handle.getFileId(), e);
response.setStatus(Nfs3Status.NFS3ERR_IO);
return response;
}
@@ -381,7 +381,7 @@ public class RpcProgramNfs3 extends RpcP
wccData = Nfs3Utils.createWccData(Nfs3Utils.getWccAttr(preOpAttr),
dfsClient, fileIdPath, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpAttr for fileIdPath: " + fileIdPath);
+ LOG.info("Can't get postOpAttr for fileIdPath: " + fileIdPath, e1);
}
if (e instanceof AccessControlException) {
return new SETATTR3Response(Nfs3Status.NFS3ERR_ACCES, wccData);
@@ -614,13 +614,11 @@ public class RpcProgramNfs3 extends RpcP
// Only do access check.
try {
// Don't read from cache. Client may not have read permission.
- attrs = Nfs3Utils.getFileAttr(
- dfsClient,
- Nfs3Utils.getFileIdPath(handle),
- iug);
+ attrs = Nfs3Utils.getFileAttr(dfsClient,
+ Nfs3Utils.getFileIdPath(handle), iug);
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Get error accessing file, fileId:" + handle.getFileId());
+ LOG.debug("Get error accessing file, fileId:" + handle.getFileId(), e);
}
return new READ3Response(Nfs3Status.NFS3ERR_IO);
}
@@ -764,7 +762,7 @@ public class RpcProgramNfs3 extends RpcP
try {
postOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpAttr for fileId: " + handle.getFileId());
+ LOG.info("Can't get postOpAttr for fileId: " + handle.getFileId(), e1);
}
WccAttr attr = preOpAttr == null ? null : Nfs3Utils.getWccAttr(preOpAttr);
WccData fileWcc = new WccData(attr, postOpAttr);
@@ -881,7 +879,7 @@ public class RpcProgramNfs3 extends RpcP
fos.close();
} catch (IOException e1) {
LOG.error("Can't close stream for dirFileId:" + dirHandle.getFileId()
- + " filename: " + fileName);
+ + " filename: " + fileName, e1);
}
}
if (dirWcc == null) {
@@ -890,7 +888,7 @@ public class RpcProgramNfs3 extends RpcP
dfsClient, dirFileIdPath, iug);
} catch (IOException e1) {
LOG.error("Can't get postOpDirAttr for dirFileId:"
- + dirHandle.getFileId());
+ + dirHandle.getFileId(), e1);
}
}
if (e instanceof AccessControlException) {
@@ -982,7 +980,7 @@ public class RpcProgramNfs3 extends RpcP
try {
postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpDirAttr for " + dirFileIdPath);
+ LOG.info("Can't get postOpDirAttr for " + dirFileIdPath, e);
}
}
WccData dirWcc = new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
@@ -1064,7 +1062,7 @@ public class RpcProgramNfs3 extends RpcP
try {
postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpDirAttr for " + dirFileIdPath);
+ LOG.info("Can't get postOpDirAttr for " + dirFileIdPath, e1);
}
}
WccData dirWcc = new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
@@ -1146,7 +1144,7 @@ public class RpcProgramNfs3 extends RpcP
try {
postOpDirAttr = Nfs3Utils.getFileAttr(dfsClient, dirFileIdPath, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpDirAttr for " + dirFileIdPath);
+ LOG.info("Can't get postOpDirAttr for " + dirFileIdPath, e1);
}
}
WccData dirWcc = new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
@@ -1236,7 +1234,7 @@ public class RpcProgramNfs3 extends RpcP
dfsClient, toDirFileIdPath, iug);
} catch (IOException e1) {
LOG.info("Can't get postOpDirAttr for " + fromDirFileIdPath + " or"
- + toDirFileIdPath);
+ + toDirFileIdPath, e1);
}
if (e instanceof AccessControlException) {
return new RENAME3Response(Nfs3Status.NFS3ERR_PERM, fromDirWcc,
@@ -1298,7 +1296,7 @@ public class RpcProgramNfs3 extends RpcP
.setPostOpAttr(Nfs3Utils.getFileAttr(dfsClient, linkDirIdPath, iug));
return new SYMLINK3Response(Nfs3Status.NFS3_OK, new FileHandle(
- objAttr.getFileid()), objAttr, dirWcc);
+ objAttr.getFileId()), objAttr, dirWcc);
} catch (IOException e) {
LOG.warn("Exception:" + e);
@@ -1585,7 +1583,7 @@ public class RpcProgramNfs3 extends RpcP
entries[0] = new READDIRPLUS3Response.EntryPlus3(
postOpDirAttr.getFileId(), ".", 0, postOpDirAttr, new FileHandle(
- postOpDirAttr.getFileid()));
+ postOpDirAttr.getFileId()));
entries[1] = new READDIRPLUS3Response.EntryPlus3(dotdotFileId, "..",
dotdotFileId, postOpDirAttr, new FileHandle(dotdotFileId));
@@ -1596,8 +1594,7 @@ public class RpcProgramNfs3 extends RpcP
try {
attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
} catch (IOException e) {
- LOG.error("Can't get file attributes for fileId:" + fileId
- + " error:" + e);
+ LOG.error("Can't get file attributes for fileId:" + fileId, e);
continue;
}
entries[i] = new READDIRPLUS3Response.EntryPlus3(fileId,
@@ -1614,8 +1611,7 @@ public class RpcProgramNfs3 extends RpcP
try {
attr = writeManager.getFileAttr(dfsClient, childHandle, iug);
} catch (IOException e) {
- LOG.error("Can't get file attributes for fileId:" + fileId
- + " error:" + e);
+ LOG.error("Can't get file attributes for fileId:" + fileId, e);
continue;
}
entries[i] = new READDIRPLUS3Response.EntryPlus3(fileId,
@@ -1842,10 +1838,9 @@ public class RpcProgramNfs3 extends RpcP
try {
postOpAttr = writeManager.getFileAttr(dfsClient, handle, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpAttr for fileId: " + handle.getFileId());
+ LOG.info("Can't get postOpAttr for fileId: " + handle.getFileId(), e1);
}
- WccData fileWcc = new WccData(preOpAttr == null ? null
- : Nfs3Utils.getWccAttr(preOpAttr), postOpAttr);
+ WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr), postOpAttr);
return new COMMIT3Response(Nfs3Status.NFS3ERR_IO, fileWcc,
Nfs3Constant.WRITE_COMMIT_VERF);
}
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java Mon Mar 24 21:05:49 2014
@@ -156,7 +156,7 @@ public class WriteManager {
}
throw e;
} catch (IOException e) {
- LOG.error("Can't apapend to file:" + fileIdPath + ", error:" + e);
+ LOG.error("Can't apapend to file:" + fileIdPath, e);
if (fos != null) {
fos.close();
}
@@ -181,7 +181,7 @@ public class WriteManager {
try {
fos.close();
} catch (IOException e) {
- LOG.error("Can't close stream for fileId:" + handle.getFileId());
+ LOG.error("Can't close stream for fileId:" + handle.getFileId(), e);
}
// Notify client to retry
WccData fileWcc = new WccData(latestAttr.getWccAttr(), latestAttr);
@@ -284,10 +284,10 @@ public class WriteManager {
// Send out the response
Nfs3FileAttributes postOpAttr = null;
try {
- String fileIdPath = Nfs3Utils.getFileIdPath(preOpAttr.getFileid());
+ String fileIdPath = Nfs3Utils.getFileIdPath(preOpAttr.getFileId());
postOpAttr = Nfs3Utils.getFileAttr(dfsClient, fileIdPath, iug);
} catch (IOException e1) {
- LOG.info("Can't get postOpAttr for fileId: " + preOpAttr.getFileid());
+ LOG.info("Can't get postOpAttr for fileId: " + preOpAttr.getFileId(), e1);
}
WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr), postOpAttr);
COMMIT3Response response = new COMMIT3Response(status, fileWcc,
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1581064&r1=1581063&r2=1581064&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar 24 21:05:49 2014
@@ -168,6 +168,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6120. Fix and improve safe mode log messages. (Arpit Agarwal)
+ HDFS-6050. NFS does not handle exceptions correctly in a few places
+ (brandonli)
+
OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery