You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ac...@apache.org on 2012/11/28 16:17:35 UTC
svn commit: r1414750 - in
/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/server/datanode/
src/main/java/org/apache/hadoop/hdfs/server/namenod...
Author: acmurthy
Date: Wed Nov 28 15:17:21 2012
New Revision: 1414750
URL: http://svn.apache.org/viewvc?rev=1414750&view=rev
Log:
Rebased on trunk.
Modified:
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1411704-1414746
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Nov 28 15:17:21 2012
@@ -164,6 +164,15 @@ Trunk (Unreleased)
HDFS-4206. Change the fields in INode and its subclasses to private.
(szetszwo)
+ HDFS-4215. Remove locking from addToParent(..) since it is used in image
+ loading, and add INode.isFile(). (szetszwo)
+
+ HDFS-4200. Reduce the size of synchronized sections in PacketResponder.
+ (suresh)
+
+ HDFS-4209. Clean up the addNode/addChild/addChildNoQuotaCheck methods in
+ FSDirectory and INodeDirectory. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
@@ -475,6 +484,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4038. Override toString() for BookKeeperEditLogInputStream.
(Vinay via umamahesh)
+ HDFS-4214. OfflineEditsViewer should print out the offset at which it
+ encountered an error. (Colin Patrick McCabe via atm)
+
OPTIMIZATIONS
BUG FIXES
@@ -621,6 +633,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4179. BackupNode: allow reads, fix checkpointing, safeMode. (shv)
+ HDFS-4216. Do not ignore QuotaExceededException when adding symlinks.
+ (szetszwo)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1411704-1414746
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Wed Nov 28 15:17:21 2012
@@ -233,7 +233,7 @@ public class DFSUtil {
/**
* Given a list of path components returns a path as a UTF8 String
*/
- public static String byteArray2String(byte[][] pathComponents) {
+ public static String byteArray2PathString(byte[][] pathComponents) {
if (pathComponents.length == 0)
return "";
if (pathComponents.length == 1 && pathComponents[0].length == 0) {
@@ -254,6 +254,14 @@ public class DFSUtil {
return null;
}
+ /** Convert an object representing a path to a string. */
+ public static String path2String(final Object path) {
+ return path == null? null
+ : path instanceof String? (String)path
+ : path instanceof byte[][]? byteArray2PathString((byte[][])path)
+ : path.toString();
+ }
+
/**
* Splits the array of bytes into array of arrays of bytes
* on byte separator
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Wed Nov 28 15:17:21 2012
@@ -82,12 +82,8 @@ import org.xml.sax.helpers.XMLReaderFact
@InterfaceStability.Evolving
public class HftpFileSystem extends FileSystem
implements DelegationTokenRenewer.Renewable {
- private static final DelegationTokenRenewer<HftpFileSystem> dtRenewer
- = new DelegationTokenRenewer<HftpFileSystem>(HftpFileSystem.class);
-
static {
HttpURLConnection.setFollowRedirects(true);
- dtRenewer.start();
}
public static final Text TOKEN_KIND = new Text("HFTP delegation");
@@ -106,6 +102,16 @@ public class HftpFileSystem extends File
private static final HftpDelegationTokenSelector hftpTokenSelector =
new HftpDelegationTokenSelector();
+ private DelegationTokenRenewer dtRenewer = null;
+
+ private synchronized void addRenewAction(final HftpFileSystem hftpFs) {
+ if (dtRenewer == null) {
+ dtRenewer = DelegationTokenRenewer.getInstance();
+ }
+
+ dtRenewer.addRenewAction(hftpFs);
+ }
+
public static final SimpleDateFormat getDateFormat() {
final SimpleDateFormat df = new SimpleDateFormat(HFTP_DATE_FORMAT);
df.setTimeZone(TimeZone.getTimeZone(HFTP_TIMEZONE));
@@ -202,7 +208,7 @@ public class HftpFileSystem extends File
if (token != null) {
setDelegationToken(token);
if (createdToken) {
- dtRenewer.addRenewAction(this);
+ addRenewAction(this);
LOG.debug("Created new DT for " + token.getService());
} else {
LOG.debug("Found existing DT for " + token.getService());
@@ -395,6 +401,14 @@ public class HftpFileSystem extends File
return new FSDataInputStream(new RangeHeaderInputStream(u));
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (dtRenewer != null) {
+ dtRenewer.removeRenewAction(this); // blocks
+ }
+ }
+
/** Class to parse and store a listing reply from the server. */
class LsParser extends DefaultHandler {
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Wed Nov 28 15:17:21 2012
@@ -638,10 +638,7 @@ class BlockReceiver implements Closeable
responder.start(); // start thread to processes responses
}
- /*
- * Receive until the last packet.
- */
- while (receivePacket() >= 0) {}
+ while (receivePacket() >= 0) { /* Receive until the last packet */ }
// wait for all outstanding packet responses. And then
// indicate responder to gracefully shutdown.
@@ -724,7 +721,7 @@ class BlockReceiver implements Closeable
static private long checksum2long(byte[] checksum) {
long crc = 0L;
for(int i=0; i<checksum.length; i++) {
- crc |= (0xffL&(long)checksum[i])<<((checksum.length-i-1)*8);
+ crc |= (0xffL&checksum[i])<<((checksum.length-i-1)*8);
}
return crc;
}
@@ -783,24 +780,23 @@ class BlockReceiver implements Closeable
NON_PIPELINE, LAST_IN_PIPELINE, HAS_DOWNSTREAM_IN_PIPELINE
}
+ private static Status[] MIRROR_ERROR_STATUS = {Status.SUCCESS, Status.ERROR};
+
/**
* Processed responses from downstream datanodes in the pipeline
* and sends back replies to the originator.
*/
class PacketResponder implements Runnable, Closeable {
-
- /** queue for packets waiting for ack */
+ /** queue for packets waiting for ack - synchronization using monitor lock */
private final LinkedList<Packet> ackQueue = new LinkedList<Packet>();
/** the thread that spawns this responder */
private final Thread receiverThread = Thread.currentThread();
- /** is this responder running? */
+ /** is this responder running? - synchronization using monitor lock */
private volatile boolean running = true;
-
/** input from the next downstream datanode */
private final DataInputStream downstreamIn;
/** output to upstream datanode/client */
private final DataOutputStream upstreamOut;
-
/** The type of this responder */
private final PacketResponderType type;
/** for log and error messages */
@@ -812,8 +808,7 @@ class BlockReceiver implements Closeable
}
PacketResponder(final DataOutputStream upstreamOut,
- final DataInputStream downstreamIn,
- final DatanodeInfo[] downstreams) {
+ final DataInputStream downstreamIn, final DatanodeInfo[] downstreams) {
this.downstreamIn = downstreamIn;
this.upstreamOut = upstreamOut;
@@ -830,31 +825,49 @@ class BlockReceiver implements Closeable
this.myString = b.toString();
}
+ private boolean isRunning() {
+ return running && datanode.shouldRun;
+ }
+
/**
* enqueue the seqno that is still be to acked by the downstream datanode.
* @param seqno
* @param lastPacketInBlock
* @param offsetInBlock
*/
- synchronized void enqueue(final long seqno,
- final boolean lastPacketInBlock, final long offsetInBlock) {
- if (running) {
- final Packet p = new Packet(seqno, lastPacketInBlock, offsetInBlock,
- System.nanoTime());
- if(LOG.isDebugEnabled()) {
- LOG.debug(myString + ": enqueue " + p);
+ void enqueue(final long seqno, final boolean lastPacketInBlock,
+ final long offsetInBlock) {
+ final Packet p = new Packet(seqno, lastPacketInBlock, offsetInBlock,
+ System.nanoTime());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(myString + ": enqueue " + p);
+ }
+ synchronized(this) {
+ if (running) {
+ ackQueue.addLast(p);
+ notifyAll();
}
- ackQueue.addLast(p);
- notifyAll();
}
}
+
+ /** Wait for a packet with given {@code seqno} to be enqueued to ackQueue */
+ synchronized Packet waitForAckHead(long seqno) throws InterruptedException {
+ while (isRunning() && ackQueue.size() == 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(myString + ": seqno=" + seqno +
+ " waiting for local datanode to finish write.");
+ }
+ wait();
+ }
+ return isRunning() ? ackQueue.getFirst() : null;
+ }
/**
* wait for all pending packets to be acked. Then shutdown thread.
*/
@Override
public synchronized void close() {
- while (running && ackQueue.size() != 0 && datanode.shouldRun) {
+ while (isRunning() && ackQueue.size() != 0) {
try {
wait();
} catch (InterruptedException e) {
@@ -877,147 +890,97 @@ class BlockReceiver implements Closeable
public void run() {
boolean lastPacketInBlock = false;
final long startTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0;
- while (running && datanode.shouldRun && !lastPacketInBlock) {
-
+ while (isRunning() && !lastPacketInBlock) {
long totalAckTimeNanos = 0;
boolean isInterrupted = false;
try {
- Packet pkt = null;
- long expected = -2;
- PipelineAck ack = new PipelineAck();
- long seqno = PipelineAck.UNKOWN_SEQNO;
- long ackRecvNanoTime = 0;
- try {
- if (type != PacketResponderType.LAST_IN_PIPELINE
- && !mirrorError) {
- // read an ack from downstream datanode
- ack.readFields(downstreamIn);
- ackRecvNanoTime = System.nanoTime();
- if (LOG.isDebugEnabled()) {
- LOG.debug(myString + " got " + ack);
- }
- seqno = ack.getSeqno();
+ Packet pkt = null;
+ long expected = -2;
+ PipelineAck ack = new PipelineAck();
+ long seqno = PipelineAck.UNKOWN_SEQNO;
+ long ackRecvNanoTime = 0;
+ try {
+ if (type != PacketResponderType.LAST_IN_PIPELINE && !mirrorError) {
+ // read an ack from downstream datanode
+ ack.readFields(downstreamIn);
+ ackRecvNanoTime = System.nanoTime();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(myString + " got " + ack);
}
- if (seqno != PipelineAck.UNKOWN_SEQNO
- || type == PacketResponderType.LAST_IN_PIPELINE) {
- synchronized (this) {
- while (running && datanode.shouldRun && ackQueue.size() == 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(myString + ": seqno=" + seqno +
- " waiting for local datanode to finish write.");
- }
- wait();
- }
- if (!running || !datanode.shouldRun) {
- break;
- }
- pkt = ackQueue.getFirst();
- expected = pkt.seqno;
- if (type == PacketResponderType.HAS_DOWNSTREAM_IN_PIPELINE
- && seqno != expected) {
- throw new IOException(myString + "seqno: expected="
- + expected + ", received=" + seqno);
- }
- if (type == PacketResponderType.HAS_DOWNSTREAM_IN_PIPELINE) {
- // The total ack time includes the ack times of downstream nodes.
- // The value is 0 if this responder doesn't have a downstream
- // DN in the pipeline.
- totalAckTimeNanos = ackRecvNanoTime - pkt.ackEnqueueNanoTime;
- // Report the elapsed time from ack send to ack receive minus
- // the downstream ack time.
- long ackTimeNanos = totalAckTimeNanos - ack.getDownstreamAckTimeNanos();
- if (ackTimeNanos < 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Calculated invalid ack time: " + ackTimeNanos + "ns.");
- }
- } else {
- datanode.metrics.addPacketAckRoundTripTimeNanos(ackTimeNanos);
- }
+ seqno = ack.getSeqno();
+ }
+ if (seqno != PipelineAck.UNKOWN_SEQNO
+ || type == PacketResponderType.LAST_IN_PIPELINE) {
+ pkt = waitForAckHead(seqno);
+ if (!isRunning()) {
+ break;
+ }
+ expected = pkt.seqno;
+ if (type == PacketResponderType.HAS_DOWNSTREAM_IN_PIPELINE
+ && seqno != expected) {
+ throw new IOException(myString + "seqno: expected=" + expected
+ + ", received=" + seqno);
+ }
+ if (type == PacketResponderType.HAS_DOWNSTREAM_IN_PIPELINE) {
+ // The total ack time includes the ack times of downstream
+ // nodes.
+ // The value is 0 if this responder doesn't have a downstream
+ // DN in the pipeline.
+ totalAckTimeNanos = ackRecvNanoTime - pkt.ackEnqueueNanoTime;
+ // Report the elapsed time from ack send to ack receive minus
+ // the downstream ack time.
+ long ackTimeNanos = totalAckTimeNanos
+ - ack.getDownstreamAckTimeNanos();
+ if (ackTimeNanos < 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Calculated invalid ack time: " + ackTimeNanos
+ + "ns.");
}
- lastPacketInBlock = pkt.lastPacketInBlock;
+ } else {
+ datanode.metrics.addPacketAckRoundTripTimeNanos(ackTimeNanos);
}
}
- } catch (InterruptedException ine) {
+ lastPacketInBlock = pkt.lastPacketInBlock;
+ }
+ } catch (InterruptedException ine) {
+ isInterrupted = true;
+ } catch (IOException ioe) {
+ if (Thread.interrupted()) {
isInterrupted = true;
- } catch (IOException ioe) {
- if (Thread.interrupted()) {
- isInterrupted = true;
- } else {
- // continue to run even if can not read from mirror
- // notify client of the error
- // and wait for the client to shut down the pipeline
- mirrorError = true;
- LOG.info(myString, ioe);
- }
+ } else {
+ // continue to run even if can not read from mirror
+ // notify client of the error
+ // and wait for the client to shut down the pipeline
+ mirrorError = true;
+ LOG.info(myString, ioe);
}
+ }
- if (Thread.interrupted() || isInterrupted) {
- /* The receiver thread cancelled this thread.
- * We could also check any other status updates from the
- * receiver thread (e.g. if it is ok to write to replyOut).
- * It is prudent to not send any more status back to the client
- * because this datanode has a problem. The upstream datanode
- * will detect that this datanode is bad, and rightly so.
- */
- LOG.info(myString + ": Thread is interrupted.");
- running = false;
- continue;
- }
-
- // If this is the last packet in block, then close block
- // file and finalize the block before responding success
- if (lastPacketInBlock) {
- BlockReceiver.this.close();
- final long endTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime() : 0;
- block.setNumBytes(replicaInfo.getNumBytes());
- datanode.data.finalizeBlock(block);
- datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT);
- if (ClientTraceLog.isInfoEnabled() && isClient) {
- long offset = 0;
- DatanodeRegistration dnR =
- datanode.getDNRegistrationForBP(block.getBlockPoolId());
- ClientTraceLog.info(String.format(DN_CLIENTTRACE_FORMAT,
- inAddr, myAddr, block.getNumBytes(),
- "HDFS_WRITE", clientname, offset,
- dnR.getStorageID(), block, endTime-startTime));
- } else {
- LOG.info("Received " + block + " size "
- + block.getNumBytes() + " from " + inAddr);
- }
- }
+ if (Thread.interrupted() || isInterrupted) {
+ /*
+ * The receiver thread cancelled this thread. We could also check
+ * any other status updates from the receiver thread (e.g. if it is
+ * ok to write to replyOut). It is prudent to not send any more
+ * status back to the client because this datanode has a problem.
+ * The upstream datanode will detect that this datanode is bad, and
+ * rightly so.
+ */
+ LOG.info(myString + ": Thread is interrupted.");
+ running = false;
+ continue;
+ }
- // construct my ack message
- Status[] replies = null;
- if (mirrorError) { // ack read error
- replies = new Status[2];
- replies[0] = Status.SUCCESS;
- replies[1] = Status.ERROR;
- } else {
- short ackLen = type == PacketResponderType.LAST_IN_PIPELINE? 0
- : ack.getNumOfReplies();
- replies = new Status[1+ackLen];
- replies[0] = Status.SUCCESS;
- for (int i=0; i<ackLen; i++) {
- replies[i+1] = ack.getReply(i);
- }
- }
- PipelineAck replyAck = new PipelineAck(expected, replies, totalAckTimeNanos);
-
- if (replyAck.isSuccess() &&
- pkt.offsetInBlock > replicaInfo.getBytesAcked())
- replicaInfo.setBytesAcked(pkt.offsetInBlock);
-
- // send my ack back to upstream datanode
- replyAck.write(upstreamOut);
- upstreamOut.flush();
- if (LOG.isDebugEnabled()) {
- LOG.debug(myString + ", replyAck=" + replyAck);
- }
- if (pkt != null) {
- // remove the packet from the ack queue
- removeAckHead();
- // update bytes acked
- }
+ if (lastPacketInBlock) {
+ // Finalize the block and close the block file
+ finalizeBlock(startTime);
+ }
+
+ sendAckUpstream(ack, expected, totalAckTimeNanos,
+ (pkt != null ? pkt.offsetInBlock : 0));
+ if (pkt != null) {
+ // remove the packet from the ack queue
+ removeAckHead();
+ }
} catch (IOException e) {
LOG.warn("IOException in BlockReceiver.run(): ", e);
if (running) {
@@ -1044,6 +1007,66 @@ class BlockReceiver implements Closeable
}
/**
+ * Finalize the block and close the block file
+ * @param startTime time when BlockReceiver started receiving the block
+ */
+ private void finalizeBlock(long startTime) throws IOException {
+ BlockReceiver.this.close();
+ final long endTime = ClientTraceLog.isInfoEnabled() ? System.nanoTime()
+ : 0;
+ block.setNumBytes(replicaInfo.getNumBytes());
+ datanode.data.finalizeBlock(block);
+ datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT);
+ if (ClientTraceLog.isInfoEnabled() && isClient) {
+ long offset = 0;
+ DatanodeRegistration dnR = datanode.getDNRegistrationForBP(block
+ .getBlockPoolId());
+ ClientTraceLog.info(String.format(DN_CLIENTTRACE_FORMAT, inAddr,
+ myAddr, block.getNumBytes(), "HDFS_WRITE", clientname, offset,
+ dnR.getStorageID(), block, endTime - startTime));
+ } else {
+ LOG.info("Received " + block + " size " + block.getNumBytes()
+ + " from " + inAddr);
+ }
+ }
+
+ /**
+ * @param ack Ack received from downstream
+ * @param seqno sequence number of ack to be sent upstream
+ * @param totalAckTimeNanos total ack time including all the downstream
+ * nodes
+ * @param offsetInBlock offset in block for the data in packet
+ */
+ private void sendAckUpstream(PipelineAck ack, long seqno,
+ long totalAckTimeNanos, long offsetInBlock) throws IOException {
+ Status[] replies = null;
+ if (mirrorError) { // ack read error
+ replies = MIRROR_ERROR_STATUS;
+ } else {
+ short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack
+ .getNumOfReplies();
+ replies = new Status[1 + ackLen];
+ replies[0] = Status.SUCCESS;
+ for (int i = 0; i < ackLen; i++) {
+ replies[i + 1] = ack.getReply(i);
+ }
+ }
+ PipelineAck replyAck = new PipelineAck(seqno, replies,
+ totalAckTimeNanos);
+ if (replyAck.isSuccess()
+ && offsetInBlock > replicaInfo.getBytesAcked()) {
+ replicaInfo.setBytesAcked(offsetInBlock);
+ }
+
+ // send my ack back to upstream datanode
+ replyAck.write(upstreamOut);
+ upstreamOut.flush();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(myString + ", replyAck=" + replyAck);
+ }
+ }
+
+ /**
* Remove a packet from the head of the ack queue
*
* This should be called only when the ack queue is not empty
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Wed Nov 28 15:17:21 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -74,12 +75,15 @@ import com.google.common.base.Preconditi
*
*************************************************/
public class FSDirectory implements Closeable {
+ private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) {
+ return new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
+ namesystem.createFsOwnerPermissions(new FsPermission((short)0755)));
+ }
INodeDirectoryWithQuota rootDir;
FSImage fsImage;
private final FSNamesystem namesystem;
private volatile boolean ready = false;
- private static final long UNKNOWN_DISK_SPACE = -1;
private final int maxComponentLength;
private final int maxDirItems;
private final int lsLimit; // max list limit
@@ -122,9 +126,7 @@ public class FSDirectory implements Clos
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
this.dirLock = new ReentrantReadWriteLock(true); // fair
this.cond = dirLock.writeLock().newCondition();
- rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
- ns.createFsOwnerPermissions(new FsPermission((short)0755)),
- Long.MAX_VALUE, UNKNOWN_DISK_SPACE);
+ rootDir = createRoot(ns);
this.fsImage = fsImage;
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
@@ -255,13 +257,14 @@ public class FSDirectory implements Clos
permissions,replication,
preferredBlockSize, modTime, clientName,
clientMachine, clientNode);
+ boolean added = false;
writeLock();
try {
- newNode = addNode(path, newNode, UNKNOWN_DISK_SPACE);
+ added = addINode(path, newNode);
} finally {
writeUnlock();
}
- if (newNode == null) {
+ if (!added) {
NameNode.stateChangeLog.info("DIR* addFile: failed to add " + path);
return null;
}
@@ -281,7 +284,7 @@ public class FSDirectory implements Clos
boolean underConstruction,
String clientName,
String clientMachine) {
- INode newNode;
+ final INode newNode;
assert hasWriteLock();
if (underConstruction) {
newNode = new INodeFileUnderConstruction(
@@ -294,45 +297,17 @@ public class FSDirectory implements Clos
}
try {
- newNode = addNode(path, newNode, UNKNOWN_DISK_SPACE);
+ if (addINode(path, newNode)) {
+ return newNode;
+ }
} catch (IOException e) {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
"DIR* FSDirectory.unprotectedAddFile: exception when add " + path
+ " to the file system", e);
}
- return null;
}
- return newNode;
- }
-
- INodeDirectory addToParent(INodeDirectory parentINode,
- INode newNode, boolean propagateModTime) {
- // NOTE: This does not update space counts for parents
- INodeDirectory newParent = null;
- writeLock();
- try {
- try {
- newParent = rootDir.addToParent(newNode, parentINode,
- propagateModTime);
- cacheName(newNode);
- } catch (FileNotFoundException e) {
- return null;
- }
- if(newParent == null)
- return null;
- if(!newNode.isDirectory() && !newNode.isSymlink()) {
- // Add file->block mapping
- INodeFile newF = (INodeFile)newNode;
- BlockInfo[] blocks = newF.getBlocks();
- for (int i = 0; i < blocks.length; i++) {
- newF.setBlock(i, getBlockManager().addBlockCollection(blocks[i], newF));
- }
- }
- } finally {
- writeUnlock();
- }
- return newParent;
+ return null;
}
/**
@@ -574,12 +549,12 @@ public class FSDirectory implements Clos
// Ensure dst has quota to accommodate rename
verifyQuotaForRename(srcInodes, dstInodes);
- INode dstChild = null;
+ boolean added = false;
INode srcChild = null;
String srcChildName = null;
try {
// remove src
- srcChild = removeChild(srcInodesInPath, srcInodes.length-1);
+ srcChild = removeLastINode(srcInodesInPath);
if (srcChild == null) {
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ "failed to rename " + src + " to " + dst
@@ -590,9 +565,8 @@ public class FSDirectory implements Clos
srcChild.setLocalName(dstComponents[dstInodes.length-1]);
// add src to the destination
- dstChild = addChildNoQuotaCheck(dstInodesInPath, dstInodes.length-1,
- srcChild, UNKNOWN_DISK_SPACE);
- if (dstChild != null) {
+ added = addLastINodeNoQuotaCheck(dstInodesInPath, srcChild);
+ if (added) {
srcChild = null;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedRenameTo: "
@@ -604,11 +578,10 @@ public class FSDirectory implements Clos
return true;
}
} finally {
- if (dstChild == null && srcChild != null) {
+ if (!added && srcChild != null) {
// put it back
srcChild.setLocalName(srcChildName);
- addChildNoQuotaCheck(srcInodesInPath, srcInodes.length - 1, srcChild,
- UNKNOWN_DISK_SPACE);
+ addLastINodeNoQuotaCheck(srcInodesInPath, srcChild);
}
}
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
@@ -728,7 +701,7 @@ public class FSDirectory implements Clos
// Ensure dst has quota to accommodate rename
verifyQuotaForRename(srcInodes, dstInodes);
- INode removedSrc = removeChild(srcInodesInPath, srcInodes.length - 1);
+ INode removedSrc = removeLastINode(srcInodesInPath);
if (removedSrc == null) {
error = "Failed to rename " + src + " to " + dst
+ " because the source can not be removed";
@@ -741,18 +714,13 @@ public class FSDirectory implements Clos
INode removedDst = null;
try {
if (dstInode != null) { // dst exists remove it
- removedDst = removeChild(dstInodesInPath, dstInodes.length - 1);
+ removedDst = removeLastINode(dstInodesInPath);
dstChildName = removedDst.getLocalName();
}
- INode dstChild = null;
removedSrc.setLocalName(dstComponents[dstInodes.length - 1]);
// add src as dst to complete rename
- dstChild = addChildNoQuotaCheck(dstInodesInPath, dstInodes.length - 1,
- removedSrc, UNKNOWN_DISK_SPACE);
-
- int filesDeleted = 0;
- if (dstChild != null) {
+ if (addLastINodeNoQuotaCheck(dstInodesInPath, removedSrc)) {
removedSrc = null;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
@@ -763,6 +731,7 @@ public class FSDirectory implements Clos
dstInodes[dstInodes.length - 2].setModificationTime(timestamp);
// Collect the blocks and remove the lease for previous dst
+ int filesDeleted = 0;
if (removedDst != null) {
INode rmdst = removedDst;
removedDst = null;
@@ -776,14 +745,12 @@ public class FSDirectory implements Clos
if (removedSrc != null) {
// Rename failed - restore src
removedSrc.setLocalName(srcChildName);
- addChildNoQuotaCheck(srcInodesInPath, srcInodes.length - 1, removedSrc,
- UNKNOWN_DISK_SPACE);
+ addLastINodeNoQuotaCheck(srcInodesInPath, removedSrc);
}
if (removedDst != null) {
// Rename failed - restore dst
removedDst.setLocalName(dstChildName);
- addChildNoQuotaCheck(dstInodesInPath, dstInodes.length - 1, removedDst,
- UNKNOWN_DISK_SPACE);
+ addLastINodeNoQuotaCheck(dstInodesInPath, removedDst);
}
}
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
@@ -825,11 +792,7 @@ public class FSDirectory implements Clos
final INodesInPath inodesInPath = rootDir.getExistingPathINodes(src, true);
final INode[] inodes = inodesInPath.getINodes();
INode inode = inodes[inodes.length - 1];
- if (inode == null) {
- return null;
- }
- assert !inode.isSymlink();
- if (inode.isDirectory()) {
+ if (inode == null || !inode.isFile()) {
return null;
}
INodeFile fileNode = (INodeFile)inode;
@@ -848,22 +811,15 @@ public class FSDirectory implements Clos
}
/**
- * Get the blocksize of a file
- * @param filename the filename
- * @return the number of bytes
+ * @param path the file path
+ * @return the block size of the file.
*/
- long getPreferredBlockSize(String filename) throws UnresolvedLinkException,
+ long getPreferredBlockSize(String path) throws UnresolvedLinkException,
FileNotFoundException, IOException {
readLock();
try {
- INode inode = rootDir.getNode(filename, false);
- if (inode == null) {
- throw new FileNotFoundException("File does not exist: " + filename);
- }
- if (inode.isDirectory() || inode.isSymlink()) {
- throw new IOException("Getting block size of non-file: "+ filename);
- }
- return ((INodeFile)inode).getPreferredBlockSize();
+ return INodeFile.valueOf(rootDir.getNode(path, false), path
+ ).getPreferredBlockSize();
} finally {
readUnlock();
}
@@ -877,9 +833,7 @@ public class FSDirectory implements Clos
if (inode == null) {
return false;
}
- return inode.isDirectory() || inode.isSymlink()
- ? true
- : ((INodeFile)inode).getBlocks() != null;
+ return !inode.isFile() || ((INodeFile)inode).getBlocks() != null;
} finally {
readUnlock();
}
@@ -1095,14 +1049,13 @@ public class FSDirectory implements Clos
" because the root is not allowed to be deleted");
return 0;
}
- int pos = inodes.length - 1;
// Remove the node from the namespace
- targetNode = removeChild(inodesInPath, pos);
+ targetNode = removeLastINode(inodesInPath);
if (targetNode == null) {
return 0;
}
// set the parent's modification time
- inodes[pos-1].setModificationTime(mtime);
+ inodes[inodes.length - 2].setModificationTime(mtime);
int filesRemoved = targetNode.collectSubtreeBlocksAndClear(collectedBlocks);
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "
@@ -1131,7 +1084,7 @@ public class FSDirectory implements Clos
/* Currently oldnode and newnode are assumed to contain the same
* blocks. Otherwise, blocks need to be removed from the blocksMap.
*/
- rootDir.addNode(path, newnode);
+ rootDir.addINode(path, newnode);
int index = 0;
for (BlockInfo b : newnode.getBlocks()) {
@@ -1214,14 +1167,8 @@ public class FSDirectory implements Clos
waitForReady();
readLock();
try {
- INode targetNode = rootDir.getNode(src, false);
- if (targetNode == null)
- return null;
- if (targetNode.isDirectory())
- return null;
- if (targetNode.isSymlink())
- return null;
- return ((INodeFile)targetNode).getBlocks();
+ final INode i = rootDir.getNode(src, false);
+ return i != null && i.isFile()? ((INodeFile)i).getBlocks(): null;
} finally {
readUnlock();
}
@@ -1239,21 +1186,6 @@ public class FSDirectory implements Clos
}
}
- /**
- * Get the parent node of path.
- *
- * @param path the path to explore
- * @return its parent node
- */
- INodeDirectory getParent(byte[][] path)
- throws FileNotFoundException, UnresolvedLinkException {
- readLock();
- try {
- return rootDir.getParent(path);
- } finally {
- readUnlock();
- }
- }
/**
* Check whether the filepath could be created
@@ -1295,20 +1227,17 @@ public class FSDirectory implements Clos
* @param nsDelta the delta change of namespace
* @param dsDelta the delta change of diskspace
* @throws QuotaExceededException if the new count violates any quota limit
- * @throws FileNotFound if path does not exist.
+ * @throws FileNotFoundException if path does not exist.
*/
void updateSpaceConsumed(String path, long nsDelta, long dsDelta)
- throws QuotaExceededException,
- FileNotFoundException,
- UnresolvedLinkException {
+ throws QuotaExceededException, FileNotFoundException, UnresolvedLinkException {
writeLock();
try {
final INodesInPath inodesInPath = rootDir.getExistingPathINodes(path, false);
final INode[] inodes = inodesInPath.getINodes();
int len = inodes.length;
if (inodes[len - 1] == null) {
- throw new FileNotFoundException(path +
- " does not exist under rootDir.");
+ throw new FileNotFoundException("Path not found: " + path);
}
updateCount(inodesInPath, len-1, nsDelta, dsDelta, true);
} finally {
@@ -1532,15 +1461,17 @@ public class FSDirectory implements Clos
long timestamp) throws QuotaExceededException {
assert hasWriteLock();
final INodeDirectory dir = new INodeDirectory(name, permission, timestamp);
- final INode inode = addChild(inodesInPath, pos, dir, -1, true);
- inodesInPath.setINode(pos, inode);
+ if (addChild(inodesInPath, pos, dir, true)) {
+ inodesInPath.setINode(pos, dir);
+ }
}
- /** Add a node child to the namespace. The full path name of the node is src.
- * childDiskspace should be -1, if unknown.
+ /**
+ * Add the given child to the namespace.
+ * @param src The full path name of the child node.
* @throw QuotaExceededException is thrown if it violates quota limit
*/
- private <T extends INode> T addNode(String src, T child, long childDiskspace
+ private boolean addINode(String src, INode child
) throws QuotaExceededException, UnresolvedLinkException {
byte[][] components = INode.getPathComponents(src);
byte[] path = components[components.length-1];
@@ -1550,8 +1481,7 @@ public class FSDirectory implements Clos
try {
INodesInPath inodesInPath = rootDir.getExistingPathINodes(components,
components.length, false);
- return addChild(inodesInPath, inodesInPath.getINodes().length-1, child,
- childDiskspace, true);
+ return addLastINode(inodesInPath, child, true);
} finally {
writeUnlock();
}
@@ -1675,14 +1605,24 @@ public class FSDirectory implements Clos
}
}
+ /**
+ * The same as {@link #addChild(INodesInPath, int, INode, boolean)}
+ * with pos = length - 1.
+ */
+ private boolean addLastINode(INodesInPath inodesInPath,
+ INode inode, boolean checkQuota) throws QuotaExceededException {
+ final int pos = inodesInPath.getINodes().length - 1;
+ return addChild(inodesInPath, pos, inode, checkQuota);
+ }
+
/** Add a node child to the inodes at index pos.
* Its ancestors are stored at [0, pos-1].
- * @return the added node.
+ * @return false if the child with this name already exists;
+ * otherwise return true;
* @throw QuotaExceededException is thrown if it violates quota limit
*/
- private <T extends INode> T addChild(INodesInPath inodesInPath, int pos,
- T child, long childDiskspace,
- boolean checkQuota) throws QuotaExceededException {
+ private boolean addChild(INodesInPath inodesInPath, int pos,
+ INode child, boolean checkQuota) throws QuotaExceededException {
final INode[] inodes = inodesInPath.getINodes();
// The filesystem limits are not really quotas, so this check may appear
// odd. It's because a rename operation deletes the src, tries to add
@@ -1696,38 +1636,34 @@ public class FSDirectory implements Clos
INode.DirCounts counts = new INode.DirCounts();
child.spaceConsumedInTree(counts);
- if (childDiskspace < 0) {
- childDiskspace = counts.getDsCount();
- }
- updateCount(inodesInPath, pos, counts.getNsCount(), childDiskspace, checkQuota);
+ updateCount(inodesInPath, pos, counts.getNsCount(), counts.getDsCount(), checkQuota);
if (inodes[pos-1] == null) {
throw new NullPointerException("Panic: parent does not exist");
}
- final T addedNode = ((INodeDirectory)inodes[pos-1]).addChild(child, true);
- if (addedNode == null) {
- updateCount(inodesInPath, pos, -counts.getNsCount(), -childDiskspace, true);
+ final boolean added = ((INodeDirectory)inodes[pos-1]).addChild(child, true);
+ if (!added) {
+ updateCount(inodesInPath, pos, -counts.getNsCount(), -counts.getDsCount(), true);
}
- return addedNode;
+ return added;
}
- private <T extends INode> T addChildNoQuotaCheck(INodesInPath inodesInPath,
- int pos, T child, long childDiskspace) {
- T inode = null;
+ private boolean addLastINodeNoQuotaCheck(INodesInPath inodesInPath, INode i) {
try {
- inode = addChild(inodesInPath, pos, child, childDiskspace, false);
+ return addLastINode(inodesInPath, i, false);
} catch (QuotaExceededException e) {
NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
}
- return inode;
+ return false;
}
- /** Remove an inode at index pos from the namespace.
- * Its ancestors are stored at [0, pos-1].
+ /**
+ * Remove the last inode in the path from the namespace.
* Count of each ancestor with quota is also updated.
- * Return the removed node; null if the removal fails.
+ * @return the removed node; null if the removal fails.
*/
- private INode removeChild(final INodesInPath inodesInPath, int pos) {
+ private INode removeLastINode(final INodesInPath inodesInPath) {
final INode[] inodes = inodesInPath.getINodes();
+ final int pos = inodes.length - 1;
INode removedNode = ((INodeDirectory)inodes[pos-1]).removeChild(inodes[pos]);
if (removedNode != null) {
INode.DirCounts counts = new INode.DirCounts();
@@ -1846,14 +1782,15 @@ public class FSDirectory implements Clos
* See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
* Sets quota for for a directory.
* @returns INodeDirectory if any of the quotas have changed. null other wise.
- * @throws FileNotFoundException if the path does not exist or is a file
+ * @throws FileNotFoundException if the path does not exist.
+ * @throws PathIsNotDirectoryException if the path is not a directory.
* @throws QuotaExceededException if the directory tree size is
* greater than the given quota
* @throws UnresolvedLinkException if a symlink is encountered in src.
*/
INodeDirectory unprotectedSetQuota(String src, long nsQuota, long dsQuota)
- throws FileNotFoundException, QuotaExceededException,
- UnresolvedLinkException {
+ throws FileNotFoundException, PathIsNotDirectoryException,
+ QuotaExceededException, UnresolvedLinkException {
assert hasWriteLock();
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
@@ -1869,15 +1806,10 @@ public class FSDirectory implements Clos
final INodesInPath inodesInPath = rootDir.getExistingPathINodes(src, true);
final INode[] inodes = inodesInPath.getINodes();
- INode targetNode = inodes[inodes.length-1];
- if (targetNode == null) {
- throw new FileNotFoundException("Directory does not exist: " + srcs);
- } else if (!targetNode.isDirectory()) {
- throw new FileNotFoundException("Cannot set quota on a file: " + srcs);
- } else if (targetNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
+ INodeDirectory dirNode = INodeDirectory.valueOf(inodes[inodes.length-1], srcs);
+ if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
throw new IllegalArgumentException("Cannot clear namespace quota on root.");
} else { // a directory inode
- INodeDirectory dirNode = (INodeDirectory)targetNode;
long oldNsQuota = dirNode.getNsQuota();
long oldDsQuota = dirNode.getDsQuota();
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
@@ -1911,13 +1843,12 @@ public class FSDirectory implements Clos
}
/**
- * See {@link ClientProtocol#setQuota(String, long, long)} for the
- * contract.
+ * See {@link ClientProtocol#setQuota(String, long, long)} for the contract.
* @see #unprotectedSetQuota(String, long, long)
*/
void setQuota(String src, long nsQuota, long dsQuota)
- throws FileNotFoundException, QuotaExceededException,
- UnresolvedLinkException {
+ throws FileNotFoundException, PathIsNotDirectoryException,
+ QuotaExceededException, UnresolvedLinkException {
writeLock();
try {
INodeDirectory dir = unprotectedSetQuota(src, nsQuota, dsQuota);
@@ -1992,9 +1923,7 @@ public class FSDirectory implements Clos
writeLock();
try {
setReady(false);
- rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
- getFSNamesystem().createFsOwnerPermissions(new FsPermission((short)0755)),
- Integer.MAX_VALUE, -1);
+ rootDir = createRoot(getFSNamesystem());
nameCache.reset();
} finally {
writeUnlock();
@@ -2089,7 +2018,7 @@ public class FSDirectory implements Clos
INodeSymlink addSymlink(String path, String target,
PermissionStatus dirPerms, boolean createParent)
throws UnresolvedLinkException, FileAlreadyExistsException,
- QuotaExceededException, IOException {
+ QuotaExceededException {
waitForReady();
final long modTime = now();
@@ -2103,7 +2032,7 @@ public class FSDirectory implements Clos
INodeSymlink newNode = null;
writeLock();
try {
- newNode = unprotectedSymlink(path, target, modTime, modTime,
+ newNode = unprotectedAddSymlink(path, target, modTime, modTime,
new PermissionStatus(userName, null, FsPermission.getDefault()));
} finally {
writeUnlock();
@@ -2123,23 +2052,12 @@ public class FSDirectory implements Clos
/**
* Add the specified path into the namespace. Invoked from edit log processing.
*/
- INodeSymlink unprotectedSymlink(String path, String target, long modTime,
+ INodeSymlink unprotectedAddSymlink(String path, String target, long mtime,
long atime, PermissionStatus perm)
- throws UnresolvedLinkException {
+ throws UnresolvedLinkException, QuotaExceededException {
assert hasWriteLock();
- INodeSymlink newNode = new INodeSymlink(target, modTime, atime, perm);
- try {
- newNode = addNode(path, newNode, UNKNOWN_DISK_SPACE);
- } catch (UnresolvedLinkException e) {
- /* All UnresolvedLinkExceptions should have been resolved by now, but we
- * should re-throw them in case that changes so they are not swallowed
- * by catching IOException below.
- */
- throw e;
- } catch (IOException e) {
- return null;
- }
- return newNode;
+ final INodeSymlink symlink = new INodeSymlink(target, mtime, atime, perm);
+ return addINode(path, symlink)? symlink: null;
}
/**
@@ -2148,7 +2066,7 @@ public class FSDirectory implements Clos
*/
void cacheName(INode inode) {
// Name is cached only for files
- if (inode.isDirectory() || inode.isSymlink()) {
+ if (!inode.isFile()) {
return;
}
ByteArray name = new ByteArray(inode.getLocalNameBytes());
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Wed Nov 28 15:17:21 2012
@@ -426,7 +426,7 @@ public class FSEditLogLoader {
}
case OP_SYMLINK: {
SymlinkOp symlinkOp = (SymlinkOp)op;
- fsDir.unprotectedSymlink(symlinkOp.path, symlinkOp.value,
+ fsDir.unprotectedAddSymlink(symlinkOp.path, symlinkOp.value,
symlinkOp.mtime, symlinkOp.atime,
symlinkOp.permissionStatus);
break;
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java Wed Nov 28 15:17:21 2012
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
@@ -202,7 +203,7 @@ class FSImageFormat {
fsDir.rootDir.setQuota(nsQuota, dsQuota);
}
fsDir.rootDir.setModificationTime(root.getModificationTime());
- fsDir.rootDir.setPermissionStatus(root.getPermissionStatus());
+ fsDir.rootDir.clonePermissionStatus(root);
}
/**
@@ -258,7 +259,7 @@ class FSImageFormat {
// add to parent
newNode.setLocalName(localName);
- namesystem.dir.addToParent(parent, newNode, false);
+ addToParent(parent, newNode);
}
return numChildren;
}
@@ -287,13 +288,36 @@ class FSImageFormat {
}
// check if the new inode belongs to the same parent
if(!isParent(pathComponents, parentPath)) {
- parentINode = fsDir.getParent(pathComponents);
+ parentINode = fsDir.rootDir.getParent(pathComponents);
parentPath = getParent(pathComponents);
}
// add new inode
newNode.setLocalName(pathComponents[pathComponents.length-1]);
- parentINode = fsDir.addToParent(parentINode, newNode, false);
+ addToParent(parentINode, newNode);
+ }
+ }
+
+ /**
+ * Add the child node to parent and, if child is a file, update block map.
+ * This method is only used for image loading so that synchronization,
+ * modification time update and space count update are not needed.
+ */
+ void addToParent(INodeDirectory parent, INode child) {
+ // NOTE: This does not update space counts for parents
+ if (!parent.addChild(child, false)) {
+ return;
+ }
+ namesystem.dir.cacheName(child);
+
+ if (child.isFile()) {
+ // Add file->block mapping
+ final INodeFile file = (INodeFile)child;
+ final BlockInfo[] blocks = file.getBlocks();
+ final BlockManager bm = namesystem.getBlockManager();
+ for (int i = 0; i < blocks.length; i++) {
+ file.setBlock(i, bm.addBlockCollection(blocks[i], file));
+ }
}
}
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Wed Nov 28 15:17:21 2012
@@ -86,17 +86,17 @@ abstract class INode implements Comparab
return (record & ~MASK) | (bits << OFFSET);
}
- /** Set the {@link PermissionStatus} */
+ /** Encode the {@link PermissionStatus} to a long. */
static long toLong(PermissionStatus ps) {
long permission = 0L;
final int user = SerialNumberManager.INSTANCE.getUserSerialNumber(
ps.getUserName());
- permission = PermissionStatusFormat.USER.combine(user, permission);
+ permission = USER.combine(user, permission);
final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber(
ps.getGroupName());
- permission = PermissionStatusFormat.GROUP.combine(group, permission);
+ permission = GROUP.combine(group, permission);
final int mode = ps.getPermission().toShort();
- permission = PermissionStatusFormat.MODE.combine(mode, permission);
+ permission = MODE.combine(mode, permission);
return permission;
}
}
@@ -110,8 +110,9 @@ abstract class INode implements Comparab
*/
private byte[] name = null;
/**
- * Permission encoded using PermissionStatusFormat.
- * Codes other than {@link #updatePermissionStatus(PermissionStatusFormat, long)}.
+ * Permission encoded using {@link PermissionStatusFormat}.
+ * Codes other than {@link #clonePermissionStatus(INode)}
+ * and {@link #updatePermissionStatus(PermissionStatusFormat, long)}
* should not modify it.
*/
private long permission = 0L;
@@ -155,11 +156,9 @@ abstract class INode implements Comparab
return name.length == 0;
}
- /** Set the {@link PermissionStatus} */
- protected void setPermissionStatus(PermissionStatus ps) {
- setUser(ps.getUserName());
- setGroup(ps.getGroupName());
- setPermission(ps.getPermission());
+ /** Clone the {@link PermissionStatus}. */
+ void clonePermissionStatus(INode that) {
+ this.permission = that.permission;
}
/** Get the {@link PermissionStatus} */
protected PermissionStatus getPermissionStatus() {
@@ -202,6 +201,13 @@ abstract class INode implements Comparab
}
/**
+ * Check whether it's a file.
+ */
+ public boolean isFile() {
+ return false;
+ }
+
+ /**
* Check whether it's a directory
*/
public boolean isDirectory() {
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java Wed Nov 28 15:17:21 2012
@@ -18,12 +18,12 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.FileNotFoundException;
-import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
@@ -36,13 +36,14 @@ import com.google.common.annotations.Vis
*/
class INodeDirectory extends INode {
/** Cast INode to INodeDirectory. */
- public static INodeDirectory valueOf(INode inode, String path
- ) throws IOException {
+ public static INodeDirectory valueOf(INode inode, Object path
+ ) throws FileNotFoundException, PathIsNotDirectoryException {
if (inode == null) {
- throw new IOException("Directory does not exist: " + path);
+ throw new FileNotFoundException("Directory does not exist: "
+ + DFSUtil.path2String(path));
}
if (!inode.isDirectory()) {
- throw new IOException("Path is not a directory: " + path);
+ throw new PathIsNotDirectoryException(DFSUtil.path2String(path));
}
return (INodeDirectory)inode;
}
@@ -277,16 +278,17 @@ class INodeDirectory extends INode {
* @param setModTime set modification time for the parent node
* not needed when replaying the addition and
* the parent already has the proper mod time
- * @return null if the child with this name already exists;
- * node, otherwise
+ * @return false if the child with this name already exists;
+ * otherwise, return true;
*/
- <T extends INode> T addChild(final T node, boolean setModTime) {
+ boolean addChild(final INode node, final boolean setModTime) {
if (children == null) {
children = new ArrayList<INode>(DEFAULT_FILES_PER_DIRECTORY);
}
final int low = searchChildren(node);
- if(low >= 0)
- return null;
+ if (low >= 0) {
+ return false;
+ }
node.parent = this;
children.add(-low - 1, node);
// update modification time of the parent directory
@@ -295,7 +297,7 @@ class INodeDirectory extends INode {
if (node.getGroupName() == null) {
node.setGroup(getGroupName());
}
- return node;
+ return true;
}
/**
@@ -304,70 +306,32 @@ class INodeDirectory extends INode {
*
* @param path file path
* @param newNode INode to be added
- * @return null if the node already exists; inserted INode, otherwise
+ * @return false if the node already exists; otherwise, return true;
* @throws FileNotFoundException if parent does not exist or
* @throws UnresolvedLinkException if any path component is a symbolic link
* is not a directory.
*/
- <T extends INode> T addNode(String path, T newNode
- ) throws FileNotFoundException, UnresolvedLinkException {
+ boolean addINode(String path, INode newNode
+ ) throws FileNotFoundException, PathIsNotDirectoryException,
+ UnresolvedLinkException {
byte[][] pathComponents = getPathComponents(path);
- return addToParent(pathComponents, newNode, true) == null? null: newNode;
- }
-
- /**
- * Add new inode to the parent if specified.
- * Optimized version of addNode() if parent is not null.
- *
- * @return parent INode if new inode is inserted
- * or null if it already exists.
- * @throws FileNotFoundException if parent does not exist or
- * is not a directory.
- */
- INodeDirectory addToParent(INode newNode, INodeDirectory parent,
- boolean propagateModTime) throws FileNotFoundException {
+ if (pathComponents.length < 2) { // add root
+ return false;
+ }
+ newNode.setLocalName(pathComponents[pathComponents.length - 1]);
// insert into the parent children list
- if(parent.addChild(newNode, propagateModTime) == null)
- return null;
- return parent;
+ INodeDirectory parent = getParent(pathComponents);
+ return parent.addChild(newNode, true);
}
INodeDirectory getParent(byte[][] pathComponents
- ) throws FileNotFoundException, UnresolvedLinkException {
+ ) throws FileNotFoundException, PathIsNotDirectoryException,
+ UnresolvedLinkException {
if (pathComponents.length < 2) // add root
return null;
// Gets the parent INode
INodesInPath inodes = getExistingPathINodes(pathComponents, 2, false);
- INode inode = inodes.inodes[0];
- if (inode == null) {
- throw new FileNotFoundException("Parent path does not exist: "+
- DFSUtil.byteArray2String(pathComponents));
- }
- if (!inode.isDirectory()) {
- throw new FileNotFoundException("Parent path is not a directory: "+
- DFSUtil.byteArray2String(pathComponents));
- }
- return (INodeDirectory)inode;
- }
-
- /**
- * Add new inode
- * Optimized version of addNode()
- *
- * @return parent INode if new inode is inserted
- * or null if it already exists.
- * @throws FileNotFoundException if parent does not exist or
- * is not a directory.
- */
- INodeDirectory addToParent(byte[][] pathComponents, INode newNode,
- boolean propagateModTime) throws FileNotFoundException, UnresolvedLinkException {
- if (pathComponents.length < 2) { // add root
- return null;
- }
- newNode.setLocalName(pathComponents[pathComponents.length - 1]);
- // insert into the parent children list
- INodeDirectory parent = getParent(pathComponents);
- return parent.addChild(newNode, propagateModTime) == null? null: parent;
+ return INodeDirectory.valueOf(inodes.inodes[0], pathComponents);
}
@Override
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java Wed Nov 28 15:17:21 2012
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.na
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -26,9 +27,13 @@ import org.apache.hadoop.hdfs.protocol.Q
* Directory INode class that has a quota restriction
*/
class INodeDirectoryWithQuota extends INodeDirectory {
- private long nsQuota; /// NameSpace quota
+ /** Name space quota */
+ private long nsQuota = Long.MAX_VALUE;
+ /** Name space count */
private long nsCount = 1L;
- private long dsQuota; /// disk space quota
+ /** Disk space quota */
+ private long dsQuota = HdfsConstants.QUOTA_RESET;
+ /** Disk space count */
private long diskspace = 0L;
/** Convert an existing directory inode to one with the given quota
@@ -57,11 +62,8 @@ class INodeDirectoryWithQuota extends IN
}
/** constructor with no quota verification */
- INodeDirectoryWithQuota(String name, PermissionStatus permissions,
- long nsQuota, long dsQuota) {
+ INodeDirectoryWithQuota(String name, PermissionStatus permissions) {
super(name, permissions);
- this.nsQuota = nsQuota;
- this.dsQuota = dsQuota;
}
/** Get this directory's namespace quota
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java Wed Nov 28 15:17:21 2012
@@ -33,7 +33,8 @@ import org.apache.hadoop.hdfs.server.blo
@InterfaceAudience.Private
public class INodeFile extends INode implements BlockCollection {
/** Cast INode to INodeFile. */
- public static INodeFile valueOf(INode inode, String path) throws IOException {
+ public static INodeFile valueOf(INode inode, String path
+ ) throws FileNotFoundException {
if (inode == null) {
throw new FileNotFoundException("File does not exist: " + path);
}
@@ -94,6 +95,12 @@ public class INodeFile extends INode imp
this.blocks = blklist;
}
+ /** @return true unconditionally. */
+ @Override
+ public final boolean isFile() {
+ return true;
+ }
+
/**
* Set the {@link FsPermission} of this {@link INodeFile}.
* Since this is a file,
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java Wed Nov 28 15:17:21 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
@@ -36,10 +37,10 @@ import org.apache.hadoop.hdfs.server.com
class INodeFileUnderConstruction extends INodeFile implements MutableBlockCollection {
/** Cast INode to INodeFileUnderConstruction. */
public static INodeFileUnderConstruction valueOf(INode inode, String path
- ) throws IOException {
+ ) throws FileNotFoundException {
final INodeFile file = INodeFile.valueOf(inode, path);
if (!file.isUnderConstruction()) {
- throw new IOException("File is not under construction: " + path);
+ throw new FileNotFoundException("File is not under construction: " + path);
}
return (INodeFileUnderConstruction)file;
}
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsBinaryLoader.java Wed Nov 28 15:17:21 2012
@@ -79,6 +79,7 @@ class OfflineEditsBinaryLoader implement
} catch (IOException e) {
if (!recoveryMode) {
// Tell the visitor to clean up, then re-throw the exception
+ LOG.error("Got IOException at position " + inputStream.getPosition());
visitor.close(e);
throw e;
}
@@ -87,6 +88,7 @@ class OfflineEditsBinaryLoader implement
} catch (RuntimeException e) {
if (!recoveryMode) {
// Tell the visitor to clean up, then re-throw the exception
+ LOG.error("Got RuntimeException at position " + inputStream.getPosition());
visitor.close(e);
throw e;
}
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Wed Nov 28 15:17:21 2012
@@ -124,15 +124,14 @@ public class WebHdfsFileSystem extends F
public static final WebHdfsDelegationTokenSelector DT_SELECTOR
= new WebHdfsDelegationTokenSelector();
- private static DelegationTokenRenewer<WebHdfsFileSystem> DT_RENEWER = null;
+ private DelegationTokenRenewer dtRenewer = null;
- private static synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
- if (DT_RENEWER == null) {
- DT_RENEWER = new DelegationTokenRenewer<WebHdfsFileSystem>(WebHdfsFileSystem.class);
- DT_RENEWER.start();
+ private synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
+ if (dtRenewer == null) {
+ dtRenewer = DelegationTokenRenewer.getInstance();
}
- DT_RENEWER.addRenewAction(webhdfs);
+ dtRenewer.addRenewAction(webhdfs);
}
/** Is WebHDFS enabled in conf? */
@@ -766,6 +765,14 @@ public class WebHdfsFileSystem extends F
new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
}
+ @Override
+ public void close() throws IOException {
+ super.close();
+ if (dtRenewer != null) {
+ dtRenewer.removeRenewAction(this); // blocks
+ }
+ }
+
class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
OffsetUrlOpener(final URL url) {
super(url);
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1411704-1414746
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1411704-1414746
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1411704-1414746
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1411704-1414746
Propchange: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1411704-1414746
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestFcHdfsSymlink.java Wed Nov 28 15:17:21 2012
@@ -28,9 +28,11 @@ import org.apache.commons.logging.impl.L
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
@@ -51,6 +53,7 @@ public class TestFcHdfsSymlink extends F
private static MiniDFSCluster cluster;
private static WebHdfsFileSystem webhdfs;
+ private static DistributedFileSystem dfs;
@Override
@@ -89,6 +92,7 @@ public class TestFcHdfsSymlink extends F
cluster = new MiniDFSCluster.Builder(conf).build();
fc = FileContext.getFileContext(cluster.getURI(0));
webhdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
+ dfs = cluster.getFileSystem();
}
@AfterClass
@@ -317,4 +321,27 @@ public class TestFcHdfsSymlink extends F
assertEquals(2, fc.getFileStatus(link).getReplication());
assertEquals(2, fc.getFileStatus(file).getReplication());
}
+
+ @Test
+ /** Test craeteSymlink(..) with quota. */
+ public void testQuota() throws IOException {
+ final Path dir = new Path(testBaseDir1());
+ dfs.setQuota(dir, 3, HdfsConstants.QUOTA_DONT_SET);
+
+ final Path file = new Path(dir, "file");
+ createAndWriteFile(file);
+
+ //creating the first link should succeed
+ final Path link1 = new Path(dir, "link1");
+ fc.createSymlink(file, link1, false);
+
+ try {
+ //creating the second link should fail with QuotaExceededException.
+ final Path link2 = new Path(dir, "link2");
+ fc.createSymlink(file, link2, false);
+ fail("Created symlink despite quota violation");
+ } catch(QuotaExceededException qee) {
+ //expected
+ }
+ }
}
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java Wed Nov 28 15:17:21 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -33,6 +34,8 @@ public class TestHDFSFileSystemContract
@Override
protected void setUp() throws Exception {
Configuration conf = new HdfsConfiguration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+ FileSystemContractBaseTest.TEST_UMASK);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
fs = cluster.getFileSystem();
defaultWorkingDirectory = "/user/" +
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java Wed Nov 28 15:17:21 2012
@@ -73,7 +73,7 @@ public class TestFsLimits {
fileAsURI(new File(MiniDFSCluster.getBaseDirectory(),
"namenode")).toString());
- rootInode = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME, perms, 0L, 0L);
+ rootInode = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME, perms);
inodes = new INode[]{ rootInode, null };
fs = null;
fsIsReady = true;
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Wed Nov 28 15:17:21 2012
@@ -26,6 +26,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -239,8 +240,8 @@ public class TestINodeFile {
try {
INodeDirectory.valueOf(from, path);
fail();
- } catch(IOException ioe) {
- assertTrue(ioe.getMessage().contains("Directory does not exist"));
+ } catch(FileNotFoundException e) {
+ assertTrue(e.getMessage().contains("Directory does not exist"));
}
}
@@ -264,8 +265,7 @@ public class TestINodeFile {
try {
INodeDirectory.valueOf(from, path);
fail();
- } catch(IOException ioe) {
- assertTrue(ioe.getMessage().contains("Path is not a directory"));
+ } catch(PathIsNotDirectoryException e) {
}
}
@@ -286,8 +286,7 @@ public class TestINodeFile {
try {
INodeDirectory.valueOf(from, path);
fail();
- } catch(IOException ioe) {
- assertTrue(ioe.getMessage().contains("Path is not a directory"));
+ } catch(PathIsNotDirectoryException e) {
}
}
Modified: hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1414750&r1=1414749&r2=1414750&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml (original)
+++ hadoop/common/branches/MR-2454/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Wed Nov 28 15:17:21 2012
@@ -15488,7 +15488,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Cannot set quota on a file: /test/file1</expected-output>
+ <expected-output>setQuota: `/test/file1': Is not a directory</expected-output>
</comparator>
</comparators>
</test>