You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by vi...@apache.org on 2013/10/30 23:22:22 UTC
svn commit: r1537330 [6/11] - in
/hadoop/common/branches/YARN-321/hadoop-hdfs-project: ./
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/
hadoop-hdfs-nfs/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/sr...
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java Wed Oct 30 22:21:59 2013
@@ -18,10 +18,9 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
@@ -37,9 +36,9 @@ import javax.servlet.jsp.JspWriter;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -47,20 +46,23 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.http.HtmlQuoting;
-import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
@InterfaceAudience.Private
public class DatanodeJspHelper {
+ private static final int PREV_BLOCK = -1;
+ private static final int NEXT_BLOCK = 1;
+
private static DFSClient getDFSClient(final UserGroupInformation user,
final String addr,
final Configuration conf
@@ -143,10 +145,10 @@ public class DatanodeJspHelper {
out.print("Empty file");
} else {
DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock, conf);
- String fqdn = canonicalize(chosenNode.getIpAddr());
int datanodePort = chosenNode.getXferPort();
- String redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":"
- + chosenNode.getInfoPort() + "/browseBlock.jsp?blockId="
+ String redirectLocation = JspHelper.Url.url(req.getScheme(),
+ chosenNode)
+ + "/browseBlock.jsp?blockId="
+ firstBlock.getBlock().getBlockId() + "&blockSize="
+ firstBlock.getBlock().getNumBytes() + "&genstamp="
+ firstBlock.getBlock().getGenerationStamp() + "&filename="
@@ -225,7 +227,7 @@ public class DatanodeJspHelper {
JspHelper.addTableFooter(out);
}
}
- out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ out.print("<br><a href=\"///"
+ canonicalize(nnAddr) + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -302,8 +304,7 @@ public class DatanodeJspHelper {
Long.MAX_VALUE).getLocatedBlocks();
// Add the various links for looking at the file contents
// URL for downloading the full file
- String downloadUrl = HttpConfig.getSchemePrefix() + req.getServerName() + ":"
- + req.getServerPort() + "/streamFile" + ServletUtil.encodePath(filename)
+ String downloadUrl = "/streamFile" + ServletUtil.encodePath(filename)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr, true)
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a name=\"viewOptions\"></a>");
@@ -319,8 +320,8 @@ public class DatanodeJspHelper {
dfs.close();
return;
}
- String fqdn = canonicalize(chosenNode.getIpAddr());
- String tailUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + chosenNode.getInfoPort()
+
+ String tailUrl = "///" + JspHelper.Url.authority(req.getScheme(), chosenNode)
+ "/tail.jsp?filename=" + URLEncoder.encode(filename, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
+ "&chunkSizeToView=" + chunkSizeToView
@@ -368,8 +369,7 @@ public class DatanodeJspHelper {
for (int j = 0; j < locs.length; j++) {
String datanodeAddr = locs[j].getXferAddr();
datanodePort = locs[j].getXferPort();
- fqdn = canonicalize(locs[j].getIpAddr());
- String blockUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + locs[j].getInfoPort()
+ String blockUrl = "///" + JspHelper.Url.authority(req.getScheme(), locs[j])
+ "/browseBlock.jsp?blockId=" + blockidstring
+ "&blockSize=" + blockSize
+ "&filename=" + URLEncoder.encode(filename, "UTF-8")
@@ -380,7 +380,7 @@ public class DatanodeJspHelper {
+ JspHelper.getDelegationTokenUrlParam(tokenString)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
- String blockInfoUrl = HttpConfig.getSchemePrefix() + nnCanonicalName + ":"
+ String blockInfoUrl = "///" + nnCanonicalName + ":"
+ namenodeInfoPort
+ "/block_info_xml.jsp?blockId=" + blockidstring;
out.print("<td> </td><td><a href=\"" + blockUrl + "\">"
@@ -391,7 +391,7 @@ public class DatanodeJspHelper {
}
out.println("</table>");
out.print("<hr>");
- out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ out.print("<br><a href=\"///"
+ nnCanonicalName + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -491,9 +491,7 @@ public class DatanodeJspHelper {
String parent = new File(filename).getParent();
JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, parent, nnAddr);
out.print("<hr>");
- out.print("<a href=\"" + HttpConfig.getSchemePrefix()
- + req.getServerName() + ":" + req.getServerPort()
- + "/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
+ out.print("<a href=\"/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
+ JspHelper.getDelegationTokenUrlParam(tokenString)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
@@ -501,112 +499,23 @@ public class DatanodeJspHelper {
out.print("<a href=\"#viewOptions\">Advanced view/download options</a><br>");
out.print("<hr>");
- // Determine the prev & next blocks
- long nextStartOffset = 0;
- long nextBlockSize = 0;
- String nextBlockIdStr = null;
- String nextGenStamp = null;
- String nextHost = req.getServerName();
- int nextPort = req.getServerPort();
- int nextDatanodePort = datanodePort;
- // determine data for the next link
- if (startOffset + chunkSizeToView >= blockSize) {
- // we have to go to the next block from this point onwards
- List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
- Long.MAX_VALUE).getLocatedBlocks();
- for (int i = 0; i < blocks.size(); i++) {
- if (blocks.get(i).getBlock().getBlockId() == blockId) {
- if (i != blocks.size() - 1) {
- LocatedBlock nextBlock = blocks.get(i + 1);
- nextBlockIdStr = Long.toString(nextBlock.getBlock().getBlockId());
- nextGenStamp = Long.toString(nextBlock.getBlock()
- .getGenerationStamp());
- nextStartOffset = 0;
- nextBlockSize = nextBlock.getBlock().getNumBytes();
- DatanodeInfo d = JspHelper.bestNode(nextBlock, conf);
- nextDatanodePort = d.getXferPort();
- nextHost = d.getIpAddr();
- nextPort = d.getInfoPort();
- }
- }
- }
- } else {
- // we are in the same block
- nextBlockIdStr = blockId.toString();
- nextStartOffset = startOffset + chunkSizeToView;
- nextBlockSize = blockSize;
- nextGenStamp = genStamp.toString();
- }
- String nextUrl = null;
- if (nextBlockIdStr != null) {
- nextUrl = HttpConfig.getSchemePrefix() + canonicalize(nextHost) + ":" + nextPort
- + "/browseBlock.jsp?blockId=" + nextBlockIdStr
- + "&blockSize=" + nextBlockSize
- + "&startOffset=" + nextStartOffset
- + "&genstamp=" + nextGenStamp
- + "&filename=" + URLEncoder.encode(filename, "UTF-8")
- + "&chunkSizeToView=" + chunkSizeToView
- + "&datanodePort=" + nextDatanodePort
- + "&namenodeInfoPort=" + namenodeInfoPort
- + JspHelper.getDelegationTokenUrlParam(tokenString)
- + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ String authority = req.getServerName() + ":" + req.getServerPort();
+ String nextUrl = generateLinksForAdjacentBlock(NEXT_BLOCK, authority,
+ datanodePort, startOffset, chunkSizeToView, blockSize, blockId,
+ genStamp, dfs, filename, conf, req.getScheme(), tokenString,
+ namenodeInfoPort, nnAddr);
+ if (nextUrl != null) {
out.print("<a href=\"" + nextUrl + "\">View Next chunk</a> ");
}
- // determine data for the prev link
- String prevBlockIdStr = null;
- String prevGenStamp = null;
- long prevStartOffset = 0;
- long prevBlockSize = 0;
- String prevHost = req.getServerName();
- int prevPort = req.getServerPort();
- int prevDatanodePort = datanodePort;
- if (startOffset == 0) {
- List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
- Long.MAX_VALUE).getLocatedBlocks();
- for (int i = 0; i < blocks.size(); i++) {
- if (blocks.get(i).getBlock().getBlockId() == blockId) {
- if (i != 0) {
- LocatedBlock prevBlock = blocks.get(i - 1);
- prevBlockIdStr = Long.toString(prevBlock.getBlock().getBlockId());
- prevGenStamp = Long.toString(prevBlock.getBlock()
- .getGenerationStamp());
- prevStartOffset = prevBlock.getBlock().getNumBytes()
- - chunkSizeToView;
- if (prevStartOffset < 0)
- prevStartOffset = 0;
- prevBlockSize = prevBlock.getBlock().getNumBytes();
- DatanodeInfo d = JspHelper.bestNode(prevBlock, conf);
- prevDatanodePort = d.getXferPort();
- prevHost = d.getIpAddr();
- prevPort = d.getInfoPort();
- }
- }
- }
- } else {
- // we are in the same block
- prevBlockIdStr = blockId.toString();
- prevStartOffset = startOffset - chunkSizeToView;
- if (prevStartOffset < 0)
- prevStartOffset = 0;
- prevBlockSize = blockSize;
- prevGenStamp = genStamp.toString();
- }
- String prevUrl = null;
- if (prevBlockIdStr != null) {
- prevUrl = HttpConfig.getSchemePrefix() + canonicalize(prevHost) + ":" + prevPort
- + "/browseBlock.jsp?blockId=" + prevBlockIdStr
- + "&blockSize=" + prevBlockSize
- + "&startOffset=" + prevStartOffset
- + "&filename=" + URLEncoder.encode(filename, "UTF-8")
- + "&chunkSizeToView=" + chunkSizeToView
- + "&genstamp=" + prevGenStamp
- + "&datanodePort=" + prevDatanodePort
- + "&namenodeInfoPort=" + namenodeInfoPort
- + JspHelper.getDelegationTokenUrlParam(tokenString)
- + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ String prevUrl = generateLinksForAdjacentBlock(PREV_BLOCK, authority,
+ datanodePort, startOffset, chunkSizeToView, blockSize, blockId,
+ genStamp, dfs, filename, conf, req.getScheme(), tokenString,
+ namenodeInfoPort, nnAddr);
+ if (prevUrl != null) {
out.print("<a href=\"" + prevUrl + "\">View Prev chunk</a> ");
}
+
out.print("<hr>");
out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
try {
@@ -621,6 +530,71 @@ public class DatanodeJspHelper {
dfs.close();
}
+ private static String generateLinksForAdjacentBlock(final int direction,
+ String authority, int datanodePort, long startOffset,
+ int chunkSizeToView, long blockSize, long blockId, Long genStamp,
+ final DFSClient dfs, final String filename, final Configuration conf,
+ final String scheme, final String tokenString,
+ final int namenodeInfoPort, final String nnAddr)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+
+ boolean found = false;
+ if ((direction == NEXT_BLOCK && startOffset + chunkSizeToView < blockSize)
+ || (direction == PREV_BLOCK && startOffset != 0)) {
+ // we are in the same block
+ found = true;
+
+ if (direction == NEXT_BLOCK) {
+ startOffset = startOffset + chunkSizeToView;
+ } else {
+ startOffset = Math.max(0, startOffset - chunkSizeToView);
+ }
+ } else {
+ List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
+ Long.MAX_VALUE).getLocatedBlocks();
+
+ final long curBlockId = blockId;
+ int curBlockIdx = Iterables.indexOf(blocks, new Predicate<LocatedBlock>() {
+ @Override
+ public boolean apply(LocatedBlock b) {
+ return b.getBlock().getBlockId() == curBlockId;
+ }
+ });
+ found = curBlockIdx != -1 &&
+ ((direction == NEXT_BLOCK && curBlockIdx < blocks.size() - 1)
+ || (direction == PREV_BLOCK && curBlockIdx > 0));
+
+ if (found) {
+ LocatedBlock nextBlock = blocks.get(curBlockIdx + direction);
+
+ blockId = nextBlock.getBlock().getBlockId();
+ genStamp = nextBlock.getBlock().getGenerationStamp();
+ startOffset = 0;
+ blockSize = nextBlock.getBlock().getNumBytes();
+ DatanodeInfo d = JspHelper.bestNode(nextBlock, conf);
+ datanodePort = d.getXferPort();
+ authority = JspHelper.Url.authority(scheme, d);
+ }
+ }
+
+ if (found) {
+ return "///" + authority
+ + "/browseBlock.jsp?blockId=" + blockId
+ + "&blockSize=" + blockSize
+ + "&startOffset=" + startOffset
+ + "&genstamp=" + genStamp
+ + "&filename=" + URLEncoder.encode(filename, "UTF-8")
+ + "&chunkSizeToView=" + chunkSizeToView
+ + "&datanodePort=" + datanodePort
+ + "&namenodeInfoPort=" + namenodeInfoPort
+ + JspHelper.getDelegationTokenUrlParam(tokenString)
+ + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ } else {
+ return null;
+ }
+ }
+
static void generateFileChunksForTail(JspWriter out, HttpServletRequest req,
Configuration conf
) throws IOException,
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java Wed Oct 30 22:21:59 2013
@@ -191,6 +191,11 @@ public class DirectoryScanner implements
private final FsVolumeSpi volume;
+ /**
+ * Get the file's length in async block scan
+ */
+ private final long blockFileLength;
+
private final static Pattern CONDENSED_PATH_REGEX =
Pattern.compile("(?<!^)(\\\\|/){2,}");
@@ -235,6 +240,7 @@ public class DirectoryScanner implements
getCondensedPath(vol.getBasePath());
this.blockSuffix = blockFile == null ? null :
getSuffix(blockFile, condensedVolPath);
+ this.blockFileLength = (blockFile != null) ? blockFile.length() : 0;
if (metaFile == null) {
this.metaSuffix = null;
} else if (blockFile == null) {
@@ -251,6 +257,10 @@ public class DirectoryScanner implements
new File(volume.getBasePath(), blockSuffix);
}
+ long getBlockFileLength() {
+ return blockFileLength;
+ }
+
File getMetaFile() {
if (metaSuffix == null) {
return null;
@@ -458,7 +468,7 @@ public class DirectoryScanner implements
// Block metadata file exits and block file is missing
addDifference(diffRecord, statsRecord, info);
} else if (info.getGenStamp() != memBlock.getGenerationStamp()
- || info.getBlockFile().length() != memBlock.getNumBytes()) {
+ || info.getBlockFileLength() != memBlock.getNumBytes()) {
// Block metadata file is missing or has wrong generation stamp,
// or block file length is different than expected
statsRecord.mismatchBlocks++;
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/RollingLogs.java Wed Oct 30 22:21:59 2013
@@ -33,6 +33,12 @@ public interface RollingLogs {
public interface LineIterator extends Iterator<String>, Closeable {
/** Is the iterator iterating the previous? */
public boolean isPrevious();
+
+ /**
+ * Is the last read entry from previous? This should be called after
+ * reading.
+ */
+ public boolean isLastReadFromPrevious();
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RollingLogsImpl.java Wed Oct 30 22:21:59 2013
@@ -134,6 +134,7 @@ class RollingLogsImpl implements Rolling
*/
private class Reader implements RollingLogs.LineIterator {
private File file;
+ private File lastReadFile;
private BufferedReader reader;
private String line;
private boolean closed = false;
@@ -149,6 +150,11 @@ class RollingLogsImpl implements Rolling
return file == prev;
}
+ @Override
+ public boolean isLastReadFromPrevious() {
+ return lastReadFile == prev;
+ }
+
private boolean openFile() throws IOException {
for(int i=0; i<2; i++) {
@@ -203,6 +209,7 @@ class RollingLogsImpl implements Rolling
public String next() {
String curLine = line;
try {
+ lastReadFile = file;
readNext();
} catch (IOException e) {
DataBlockScanner.LOG.warn("Failed to read next line.", e);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Wed Oct 30 22:21:59 2013
@@ -413,8 +413,15 @@ public class DatanodeWebHdfsMethods {
final long n = length.getValue() != null ?
Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
in.getVisibleLength() - offset.getValue();
- return Response.ok(new OpenEntity(in, n, dfsclient)).type(
- MediaType.APPLICATION_OCTET_STREAM).build();
+
+ /**
+ * Allow the Web UI to perform an AJAX request to get the data.
+ */
+ return Response.ok(new OpenEntity(in, n, dfsclient))
+ .type(MediaType.APPLICATION_OCTET_STREAM)
+ .header("Access-Control-Allow-Methods", "GET")
+ .header("Access-Control-Allow-Origin", "*")
+ .build();
}
case GETFILECHECKSUM:
{
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java Wed Oct 30 22:21:59 2013
@@ -827,7 +827,7 @@ class ClusterJspHelper {
doc.startTag("item");
doc.attribute("label", label);
doc.attribute("value", value);
- doc.attribute("link", HttpConfig.getSchemePrefix() + url);
+ doc.attribute("link", "///" + url);
doc.endTag(); // item
}
@@ -887,7 +887,16 @@ class ClusterJspHelper {
private static String queryMbean(String httpAddress, Configuration conf)
throws IOException {
- URL url = new URL(HttpConfig.getSchemePrefix() + httpAddress+JMX_QRY);
+ /**
+ * Although the other namenode might support HTTPS, it is fundamentally
+ * broken to get the JMX via an HTTPS connection inside the namenode,
+ * because in HTTPS set up the principal of the client and the one of
+ * the namenode differs. Therefore, there is no guarantees that the
+ * HTTPS connection can be set up.
+ *
+ * As a result, we just hard code the connection as an HTTP connection.
+ */
+ URL url = new URL("http://" + httpAddress + JMX_QRY);
return readOutput(url);
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Wed Oct 30 22:21:59 2013
@@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
import org.apache.hadoop.hdfs.util.ByteArray;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.annotations.VisibleForTesting;
@@ -949,7 +950,7 @@ public class FSDirectory implements Clos
if (removedDst != null) {
undoRemoveDst = false;
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
filesDeleted = removedDst.cleanSubtree(null,
dstIIP.getLatestSnapshot(), collectedBlocks, removedINodes, true)
.get(Quota.NAMESPACE);
@@ -1363,7 +1364,7 @@ public class FSDirectory implements Clos
QuotaExceededException, SnapshotAccessControlException {
assert hasWriteLock();
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
final INodesInPath inodesInPath = rootDir.getINodesInPath4Write(
normalizePath(src), false);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Wed Oct 30 22:21:59 2013
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.List;
@@ -77,6 +76,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.Holder;
import com.google.common.base.Joiner;
@@ -584,7 +584,7 @@ public class FSEditLogLoader {
case OP_DELETE_SNAPSHOT: {
DeleteSnapshotOp deleteSnapshotOp = (DeleteSnapshotOp) op;
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
fsNamesys.getSnapshotManager().deleteSnapshot(
deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName,
collectedBlocks, removedINodes);
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Wed Oct 30 22:21:59 2013
@@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DEFAULT_AUDIT_LOGGER_NAME;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY;
@@ -121,6 +123,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsServerDefaults;
@@ -162,7 +165,14 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.*;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics;
+import org.apache.hadoop.hdfs.server.blockmanagement.OutOfV1GenerationStampsException;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
@@ -175,12 +185,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
-import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
@@ -192,6 +196,12 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
+import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -201,8 +211,10 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.RetryCache;
import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
@@ -219,6 +231,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DataChecksum;
@@ -230,6 +243,7 @@ import org.mortbay.util.ajax.JSON;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
/***************************************************
@@ -295,8 +309,14 @@ public class FSNamesystem implements Nam
stat.getGroup(), symlink, path);
}
for (AuditLogger logger : auditLoggers) {
- logger.logAuditEvent(succeeded, ugi.toString(), addr,
- cmd, src, dst, status);
+ if (logger instanceof HdfsAuditLogger) {
+ HdfsAuditLogger hdfsLogger = (HdfsAuditLogger) logger;
+ hdfsLogger.logAuditEvent(succeeded, ugi.toString(), addr, cmd, src, dst,
+ status, ugi, dtSecretManager);
+ } else {
+ logger.logAuditEvent(succeeded, ugi.toString(), addr,
+ cmd, src, dst, status);
+ }
}
}
@@ -410,7 +430,7 @@ public class FSNamesystem implements Nam
private final long accessTimePrecision;
/** Lock to protect FSNamesystem. */
- private ReentrantReadWriteLock fsLock = new ReentrantReadWriteLock(true);
+ private ReentrantReadWriteLock fsLock;
/**
* Used when this NN is in standby state to read from the shared edit log.
@@ -429,6 +449,11 @@ public class FSNamesystem implements Nam
private HAContext haContext;
private final boolean haEnabled;
+
+ /**
+ * Whether the namenode is in the middle of starting the active service
+ */
+ private volatile boolean startingActiveService = false;
private INodeId inodeId;
@@ -473,6 +498,7 @@ public class FSNamesystem implements Nam
generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
leaseManager.removeAllLeases();
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
+ snapshotManager.clearSnapshottableDirs();
}
@VisibleForTesting
@@ -585,6 +611,9 @@ public class FSNamesystem implements Nam
*/
FSNamesystem(Configuration conf, FSImage fsImage, boolean ignoreRetryCache)
throws IOException {
+ boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
+ LOG.info("fsLock is fair:" + fair);
+ fsLock = new ReentrantReadWriteLock(fair);
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -874,6 +903,7 @@ public class FSNamesystem implements Nam
* @throws IOException
*/
void startActiveServices() throws IOException {
+ startingActiveService = true;
LOG.info("Starting services required for active state");
writeLock();
try {
@@ -928,8 +958,19 @@ public class FSNamesystem implements Nam
nnrmthread.start();
} finally {
writeUnlock();
+ startingActiveService = false;
}
}
+
+ /**
+ * @return Whether the namenode is transitioning to active state and is in the
+ * middle of the {@link #startActiveServices()}
+ */
+ public boolean inTransitionToActive() {
+ return haEnabled && haContext != null
+ && haContext.getState().getServiceState() == HAServiceState.ACTIVE
+ && startingActiveService;
+ }
private boolean shouldUseDelegationTokens() {
return UserGroupInformation.isSecurityEnabled() ||
@@ -1022,6 +1063,26 @@ public class FSNamesystem implements Nam
}
}
+ /**
+ * @throws RetriableException
+ * If 1) The NameNode is in SafeMode, 2) HA is enabled, and 3)
+ * NameNode is in active state
+ * @throws SafeModeException
+ * Otherwise if NameNode is in SafeMode.
+ */
+ private void checkNameNodeSafeMode(String errorMsg)
+ throws RetriableException, SafeModeException {
+ if (isInSafeMode()) {
+ SafeModeException se = new SafeModeException(errorMsg, safeMode);
+ if (haEnabled && haContext != null
+ && haContext.getState().getServiceState() == HAServiceState.ACTIVE) {
+ throw new RetriableException(se);
+ } else {
+ throw se;
+ }
+ }
+ }
+
public static Collection<URI> getNamespaceDirs(Configuration conf) {
return getStorageDirs(conf, DFS_NAMENODE_NAME_DIR_KEY);
}
@@ -1322,9 +1383,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot set permission for " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot set permission for " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
dir.setPermission(src, permission);
@@ -1361,9 +1420,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot set owner for " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot set owner for " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
checkOwner(pc, src);
if (!pc.isSuperUser()) {
@@ -1443,8 +1500,14 @@ public class FSNamesystem implements Nam
for (LocatedBlock b : ret.getLocatedBlocks()) {
// if safemode & no block locations yet then throw safemodeException
if ((b.getLocations() == null) || (b.getLocations().length == 0)) {
- throw new SafeModeException("Zero blocklocations for " + src,
- safeMode);
+ SafeModeException se = new SafeModeException(
+ "Zero blocklocations for " + src, safeMode);
+ if (haEnabled && haContext != null &&
+ haContext.getState().getServiceState() == HAServiceState.ACTIVE) {
+ throw new RetriableException(se);
+ } else {
+ throw se;
+ }
}
}
}
@@ -1585,9 +1648,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot concat " + target, safeMode);
- }
+ checkNameNodeSafeMode("Cannot concat " + target);
concatInternal(pc, target, srcs, logRetryCache);
resultingStat = getAuditFileInfo(target, false);
} finally {
@@ -1735,9 +1796,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot set times " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot set times " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
// Write access is required to set access and modification times
@@ -1764,16 +1823,16 @@ public class FSNamesystem implements Nam
void createSymlink(String target, String link,
PermissionStatus dirPerms, boolean createParent)
throws IOException, UnresolvedLinkException {
- CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
- if (cacheEntry != null && cacheEntry.isSuccess()) {
- return; // Return previous response
- }
if (!DFSUtil.isValidName(link)) {
throw new InvalidPathException("Invalid link name: " + link);
}
if (FSDirectory.isReservedName(target)) {
throw new InvalidPathException("Invalid target name: " + target);
}
+ CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+ if (cacheEntry != null && cacheEntry.isSuccess()) {
+ return; // Return previous response
+ }
boolean success = false;
try {
createSymlinkInt(target, link, dirPerms, createParent, cacheEntry != null);
@@ -1800,9 +1859,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot create symlink " + link, safeMode);
- }
+ checkNameNodeSafeMode("Cannot create symlink " + link);
link = FSDirectory.resolvePath(link, pathComponents, dir);
if (!createParent) {
verifyParentDir(link);
@@ -1860,9 +1917,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot set replication for " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot set replication for " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (isPermissionEnabled) {
checkPathAccess(pc, src, FsAction.WRITE);
@@ -1992,9 +2047,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot create file" + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot create file" + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
startFileInternal(pc, src, permissions, holder, clientMachine, create,
overwrite, createParent, replication, blockSize, logRetryCache);
@@ -2212,10 +2265,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot recover the lease of " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot recover the lease of " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
final INodeFile inode = INodeFile.valueOf(dir.getINode(src), src);
if (!inode.isUnderConstruction()) {
@@ -2366,9 +2416,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot append to file" + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot append to file" + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
lb = appendFileInternal(pc, src, holder, clientMachine, logRetryCache);
} catch (StandbyException se) {
@@ -2415,7 +2463,7 @@ public class FSNamesystem implements Nam
* client to "try again later".
*/
LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
- ExtendedBlock previous, HashMap<Node, Node> excludedNodes,
+ ExtendedBlock previous, Set<Node> excludedNodes,
List<String> favoredNodes)
throws LeaseExpiredException, NotReplicatedYetException,
QuotaExceededException, SafeModeException, UnresolvedLinkException,
@@ -2443,8 +2491,8 @@ public class FSNamesystem implements Nam
final INodeFileUnderConstruction pendingFile =
(INodeFileUnderConstruction) inodes[inodes.length - 1];
- if(onRetryBlock[0] != null) {
- // This is a retry. Just return the last block.
+ if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
+ // This is a retry. Just return the last block if having locations.
return onRetryBlock[0];
}
if (pendingFile.getBlocks().length >= maxBlocksPerFile) {
@@ -2481,9 +2529,18 @@ public class FSNamesystem implements Nam
final INodeFileUnderConstruction pendingFile =
(INodeFileUnderConstruction) inodes[inodes.length - 1];
- if(onRetryBlock[0] != null) {
- // This is a retry. Just return the last block.
- return onRetryBlock[0];
+ if (onRetryBlock[0] != null) {
+ if (onRetryBlock[0].getLocations().length > 0) {
+ // This is a retry. Just return the last block if having locations.
+ return onRetryBlock[0];
+ } else {
+ // add new chosen targets to already allocated block and return
+ BlockInfo lastBlockInFile = pendingFile.getLastBlock();
+ ((BlockInfoUnderConstruction) lastBlockInFile)
+ .setExpectedLocations(targets);
+ offset = pendingFile.computeFileSize();
+ return makeLocatedBlock(lastBlockInFile, targets, offset);
+ }
}
// commit the last block and complete it if it has minimum replicas
@@ -2518,9 +2575,7 @@ public class FSNamesystem implements Nam
checkBlock(previous);
onRetryBlock[0] = null;
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot add block to " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot add block to " + src);
// have we exceeded the configured limit of fs objects.
checkFsObjectLimit();
@@ -2614,7 +2669,7 @@ public class FSNamesystem implements Nam
/** @see NameNode#getAdditionalDatanode(String, ExtendedBlock, DatanodeInfo[], DatanodeInfo[], int, String) */
LocatedBlock getAdditionalDatanode(String src, final ExtendedBlock blk,
- final DatanodeInfo[] existings, final HashMap<Node, Node> excludes,
+ final DatanodeInfo[] existings, final Set<Node> excludes,
final int numAdditionalNodes, final String clientName
) throws IOException {
//check if the feature is enabled
@@ -2629,10 +2684,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.READ);
//check safe mode
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot add datanode; src=" + src
- + ", blk=" + blk, safeMode);
- }
+ checkNameNodeSafeMode("Cannot add datanode; src=" + src + ", blk=" + blk);
src = FSDirectory.resolvePath(src, pathComponents, dir);
//check lease
@@ -2677,10 +2729,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot abandon block " + b +
- " for fle" + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot abandon block " + b + " for fle" + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
//
@@ -2763,9 +2812,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot complete file " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot complete file " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
success = completeFileInternal(src, holder,
ExtendedBlock.getLocalBlock(last), fileId);
@@ -2941,9 +2988,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot rename " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot rename " + src);
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
checkOperation(OperationCategory.WRITE);
@@ -2993,10 +3038,6 @@ public class FSNamesystem implements Nam
/** Rename src to dst */
void renameTo(String src, String dst, Options.Rename... options)
throws IOException, UnresolvedLinkException {
- CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
- if (cacheEntry != null && cacheEntry.isSuccess()) {
- return; // Return previous response
- }
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: with options - "
+ src + " to " + dst);
@@ -3004,8 +3045,13 @@ public class FSNamesystem implements Nam
if (!DFSUtil.isValidName(dst)) {
throw new InvalidPathException("Invalid name: " + dst);
}
- FSPermissionChecker pc = getPermissionChecker();
+ final FSPermissionChecker pc = getPermissionChecker();
+
checkOperation(OperationCategory.WRITE);
+ CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+ if (cacheEntry != null && cacheEntry.isSuccess()) {
+ return; // Return previous response
+ }
byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
HdfsFileStatus resultingStat = null;
@@ -3013,9 +3059,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot rename " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot rename " + src);
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
renameToInternal(pc, src, dst, cacheEntry != null, options);
@@ -3113,7 +3157,7 @@ public class FSNamesystem implements Nam
throws AccessControlException, SafeModeException, UnresolvedLinkException,
IOException {
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
@@ -3121,9 +3165,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot delete " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot delete " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
if (!recursive && dir.isNonEmptyDirectory(src)) {
throw new IOException(src + " is non empty");
@@ -3167,21 +3209,17 @@ public class FSNamesystem implements Nam
* of blocks that need to be removed from blocksMap
*/
void removeBlocks(BlocksMapUpdateInfo blocks) {
- int start = 0;
- int end = 0;
List<Block> toDeleteList = blocks.getToDeleteList();
- while (start < toDeleteList.size()) {
- end = BLOCK_DELETION_INCREMENT + start;
- end = end > toDeleteList.size() ? toDeleteList.size() : end;
+ Iterator<Block> iter = toDeleteList.iterator();
+ while (iter.hasNext()) {
writeLock();
try {
- for (int i = start; i < end; i++) {
- blockManager.removeBlock(toDeleteList.get(i));
+ for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) {
+ blockManager.removeBlock(iter.next());
}
} finally {
writeUnlock();
}
- start = end;
}
}
@@ -3346,9 +3384,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot create directory " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot create directory " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
status = mkdirsInternal(pc, src, permissions, createParent);
if (status) {
@@ -3448,9 +3484,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot set quota on " + path, safeMode);
- }
+ checkNameNodeSafeMode("Cannot set quota on " + path);
dir.setQuota(path, nsQuota, dsQuota);
} finally {
writeUnlock();
@@ -3473,9 +3507,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot fsync file " + src, safeMode);
- }
+ checkNameNodeSafeMode("Cannot fsync file " + src);
src = FSDirectory.resolvePath(src, pathComponents, dir);
INodeFileUnderConstruction pendingFile = checkLease(src, clientName);
if (lastBlockLength > 0) {
@@ -3679,6 +3711,39 @@ public class FSNamesystem implements Nam
BlockInfo getStoredBlock(Block block) {
return blockManager.getStoredBlock(block);
}
+
+ @Override
+ public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
+ assert hasReadOrWriteLock();
+ final BlockCollection bc = blockUC.getBlockCollection();
+ if (bc == null || !(bc instanceof INodeFileUnderConstruction)) {
+ return false;
+ }
+
+ INodeFileUnderConstruction inodeUC = (INodeFileUnderConstruction) blockUC
+ .getBlockCollection();
+ String fullName = inodeUC.getName();
+ try {
+ if (fullName != null && fullName.startsWith(Path.SEPARATOR)
+ && dir.getINode(fullName) == inodeUC) {
+ // If file exists in normal path then no need to look in snapshot
+ return false;
+ }
+ } catch (UnresolvedLinkException e) {
+ LOG.error("Error while resolving the link : " + fullName, e);
+ return false;
+ }
+ /*
+ * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and
+ * bc is not in the current fsdirectory tree, bc must represent a snapshot
+ * file.
+ * 2. if fullName is not an absolute path, bc cannot be existent in the
+ * current fsdirectory tree.
+ * 3. if bc is not the current node associated with fullName, bc must be a
+ * snapshot inode.
+ */
+ return true;
+ }
void commitBlockSynchronization(ExtendedBlock lastblock,
long newgenerationstamp, long newlength,
@@ -3700,11 +3765,8 @@ public class FSNamesystem implements Nam
// If a DN tries to commit to the standby, the recovery will
// fail, and the next retry will succeed on the new NN.
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot commitBlockSynchronization while in safe mode",
- safeMode);
- }
+ checkNameNodeSafeMode(
+ "Cannot commitBlockSynchronization while in safe mode");
final BlockInfo storedBlock = getStoredBlock(
ExtendedBlock.getLocalBlock(lastblock));
if (storedBlock == null) {
@@ -3850,9 +3912,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot renew lease for " + holder, safeMode);
- }
+ checkNameNodeSafeMode("Cannot renew lease for " + holder);
leaseManager.renewLease(holder);
} finally {
writeUnlock();
@@ -3889,11 +3949,27 @@ public class FSNamesystem implements Nam
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ String startAfterString = new String(startAfter);
readLock();
try {
checkOperation(OperationCategory.READ);
src = FSDirectory.resolvePath(src, pathComponents, dir);
+ // Get file name when startAfter is an INodePath
+ if (FSDirectory.isReservedName(startAfterString)) {
+ byte[][] startAfterComponents = FSDirectory
+ .getPathComponentsForReservedPath(startAfterString);
+ try {
+ String tmp = FSDirectory.resolvePath(src, startAfterComponents, dir);
+ byte[][] regularPath = INode.getPathComponents(tmp);
+ startAfter = regularPath[regularPath.length - 1];
+ } catch (IOException e) {
+ // Possibly the inode is deleted
+ throw new DirectoryListingStartAfterNotFoundException(
+ "Can't find startAfter " + startAfterString);
+ }
+ }
+
if (isPermissionEnabled) {
if (dir.isDir(src)) {
checkPathAccess(pc, src, FsAction.READ_EXECUTE);
@@ -4175,6 +4251,14 @@ public class FSNamesystem implements Nam
return this.snapshotManager.getNumSnapshots();
}
+ @Override
+ public String getSnapshotStats() {
+ Map<String, Object> info = new HashMap<String, Object>();
+ info.put("SnapshottableDirectories", this.getNumSnapshottableDirs());
+ info.put("Snapshots", this.getNumSnapshots());
+ return JSON.toString(info);
+ }
+
int getNumberOfDatanodes(DatanodeReportType type) {
readLock();
try {
@@ -4214,19 +4298,20 @@ public class FSNamesystem implements Nam
* @throws IOException if
*/
void saveNamespace() throws AccessControlException, IOException {
+ checkOperation(OperationCategory.UNCHECKED);
+ checkSuperuserPrivilege();
+
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
- checkSuperuserPrivilege();
- checkOperation(OperationCategory.UNCHECKED);
boolean success = false;
readLock();
try {
checkOperation(OperationCategory.UNCHECKED);
if (!isInSafeMode()) {
- throw new IOException("Safe mode should be turned ON " +
- "in order to create namespace image.");
+ throw new IOException("Safe mode should be turned ON "
+ + "in order to create namespace image.");
}
getFSImage().saveNamespace(this);
success = true;
@@ -4303,7 +4388,7 @@ public class FSNamesystem implements Nam
* replicas, and calculates the ratio of safe blocks to the total number
* of blocks in the system, which is the size of blocks in
* {@link FSNamesystem#blockManager}. When the ratio reaches the
- * {@link #threshold} it starts the {@link SafeModeMonitor} daemon in order
+ * {@link #threshold} it starts the SafeModeMonitor daemon in order
* to monitor whether the safe mode {@link #extension} is passed.
* Then it leaves safe mode and destroys itself.
* <p>
@@ -4311,10 +4396,9 @@ public class FSNamesystem implements Nam
* not tracked because the name node is not intended to leave safe mode
* automatically in the case.
*
- * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction)
- * @see SafeModeMonitor
+ * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction, boolean)
*/
- class SafeModeInfo {
+ public class SafeModeInfo {
// configuration fields
/** Safe mode threshold condition %.*/
private double threshold;
@@ -4527,7 +4611,7 @@ public class FSNamesystem implements Nam
*/
private boolean needEnter() {
return (threshold != 0 && blockSafe < blockThreshold) ||
- (getNumLiveDataNodes() < datanodeThreshold) ||
+ (datanodeThreshold != 0 && getNumLiveDataNodes() < datanodeThreshold) ||
(!nameNodeHasResourcesAvailable());
}
@@ -5056,9 +5140,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.JOURNAL);
- if (isInSafeMode()) {
- throw new SafeModeException("Log not rolled", safeMode);
- }
+ checkNameNodeSafeMode("Log not rolled");
LOG.info("Roll Edit Log from " + Server.getRemoteAddress());
return getFSImage().rollEditLog();
} finally {
@@ -5079,9 +5161,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.CHECKPOINT);
- if (isInSafeMode()) {
- throw new SafeModeException("Checkpoint not started", safeMode);
- }
+ checkNameNodeSafeMode("Checkpoint not started");
LOG.info("Start checkpoint for " + backupNode.getAddress());
cmd = getFSImage().startCheckpoint(backupNode, activeNamenode);
getEditLog().logSync();
@@ -5105,19 +5185,17 @@ public class FSNamesystem implements Nam
void endCheckpoint(NamenodeRegistration registration,
CheckpointSignature sig) throws IOException {
+ checkOperation(OperationCategory.CHECKPOINT);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
- checkOperation(OperationCategory.CHECKPOINT);
boolean success = false;
readLock();
try {
checkOperation(OperationCategory.CHECKPOINT);
- if (isInSafeMode()) {
- throw new SafeModeException("Checkpoint not ended", safeMode);
- }
+ checkNameNodeSafeMode("Checkpoint not ended");
LOG.info("End checkpoint for " + registration.getAddress());
getFSImage().endCheckpoint(sig);
success = true;
@@ -5218,7 +5296,8 @@ public class FSNamesystem implements Nam
/**
* Get the total number of objects in the system.
*/
- long getMaxObjects() {
+ @Override // FSNamesystemMBean
+ public long getMaxObjects() {
return maxFsObjects;
}
@@ -5363,7 +5442,7 @@ public class FSNamesystem implements Nam
@Override // FSNamesystemMBean
public int getNumDecomDeadDataNodes() {
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
- getBlockManager().getDatanodeManager().fetchDatanodes(dead, null, true);
+ getBlockManager().getDatanodeManager().fetchDatanodes(null, dead, true);
int deadDecommissioned = 0;
for (DatanodeDescriptor node : dead) {
deadDecommissioned += node.isDecommissioned() ? 1 : 0;
@@ -5372,6 +5451,12 @@ public class FSNamesystem implements Nam
}
@Override // FSNamesystemMBean
+ public int getNumDecommissioningDataNodes() {
+ return getBlockManager().getDatanodeManager().getDecommissioningNodes()
+ .size();
+ }
+
+ @Override // FSNamesystemMBean
@Metric({"StaleDataNodes",
"Number of datanodes marked stale due to delayed heartbeat"})
public int getNumStaleDataNodes() {
@@ -5469,10 +5554,7 @@ public class FSNamesystem implements Nam
long nextGenerationStamp(boolean legacyBlock)
throws IOException, SafeModeException {
assert hasWriteLock();
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot get next generation stamp", safeMode);
- }
+ checkNameNodeSafeMode("Cannot get next generation stamp");
long gs;
if (legacyBlock) {
@@ -5525,12 +5607,9 @@ public class FSNamesystem implements Nam
/**
* Increments, logs and then returns the block ID
*/
- private long nextBlockId() throws SafeModeException {
+ private long nextBlockId() throws IOException {
assert hasWriteLock();
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot get next block ID", safeMode);
- }
+ checkNameNodeSafeMode("Cannot get next block ID");
final long blockId = blockIdGenerator.nextValue();
getEditLog().logAllocateBlockId(blockId);
// NB: callers sync the log
@@ -5540,10 +5619,8 @@ public class FSNamesystem implements Nam
private INodeFileUnderConstruction checkUCBlock(ExtendedBlock block,
String clientName) throws IOException {
assert hasWriteLock();
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot get a new generation stamp and an " +
- "access token for block " + block, safeMode);
- }
+ checkNameNodeSafeMode("Cannot get a new generation stamp and an "
+ + "access token for block " + block);
// check stored block state
BlockInfo storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
@@ -5641,11 +5718,11 @@ public class FSNamesystem implements Nam
void updatePipeline(String clientName, ExtendedBlock oldBlock,
ExtendedBlock newBlock, DatanodeID[] newNodes)
throws IOException {
+ checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
- checkOperation(OperationCategory.WRITE);
LOG.info("updatePipeline(block=" + oldBlock
+ ", newGenerationStamp=" + newBlock.getGenerationStamp()
+ ", newLength=" + newBlock.getNumBytes()
@@ -5656,9 +5733,7 @@ public class FSNamesystem implements Nam
boolean success = false;
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Pipeline not updated", safeMode);
- }
+ checkNameNodeSafeMode("Pipeline not updated");
assert newBlock.getBlockId()==oldBlock.getBlockId() : newBlock + " and "
+ oldBlock + " has different block identifier";
updatePipelineInternal(clientName, oldBlock, newBlock, newNodes,
@@ -5892,7 +5967,10 @@ public class FSNamesystem implements Nam
DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT),
conf.getLong(DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT),
- DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL, this);
+ DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL,
+ conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY,
+ DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT),
+ this);
}
/**
@@ -5915,9 +5993,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot issue delegation token", safeMode);
- }
+ checkNameNodeSafeMode("Cannot issue delegation token");
if (!isAllowedDelegationTokenOp()) {
throw new IOException(
"Delegation Token can be issued only with kerberos or web authentication");
@@ -5962,9 +6038,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot renew delegation token", safeMode);
- }
+ checkNameNodeSafeMode("Cannot renew delegation token");
if (!isAllowedDelegationTokenOp()) {
throw new IOException(
"Delegation Token can be renewed only with kerberos or web authentication");
@@ -5995,9 +6069,7 @@ public class FSNamesystem implements Nam
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot cancel delegation token", safeMode);
- }
+ checkNameNodeSafeMode("Cannot cancel delegation token");
String canceller = getRemoteUser().getUserName();
DelegationTokenIdentifier id = dtSecretManager
.cancelToken(token, canceller);
@@ -6223,14 +6295,25 @@ public class FSNamesystem implements Nam
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
blockManager.getDatanodeManager().fetchDatanodes(live, null, true);
for (DatanodeDescriptor node : live) {
- final Map<String, Object> innerinfo = new HashMap<String, Object>();
- innerinfo.put("lastContact", getLastContact(node));
- innerinfo.put("usedSpace", getDfsUsed(node));
- innerinfo.put("adminState", node.getAdminState().toString());
- innerinfo.put("nonDfsUsedSpace", node.getNonDfsUsed());
- innerinfo.put("capacity", node.getCapacity());
- innerinfo.put("numBlocks", node.numBlocks());
- innerinfo.put("version", node.getSoftwareVersion());
+ Map<String, Object> innerinfo = ImmutableMap.<String, Object>builder()
+ .put("infoAddr", node.getInfoAddr())
+ .put("infoSecureAddr", node.getInfoSecureAddr())
+ .put("xferaddr", node.getXferAddr())
+ .put("lastContact", getLastContact(node))
+ .put("usedSpace", getDfsUsed(node))
+ .put("adminState", node.getAdminState().toString())
+ .put("nonDfsUsedSpace", node.getNonDfsUsed())
+ .put("capacity", node.getCapacity())
+ .put("numBlocks", node.numBlocks())
+ .put("version", node.getSoftwareVersion())
+ .put("used", node.getDfsUsed())
+ .put("remaining", node.getRemaining())
+ .put("blockScheduled", node.getBlocksScheduled())
+ .put("blockPoolUsed", node.getBlockPoolUsed())
+ .put("blockPoolUsedPercent", node.getBlockPoolUsedPercent())
+ .put("volfails", node.getVolumeFailures())
+ .build();
+
info.put(node.getHostName(), innerinfo);
}
return JSON.toString(info);
@@ -6247,9 +6330,11 @@ public class FSNamesystem implements Nam
final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
blockManager.getDatanodeManager().fetchDatanodes(null, dead, true);
for (DatanodeDescriptor node : dead) {
- final Map<String, Object> innerinfo = new HashMap<String, Object>();
- innerinfo.put("lastContact", getLastContact(node));
- innerinfo.put("decommissioned", node.isDecommissioned());
+ Map<String, Object> innerinfo = ImmutableMap.<String, Object>builder()
+ .put("lastContact", getLastContact(node))
+ .put("decommissioned", node.isDecommissioned())
+ .put("xferaddr", node.getXferAddr())
+ .build();
info.put(node.getHostName(), innerinfo);
}
return JSON.toString(info);
@@ -6266,13 +6351,16 @@ public class FSNamesystem implements Nam
final List<DatanodeDescriptor> decomNodeList = blockManager.getDatanodeManager(
).getDecommissioningNodes();
for (DatanodeDescriptor node : decomNodeList) {
- final Map<String, Object> innerinfo = new HashMap<String, Object>();
- innerinfo.put("underReplicatedBlocks", node.decommissioningStatus
- .getUnderReplicatedBlocks());
- innerinfo.put("decommissionOnlyReplicas", node.decommissioningStatus
- .getDecommissionOnlyReplicas());
- innerinfo.put("underReplicateInOpenFiles", node.decommissioningStatus
- .getUnderReplicatedInOpenFiles());
+ Map<String, Object> innerinfo = ImmutableMap
+ .<String, Object> builder()
+ .put("xferaddr", node.getXferAddr())
+ .put("underReplicatedBlocks",
+ node.decommissioningStatus.getUnderReplicatedBlocks())
+ .put("decommissionOnlyReplicas",
+ node.decommissioningStatus.getDecommissionOnlyReplicas())
+ .put("underReplicateInOpenFiles",
+ node.decommissioningStatus.getUnderReplicatedInOpenFiles())
+ .build();
info.put(node.getHostName(), innerinfo);
}
return JSON.toString(info);
@@ -6462,11 +6550,17 @@ public class FSNamesystem implements Nam
* Verifies that the given identifier and password are valid and match.
* @param identifier Token identifier.
* @param password Password in the token.
- * @throws InvalidToken
*/
public synchronized void verifyToken(DelegationTokenIdentifier identifier,
- byte[] password) throws InvalidToken {
- getDelegationTokenSecretManager().verifyToken(identifier, password);
+ byte[] password) throws InvalidToken, RetriableException {
+ try {
+ getDelegationTokenSecretManager().verifyToken(identifier, password);
+ } catch (InvalidToken it) {
+ if (inTransitionToActive()) {
+ throw new RetriableException(it);
+ }
+ throw it;
+ }
}
@Override
@@ -6484,6 +6578,11 @@ public class FSNamesystem implements Nam
}
@VisibleForTesting
+ public void setEditLogTailerForTests(EditLogTailer tailer) {
+ this.editLogTailer = tailer;
+ }
+
+ @VisibleForTesting
void setFsLockForTests(ReentrantReadWriteLock lock) {
this.fsLock = lock;
}
@@ -6518,10 +6617,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot allow snapshot for " + path,
- safeMode);
- }
+ checkNameNodeSafeMode("Cannot allow snapshot for " + path);
checkSuperuserPrivilege();
dir.writeLock();
@@ -6546,10 +6642,7 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot disallow snapshot for " + path,
- safeMode);
- }
+ checkNameNodeSafeMode("Cannot disallow snapshot for " + path);
checkSuperuserPrivilege();
dir.writeLock();
@@ -6576,20 +6669,18 @@ public class FSNamesystem implements Nam
*/
String createSnapshot(String snapshotRoot, String snapshotName)
throws SafeModeException, IOException {
+ checkOperation(OperationCategory.WRITE);
+ final FSPermissionChecker pc = getPermissionChecker();
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return (String) cacheEntry.getPayload();
}
- final FSPermissionChecker pc = getPermissionChecker();
writeLock();
String snapshotPath = null;
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot create snapshot for "
- + snapshotRoot, safeMode);
- }
+ checkNameNodeSafeMode("Cannot create snapshot for " + snapshotRoot);
if (isPermissionEnabled) {
checkOwner(pc, snapshotRoot);
}
@@ -6628,19 +6719,17 @@ public class FSNamesystem implements Nam
*/
void renameSnapshot(String path, String snapshotOldName,
String snapshotNewName) throws SafeModeException, IOException {
+ checkOperation(OperationCategory.WRITE);
+ final FSPermissionChecker pc = getPermissionChecker();
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
}
- final FSPermissionChecker pc = getPermissionChecker();
writeLock();
boolean success = false;
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot rename snapshot for " + path,
- safeMode);
- }
+ checkNameNodeSafeMode("Cannot rename snapshot for " + path);
if (isPermissionEnabled) {
checkOwner(pc, path);
}
@@ -6673,10 +6762,10 @@ public class FSNamesystem implements Nam
public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
throws IOException {
SnapshottableDirectoryStatus[] status = null;
+ final FSPermissionChecker checker = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
- FSPermissionChecker checker = getPermissionChecker();
final String user = checker.isSuperUser()? null : checker.getUser();
status = snapshotManager.getSnapshottableDirListing(user);
} finally {
@@ -6744,7 +6833,9 @@ public class FSNamesystem implements Nam
*/
void deleteSnapshot(String snapshotRoot, String snapshotName)
throws SafeModeException, IOException {
+ checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = getPermissionChecker();
+
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -6753,14 +6844,13 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot delete snapshot for " + snapshotRoot, safeMode);
+ checkNameNodeSafeMode("Cannot delete snapshot for " + snapshotRoot);
+ if (isPermissionEnabled) {
+ checkOwner(pc, snapshotRoot);
}
- checkOwner(pc, snapshotRoot);
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ArrayList<INode>();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
dir.writeLock();
try {
snapshotManager.deleteSnapshot(snapshotRoot, snapshotName,
@@ -6802,17 +6892,22 @@ public class FSNamesystem implements Nam
* defined in the config file. It can also be explicitly listed in the
* config file.
*/
- private static class DefaultAuditLogger implements AuditLogger {
+ private static class DefaultAuditLogger extends HdfsAuditLogger {
+
+ private boolean logTokenTrackingId;
@Override
public void initialize(Configuration conf) {
- // Nothing to do.
+ logTokenTrackingId = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY,
+ DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT);
}
@Override
public void logAuditEvent(boolean succeeded, String userName,
InetAddress addr, String cmd, String src, String dst,
- FileStatus status) {
+ FileStatus status, UserGroupInformation ugi,
+ DelegationTokenSecretManager dtSecretManager) {
if (auditLog.isInfoEnabled()) {
final StringBuilder sb = auditBuffer.get();
sb.setLength(0);
@@ -6830,10 +6925,29 @@ public class FSNamesystem implements Nam
sb.append(status.getGroup()).append(":");
sb.append(status.getPermission());
}
- auditLog.info(sb);
+ if (logTokenTrackingId) {
+ sb.append("\t").append("trackingId=");
+ String trackingId = null;
+ if (ugi != null && dtSecretManager != null
+ && ugi.getAuthenticationMethod() == AuthenticationMethod.TOKEN) {
+ for (TokenIdentifier tid: ugi.getTokenIdentifiers()) {
+ if (tid instanceof DelegationTokenIdentifier) {
+ DelegationTokenIdentifier dtid =
+ (DelegationTokenIdentifier)tid;
+ trackingId = dtSecretManager.getTokenTrackingId(dtid);
+ break;
+ }
+ }
+ }
+ sb.append(trackingId);
+ }
+ logAuditMessage(sb.toString());
}
}
+ public void logAuditMessage(String message) {
+ auditLog.info(message);
+ }
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java Wed Oct 30 22:21:59 2013
@@ -57,9 +57,14 @@ public class FileChecksumServlets {
final String hostname = host instanceof DatanodeInfo
? ((DatanodeInfo)host).getHostName() : host.getIpAddr();
final String scheme = request.getScheme();
- final int port = "https".equals(scheme)
- ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY)
- : host.getInfoPort();
+ int port = host.getInfoPort();
+ if ("https".equals(scheme)) {
+ final Integer portObject = (Integer) getServletContext().getAttribute(
+ DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY);
+ if (portObject != null) {
+ port = portObject;
+ }
+ }
final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum");
String dtParam = "";
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java Wed Oct 30 22:21:59 2013
@@ -38,7 +38,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.util.ServletUtil;
/** Redirect queries about the hosted filesystem to an appropriate datanode.
- * @see org.apache.hadoop.hdfs.HftpFileSystem
+ * @see org.apache.hadoop.hdfs.web.HftpFileSystem
*/
@InterfaceAudience.Private
public class FileDataServlet extends DfsServlet {
@@ -61,9 +61,14 @@ public class FileDataServlet extends Dfs
} else {
hostname = host.getIpAddr();
}
- final int port = "https".equals(scheme)
- ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY)
- : host.getInfoPort();
+ int port = host.getInfoPort();
+ if ("https".equals(scheme)) {
+ final Integer portObject = (Integer) getServletContext().getAttribute(
+ DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY);
+ if (portObject != null) {
+ port = portObject;
+ }
+ }
String dtParam = "";
if (dt != null) {
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Wed Oct 30 22:21:59 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.util.StringUtils;
@@ -707,7 +708,7 @@ public abstract class INode implements I
}
public BlocksMapUpdateInfo() {
- toDeleteList = new ArrayList<Block>();
+ toDeleteList = new ChunkedArrayList<Block>();
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ListPathsServlet.java Wed Oct 30 22:21:59 2013
@@ -20,14 +20,13 @@ package org.apache.hadoop.hdfs.server.na
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HftpFileSystem;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.hdfs.web.HftpFileSystem;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.VersionInfo;
-
import org.znerd.xmlenc.*;
import java.io.IOException;
@@ -39,13 +38,14 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
import java.util.regex.Pattern;
+
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* Obtain meta-information about a filesystem.
- * @see org.apache.hadoop.hdfs.HftpFileSystem
+ * @see org.apache.hadoop.hdfs.web.HftpFileSystem
*/
@InterfaceAudience.Private
public class ListPathsServlet extends DfsServlet {
Modified: hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1537330&r1=1537329&r2=1537330&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Wed Oct 30 22:21:59 2013
@@ -272,10 +272,6 @@ public class NameNode implements NameNod
static NameNodeMetrics metrics;
private static final StartupProgress startupProgress = new StartupProgress();
- static {
- StartupProgressMetrics.register(startupProgress);
- }
-
/** Return the {@link FSNamesystem} object.
* @return {@link FSNamesystem} object.
*/
@@ -487,6 +483,7 @@ public class NameNode implements NameNod
loginAsNameNodeUser(conf);
NameNode.initMetrics(conf, this.getRole());
+ StartupProgressMetrics.register(startupProgress);
if (NamenodeRole.NAMENODE == role) {
startHttpServer(conf);