You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/10/16 23:07:38 UTC
svn commit: r1532910 [2/4] - in
/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/
hadoop-hdfs-nfs/src/main/java/org/apac...
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java Wed Oct 16 21:07:28 2013
@@ -17,15 +17,9 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Map;
-
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -41,11 +35,17 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HA_DT_SERVICE_PREFIX;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HA_DT_SERVICE_PREFIX;
public class HAUtil {
@@ -265,10 +265,15 @@ public class HAUtil {
tokenSelector.selectToken(haService, ugi.getTokens());
if (haToken != null) {
for (InetSocketAddress singleNNAddr : nnAddrs) {
+ // this is a minor hack to prevent physical HA tokens from being
+ // exposed to the user via UGI.getCredentials(), otherwise these
+ // cloned tokens may be inadvertently propagated to jobs
Token<DelegationTokenIdentifier> specificToken =
- new Token<DelegationTokenIdentifier>(haToken);
+ new Token.PrivateToken<DelegationTokenIdentifier>(haToken);
SecurityUtil.setTokenService(specificToken, singleNNAddr);
- ugi.addToken(specificToken);
+ Text alias =
+ new Text(HA_DT_SERVICE_PREFIX + "//" + specificToken.getService());
+ ugi.addToken(alias, specificToken);
LOG.debug("Mapped HA service delegation token for logical URI " +
haUri + " to namenode " + singleNNAddr);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Wed Oct 16 21:07:28 2013
@@ -94,7 +94,6 @@ public class HftpFileSystem extends File
private URI hftpURI;
protected URI nnUri;
- protected URI nnSecureUri;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
@@ -134,34 +133,33 @@ public class HftpFileSystem extends File
DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
}
- protected int getDefaultSecurePort() {
- return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY,
- DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT);
- }
-
+ /**
+ * We generate the address with one of the following ports, in
+ * order of preference.
+ * 1. Port from the hftp URI e.g. hftp://namenode:4000/ will return 4000.
+ * 2. Port configured via DFS_NAMENODE_HTTP_PORT_KEY
+ * 3. DFS_NAMENODE_HTTP_PORT_DEFAULT i.e. 50070.
+ *
+ * @param uri
+ * @return
+ */
protected InetSocketAddress getNamenodeAddr(URI uri) {
// use authority so user supplied uri can override port
return NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
}
- protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
- // must only use the host and the configured https port
- return NetUtils.createSocketAddrForHost(uri.getHost(), getDefaultSecurePort());
- }
-
protected URI getNamenodeUri(URI uri) {
- return DFSUtil.createUri("http", getNamenodeAddr(uri));
- }
-
- protected URI getNamenodeSecureUri(URI uri) {
- return DFSUtil.createUri("http", getNamenodeSecureAddr(uri));
+ return DFSUtil.createUri(getUnderlyingProtocol(), getNamenodeAddr(uri));
}
+ /**
+ * See the documentation of {@Link #getNamenodeAddr(URI)} for the logic
+ * behind selecting the canonical service name.
+ * @return
+ */
@Override
public String getCanonicalServiceName() {
- // unlike other filesystems, hftp's service is the secure port, not the
- // actual port in the uri
- return SecurityUtil.buildTokenService(nnSecureUri).toString();
+ return SecurityUtil.buildTokenService(nnUri).toString();
}
@Override
@@ -187,7 +185,6 @@ public class HftpFileSystem extends File
setConf(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.nnUri = getNamenodeUri(name);
- this.nnSecureUri = getNamenodeSecureUri(name);
try {
this.hftpURI = new URI(name.getScheme(), name.getAuthority(),
null, null, null);
@@ -225,7 +222,7 @@ public class HftpFileSystem extends File
protected Token<DelegationTokenIdentifier> selectDelegationToken(
UserGroupInformation ugi) {
- return hftpTokenSelector.selectToken(nnSecureUri, ugi.getTokens(), getConf());
+ return hftpTokenSelector.selectToken(nnUri, ugi.getTokens(), getConf());
}
@@ -234,6 +231,13 @@ public class HftpFileSystem extends File
return renewToken;
}
+ /**
+ * Return the underlying protocol that is used to talk to the namenode.
+ */
+ protected String getUnderlyingProtocol() {
+ return "http";
+ }
+
@Override
public synchronized <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
renewToken = token;
@@ -257,7 +261,7 @@ public class HftpFileSystem extends File
return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
@Override
public Token<?> run() throws IOException {
- final String nnHttpUrl = nnSecureUri.toString();
+ final String nnHttpUrl = nnUri.toString();
Credentials c;
try {
c = DelegationTokenFetcher.getDTfromRemote(nnHttpUrl, renewer);
@@ -301,7 +305,7 @@ public class HftpFileSystem extends File
* @throws IOException on error constructing the URL
*/
protected URL getNamenodeURL(String path, String query) throws IOException {
- final URL url = new URL("http", nnUri.getHost(),
+ final URL url = new URL(getUnderlyingProtocol(), nnUri.getHost(),
nnUri.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
@@ -703,17 +707,20 @@ public class HftpFileSystem extends File
return true;
}
+ protected String getUnderlyingProtocol() {
+ return "http";
+ }
+
@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token,
Configuration conf) throws IOException {
// update the kerberos credentials, if they are coming from a keytab
UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- // use http to renew the token
InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
return
DelegationTokenFetcher.renewDelegationToken
- (DFSUtil.createUri("http", serviceAddr).toString(),
+ (DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr).toString(),
(Token<DelegationTokenIdentifier>) token);
}
@@ -723,10 +730,9 @@ public class HftpFileSystem extends File
Configuration conf) throws IOException {
// update the kerberos credentials, if they are coming from a keytab
UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- // use http to cancel the token
InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
DelegationTokenFetcher.cancelDelegationToken
- (DFSUtil.createUri("http", serviceAddr).toString(),
+ (DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr).toString(),
(Token<DelegationTokenIdentifier>) token);
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java Wed Oct 16 21:07:28 2013
@@ -68,6 +68,14 @@ public class HsftpFileSystem extends Hft
return "hsftp";
}
+ /**
+ * Return the underlying protocol that is used to talk to the namenode.
+ */
+ @Override
+ protected String getUnderlyingProtocol() {
+ return "https";
+ }
+
@Override
public void initialize(URI name, Configuration conf) throws IOException {
super.initialize(name, conf);
@@ -134,24 +142,15 @@ public class HsftpFileSystem extends Hft
@Override
protected int getDefaultPort() {
- return getDefaultSecurePort();
+ return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT);
}
@Override
- protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
- return getNamenodeAddr(uri);
- }
-
- @Override
- protected URI getNamenodeUri(URI uri) {
- return getNamenodeSecureUri(uri);
- }
-
- @Override
protected HttpURLConnection openConnection(String path, String query)
throws IOException {
query = addDelegationTokenParam(query);
- final URL url = new URL("https", nnUri.getHost(),
+ final URL url = new URL(getUnderlyingProtocol(), nnUri.getHost(),
nnUri.getPort(), path + '?' + query);
HttpsURLConnection conn;
conn = (HttpsURLConnection)connectionFactory.openConnection(url);
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Wed Oct 16 21:07:28 2013
@@ -44,6 +44,7 @@ public class DatanodeID implements Compa
private String peerHostName; // hostname from the actual connection
private int xferPort; // data streaming port
private int infoPort; // info server port
+ private int infoSecurePort; // info server port
private int ipcPort; // IPC server port
// UUID identifying a given datanode. For upgraded Datanodes this is the
@@ -53,11 +54,12 @@ public class DatanodeID implements Compa
public DatanodeID(DatanodeID from) {
this(from.getIpAddr(),
- from.getHostName(),
- from.getDatanodeUuid(),
- from.getXferPort(),
- from.getInfoPort(),
- from.getIpcPort());
+ from.getHostName(),
+ from.getDatanodeUuid(),
+ from.getXferPort(),
+ from.getInfoPort(),
+ from.getInfoSecurePort(),
+ from.getIpcPort());
this.peerHostName = from.getPeerHostName();
}
@@ -74,12 +76,13 @@ public class DatanodeID implements Compa
* @param ipcPort ipc server port
*/
public DatanodeID(String ipAddr, String hostName, String datanodeUuid,
- int xferPort, int infoPort, int ipcPort) {
+ int xferPort, int infoPort, int infoSecurePort, int ipcPort) {
this.ipAddr = ipAddr;
this.hostName = hostName;
this.datanodeUuid = checkDatanodeUuid(datanodeUuid);
this.xferPort = xferPort;
this.infoPort = infoPort;
+ this.infoSecurePort = infoSecurePort;
this.ipcPort = ipcPort;
}
@@ -158,6 +161,13 @@ public class DatanodeID implements Compa
}
/**
+ * @return IP:infoPort string
+ */
+ public String getInfoSecureAddr() {
+ return ipAddr + ":" + infoSecurePort;
+ }
+
+ /**
* @return hostname:xferPort
*/
public String getXferAddrWithHostname() {
@@ -202,6 +212,13 @@ public class DatanodeID implements Compa
}
/**
+ * @return infoSecurePort (the port at which the HTTPS server bound to)
+ */
+ public int getInfoSecurePort() {
+ return infoSecurePort;
+ }
+
+ /**
* @return ipcPort (the port at which the IPC server bound to)
*/
public int getIpcPort() {
@@ -240,13 +257,14 @@ public class DatanodeID implements Compa
peerHostName = nodeReg.getPeerHostName();
xferPort = nodeReg.getXferPort();
infoPort = nodeReg.getInfoPort();
+ infoSecurePort = nodeReg.getInfoSecurePort();
ipcPort = nodeReg.getIpcPort();
}
/**
* Compare based on data transfer address.
*
- * @param that
+ * @param that datanode to compare with
* @return as specified by Comparable
*/
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Wed Oct 16 21:07:28 2013
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import static org.apache.hadoop.hdfs.DFSUtil.percent2String;
-
-import java.util.Date;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -32,6 +28,10 @@ import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+import java.util.Date;
+
+import static org.apache.hadoop.hdfs.DFSUtil.percent2String;
+
/**
* This class extends the primary identifier of a Datanode with ephemeral
* state, eg usage information, current administrative state, and the
@@ -108,18 +108,21 @@ public class DatanodeInfo extends Datano
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long lastUpdate, final int xceiverCount,
final AdminStates adminState) {
- this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(), nodeID.getXferPort(),
- nodeID.getInfoPort(), nodeID.getIpcPort(), capacity, dfsUsed, remaining,
- blockPoolUsed, lastUpdate, xceiverCount, location, adminState);
+ this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(),
+ nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(),
+ nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed,
+ lastUpdate, xceiverCount, location, adminState);
}
/** Constructor */
public DatanodeInfo(final String ipAddr, final String hostName,
- final String DatanodeUuid, final int xferPort, final int infoPort, final int ipcPort,
+ final String DatanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort,
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long lastUpdate, final int xceiverCount,
final String networkLocation, final AdminStates adminState) {
- super(ipAddr, hostName, DatanodeUuid, xferPort, infoPort, ipcPort);
+ super(ipAddr, hostName, DatanodeUuid, xferPort, infoPort,
+ infoSecurePort, ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
this.remaining = remaining;
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Wed Oct 16 21:07:28 2013
@@ -227,7 +227,8 @@ public class PBHelper {
// DatanodeId
public static DatanodeID convert(DatanodeIDProto dn) {
return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
- dn.getXferPort(), dn.getInfoPort(), dn.getIpcPort());
+ dn.getXferPort(), dn.getInfoPort(), dn.hasInfoSecurePort() ? dn
+ .getInfoSecurePort() : 0, dn.getIpcPort());
}
public static DatanodeIDProto convert(DatanodeID dn) {
@@ -240,6 +241,7 @@ public class PBHelper {
.setXferPort(dn.getXferPort())
.setDatanodeUuid(dn.getDatanodeUuid() != null ? dn.getDatanodeUuid() : "")
.setInfoPort(dn.getInfoPort())
+ .setInfoSecurePort(dn.getInfoSecurePort())
.setIpcPort(dn.getIpcPort()).build();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Wed Oct 16 21:07:28 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -115,6 +116,24 @@ public class DelegationTokenSecretManage
return super.retrievePassword(identifier);
}
+ @Override
+ public byte[] retriableRetrievePassword(DelegationTokenIdentifier identifier)
+ throws InvalidToken, StandbyException, RetriableException, IOException {
+ namesystem.checkOperation(OperationCategory.READ);
+ try {
+ return super.retrievePassword(identifier);
+ } catch (InvalidToken it) {
+ if (namesystem.inTransitionToActive()) {
+ // if the namesystem is currently in the middle of transition to
+ // active state, let client retry since the corresponding editlog may
+ // have not been applied yet
+ throw new RetriableException(it);
+ } else {
+ throw it;
+ }
+ }
+ }
+
/**
* Returns expiry time of a token given its identifier.
*
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Wed Oct 16 21:07:28 2013
@@ -79,6 +79,7 @@ import org.apache.hadoop.hdfs.util.Light
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.LightWeightGSet;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
@@ -94,9 +95,6 @@ public class BlockManager {
static final Log LOG = LogFactory.getLog(BlockManager.class);
public static final Log blockLog = NameNode.blockStateChangeLog;
- /** Default load factor of map */
- public static final float DEFAULT_MAP_LOAD_FACTOR = 0.75f;
-
private static final String QUEUE_REASON_CORRUPT_STATE =
"it has the wrong state or generation stamp";
@@ -248,7 +246,8 @@ public class BlockManager {
invalidateBlocks = new InvalidateBlocks(datanodeManager);
// Compute the map capacity by allocating 2% of total memory
- blocksMap = new BlocksMap(DEFAULT_MAP_LOAD_FACTOR);
+ blocksMap = new BlocksMap(
+ LightWeightGSet.computeCapacity(2.0, "BlocksMap"));
blockplacement = BlockPlacementPolicy.getInstance(
conf, stats, datanodeManager.getNetworkTopology());
pendingReplications = new PendingReplicationBlocks(conf.getInt(
@@ -1792,6 +1791,14 @@ public class BlockManager {
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent(
node.getStorageInfo(storageID), iblk, reportedState);
+ // OpenFileBlocks only inside snapshots also will be added to safemode
+ // threshold. So we need to update such blocks to safemode
+ // refer HDFS-5283
+ BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock;
+ if (namesystem.isInSnapshot(blockUC)) {
+ int numOfReplicas = blockUC.getNumExpectedLocations();
+ namesystem.incrementSafeBlockCount(numOfReplicas);
+ }
//and fall through to next clause
}
//add replica if appropriate
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java Wed Oct 16 21:07:28 2013
@@ -57,11 +57,11 @@ class BlocksMap {
/** Constant {@link LightWeightGSet} capacity. */
private final int capacity;
- private volatile GSet<Block, BlockInfo> blocks;
+ private GSet<Block, BlockInfo> blocks;
- BlocksMap(final float loadFactor) {
+ BlocksMap(int capacity) {
// Use 2% of total memory to size the GSet capacity
- this.capacity = LightWeightGSet.computeCapacity(2.0, "BlocksMap");
+ this.capacity = capacity;
this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Wed Oct 16 21:07:28 2013
@@ -17,21 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.apache.hadoop.util.Time.now;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -41,13 +29,8 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.namenode.HostFileManager;
import org.apache.hadoop.hdfs.server.namenode.HostFileManager.Entry;
@@ -55,33 +38,23 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.HostFileManager.MutableEntrySet;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
-import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
-import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
-import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
+import org.apache.hadoop.hdfs.server.protocol.*;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
-import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.util.CyclicIteration;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.CachedDNSToSwitchMapping;
-import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.net.*;
import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
-import org.apache.hadoop.net.Node;
-import org.apache.hadoop.net.NodeBase;
-import org.apache.hadoop.net.ScriptBasedMapping;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.net.InetAddresses;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+
+import static org.apache.hadoop.util.Time.now;
/**
* Manage datanodes, include decommission and other activities.
@@ -128,6 +101,8 @@ public class DatanodeManager {
private final int defaultInfoPort;
+ private final int defaultInfoSecurePort;
+
private final int defaultIpcPort;
/** Read include/exclude files*/
@@ -167,6 +142,7 @@ public class DatanodeManager {
*/
private boolean hasClusterEverBeenMultiRack = false;
+ private final boolean checkIpHostnameInRegistration;
/**
* The number of datanodes for each software version. This list should change
* during rolling upgrades.
@@ -189,7 +165,10 @@ public class DatanodeManager {
DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT)).getPort();
this.defaultInfoPort = NetUtils.createSocketAddr(
conf.get(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
- DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)).getPort();
+ DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT)).getPort();
+ this.defaultInfoSecurePort = NetUtils.createSocketAddr(
+ conf.get(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY,
+ DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)).getPort();
this.defaultIpcPort = NetUtils.createSocketAddr(
conf.get(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,
DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort();
@@ -232,6 +211,12 @@ public class DatanodeManager {
LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY
+ "=" + this.blockInvalidateLimit);
+ this.checkIpHostnameInRegistration = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT);
+ LOG.info(DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY
+ + "=" + checkIpHostnameInRegistration);
+
this.avoidStaleDataNodesForRead = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
@@ -757,11 +742,13 @@ public class DatanodeManager {
// Mostly called inside an RPC, update ip and peer hostname
String hostname = dnAddress.getHostName();
String ip = dnAddress.getHostAddress();
- if (!isNameResolved(dnAddress)) {
+ if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) {
// Reject registration of unresolved datanode to prevent performance
// impact of repetitive DNS lookups later.
- LOG.warn("Unresolved datanode registration from " + ip);
- throw new DisallowedDatanodeException(nodeReg);
+ final String message = "hostname cannot be resolved (ip="
+ + ip + ", hostname=" + hostname + ")";
+ LOG.warn("Unresolved datanode registration: " + message);
+ throw new DisallowedDatanodeException(nodeReg, message);
}
// update node registration with the ip and hostname from rpc request
nodeReg.setIpAddr(ip);
@@ -1131,6 +1118,7 @@ public class DatanodeManager {
// The IP:port is sufficient for listing in a report
dnId = new DatanodeID(hostStr, "", "", port,
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT);
} else {
String ipAddr = "";
@@ -1141,6 +1129,7 @@ public class DatanodeManager {
}
dnId = new DatanodeID(ipAddr, hostStr, "", port,
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT,
DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT);
}
return dnId;
@@ -1188,7 +1177,7 @@ public class DatanodeManager {
new DatanodeDescriptor(new DatanodeID(entry.getIpAddress(),
entry.getPrefix(), "",
entry.getPort() == 0 ? defaultXferPort : entry.getPort(),
- defaultInfoPort, defaultIpcPort));
+ defaultInfoPort, defaultInfoSecurePort, defaultIpcPort));
dn.setLastUpdate(0); // Consider this node dead for reporting
nodes.add(dn);
}
@@ -1207,17 +1196,17 @@ public class DatanodeManager {
/**
* Checks if name resolution was successful for the given address. If IP
* address and host name are the same, then it means name resolution has
- * failed. As a special case, the loopback address is also considered
+ * failed. As a special case, local addresses are also considered
* acceptable. This is particularly important on Windows, where 127.0.0.1 does
* not resolve to "localhost".
*
* @param address InetAddress to check
- * @return boolean true if name resolution successful or address is loopback
+ * @return boolean true if name resolution successful or address is local
*/
private static boolean isNameResolved(InetAddress address) {
String hostname = address.getHostName();
String ip = address.getHostAddress();
- return !hostname.equals(ip) || address.isLoopbackAddress();
+ return !hostname.equals(ip) || NetUtils.isLocalAddress(address);
}
private void setDatanodeDead(DatanodeDescriptor node) {
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Wed Oct 16 21:07:28 2013
@@ -18,24 +18,7 @@
package org.apache.hadoop.hdfs.server.common;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.URL;
-import java.net.URLEncoder;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TreeSet;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.jsp.JspWriter;
+import com.google.common.base.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,13 +30,9 @@ import org.apache.hadoop.hdfs.BlockReade
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
+import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
@@ -74,10 +53,22 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.VersionInfo;
-import com.google.common.base.Charsets;
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.jsp.JspWriter;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.*;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
@InterfaceAudience.Private
public class JspHelper {
@@ -112,7 +103,7 @@ public class JspHelper {
return super.hashCode();
}
}
-
+
// compare two records based on their frequency
private static class NodeRecordComparator implements Comparator<NodeRecord> {
@@ -126,6 +117,27 @@ public class JspHelper {
return 0;
}
}
+
+ /**
+ * A helper class that generates the correct URL for different schema.
+ *
+ */
+ public static final class Url {
+ public static String authority(String scheme, DatanodeID d) {
+ if (scheme.equals("http")) {
+ return d.getInfoAddr();
+ } else if (scheme.equals("https")) {
+ return d.getInfoSecureAddr();
+ } else {
+ throw new IllegalArgumentException("Unknown scheme:" + scheme);
+ }
+ }
+
+ public static String url(String scheme, DatanodeID d) {
+ return scheme + "://" + authority(scheme, d);
+ }
+ }
+
public static DatanodeInfo bestNode(LocatedBlocks blks, Configuration conf)
throws IOException {
HashMap<DatanodeInfo, NodeRecord> map =
@@ -217,7 +229,7 @@ public class JspHelper {
offsetIntoBlock, amtToRead, true,
"JspHelper", TcpPeerServer.peerFromSocketAndKey(s, encryptionKey),
new DatanodeID(addr.getAddress().getHostAddress(),
- addr.getHostName(), poolId, addr.getPort(), 0, 0), null,
+ addr.getHostName(), poolId, addr.getPort(), 0, 0, 0), null,
null, null, false, CachingStrategy.newDefaultStrategy());
final byte[] buf = new byte[amtToRead];
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Wed Oct 16 21:07:28 2013
@@ -18,56 +18,10 @@
package org.apache.hadoop.hdfs.server.datanode;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_STARTUP_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTPS_ENABLE_KEY;
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.SocketChannel;
-import java.security.PrivilegedExceptionAction;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.BlockingService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -84,37 +38,15 @@ import org.apache.hadoop.hdfs.HDFSPolicy
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.net.DomainPeerServer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
-import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
-import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
-import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor;
-import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
-import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
-import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
+import org.apache.hadoop.hdfs.protocol.*;
+import org.apache.hadoop.hdfs.protocol.datatransfer.*;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DNTransferAckProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.protocol.proto.InterDatanodeProtocolProtos.InterDatanodeProtocolService;
-import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolPB;
-import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolServerSideTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB;
-import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
+import org.apache.hadoop.hdfs.protocolPB.*;
+import org.apache.hadoop.hdfs.security.token.block.*;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
-import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -149,21 +81,21 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.util.*;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.JvmPauseMonitor;
-import org.apache.hadoop.util.ServicePlugin;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
import org.mortbay.util.ajax.JSON;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
+import java.io.*;
+import java.net.*;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+import java.security.PrivilegedExceptionAction;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.util.ExitUtil.terminate;
/**********************************************************
* DataNode is a class (and program) that stores a set of
@@ -245,6 +177,7 @@ public class DataNode extends Configured
private volatile boolean heartbeatsDisabledForTests = false;
private DataStorage storage = null;
private HttpServer infoServer = null;
+ private int infoSecurePort;
DataNodeMetrics metrics;
private InetSocketAddress streamingAddr;
@@ -359,16 +292,13 @@ public class DataNode extends Configured
InetSocketAddress infoSocAddr = DataNode.getInfoAddr(conf);
String infoHost = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
- this.infoServer = (secureResources == null)
- ? new HttpServer.Builder().setName("datanode")
- .setBindAddress(infoHost).setPort(tmpInfoPort)
- .setFindPort(tmpInfoPort == 0).setConf(conf)
- .setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))).build()
- : new HttpServer.Builder().setName("datanode")
- .setBindAddress(infoHost).setPort(tmpInfoPort)
- .setFindPort(tmpInfoPort == 0).setConf(conf)
- .setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
- .setConnector(secureResources.getListener()).build();
+ HttpServer.Builder builder = new HttpServer.Builder().setName("datanode")
+ .setBindAddress(infoHost).setPort(tmpInfoPort)
+ .setFindPort(tmpInfoPort == 0).setConf(conf)
+ .setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")));
+ this.infoServer = (secureResources == null) ? builder.build() :
+ builder.setConnector(secureResources.getListener()).build();
+
LOG.info("Opened info server at " + infoHost + ":" + tmpInfoPort);
if (conf.getBoolean(DFS_HTTPS_ENABLE_KEY, false)) {
boolean needClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
@@ -382,6 +312,7 @@ public class DataNode extends Configured
if(LOG.isDebugEnabled()) {
LOG.debug("Datanode listening for SSL on " + secInfoSocAddr);
}
+ infoSecurePort = secInfoSocAddr.getPort();
}
this.infoServer.addInternalServlet(null, "/streamFile/*", StreamFile.class);
this.infoServer.addInternalServlet(null, "/getFileChecksum/*",
@@ -750,7 +681,8 @@ public class DataNode extends Configured
}
DatanodeID dnId = new DatanodeID(
streamingAddr.getAddress().getHostAddress(), hostName,
- storage.getDatanodeUuid(), getXferPort(), getInfoPort(), getIpcPort());
+ storage.getDatanodeUuid(), getXferPort(), getInfoPort(),
+ infoSecurePort, getIpcPort());
return new DatanodeRegistration(dnId, storageInfo,
new ExportedBlockKeys(), VersionInfo.getVersion());
}
@@ -848,7 +780,7 @@ public class DataNode extends Configured
* If this is the first block pool to register, this also initializes
* the datanode-scoped storage.
*
- * @param bpos block pool to initialize and register with the NameNode.
+ * @param bpos Block pool offer service
* @throws IOException if the NN is inconsistent with the local storage.
*/
void initBlockPool(BPOfferService bpos) throws IOException {
@@ -2290,6 +2222,13 @@ public class DataNode extends Configured
}
/**
+ * @return the datanode's https port
+ */
+ public int getInfoSecurePort() {
+ return infoSecurePort;
+ }
+
+ /**
* Returned information is a JSON representation of a map with
* name node host name as the key and block pool Id as the value.
* Note that, if there are multiple NNs in an NA nameservice,
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java Wed Oct 16 21:07:28 2013
@@ -18,10 +18,9 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
@@ -37,9 +36,9 @@ import javax.servlet.jsp.JspWriter;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -47,20 +46,23 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer;
import org.apache.hadoop.http.HtmlQuoting;
-import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+
@InterfaceAudience.Private
public class DatanodeJspHelper {
+ private static final int PREV_BLOCK = -1;
+ private static final int NEXT_BLOCK = 1;
+
private static DFSClient getDFSClient(final UserGroupInformation user,
final String addr,
final Configuration conf
@@ -143,10 +145,10 @@ public class DatanodeJspHelper {
out.print("Empty file");
} else {
DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock, conf);
- String fqdn = canonicalize(chosenNode.getIpAddr());
int datanodePort = chosenNode.getXferPort();
- String redirectLocation = HttpConfig.getSchemePrefix() + fqdn + ":"
- + chosenNode.getInfoPort() + "/browseBlock.jsp?blockId="
+ String redirectLocation = JspHelper.Url.url(req.getScheme(),
+ chosenNode)
+ + "/browseBlock.jsp?blockId="
+ firstBlock.getBlock().getBlockId() + "&blockSize="
+ firstBlock.getBlock().getNumBytes() + "&genstamp="
+ firstBlock.getBlock().getGenerationStamp() + "&filename="
@@ -225,7 +227,7 @@ public class DatanodeJspHelper {
JspHelper.addTableFooter(out);
}
}
- out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ out.print("<br><a href=\"///"
+ canonicalize(nnAddr) + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -302,8 +304,7 @@ public class DatanodeJspHelper {
Long.MAX_VALUE).getLocatedBlocks();
// Add the various links for looking at the file contents
// URL for downloading the full file
- String downloadUrl = HttpConfig.getSchemePrefix() + req.getServerName() + ":"
- + req.getServerPort() + "/streamFile" + ServletUtil.encodePath(filename)
+ String downloadUrl = "/streamFile" + ServletUtil.encodePath(filename)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr, true)
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a name=\"viewOptions\"></a>");
@@ -319,8 +320,8 @@ public class DatanodeJspHelper {
dfs.close();
return;
}
- String fqdn = canonicalize(chosenNode.getIpAddr());
- String tailUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + chosenNode.getInfoPort()
+
+ String tailUrl = "///" + JspHelper.Url.authority(req.getScheme(), chosenNode)
+ "/tail.jsp?filename=" + URLEncoder.encode(filename, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
+ "&chunkSizeToView=" + chunkSizeToView
@@ -368,8 +369,7 @@ public class DatanodeJspHelper {
for (int j = 0; j < locs.length; j++) {
String datanodeAddr = locs[j].getXferAddr();
datanodePort = locs[j].getXferPort();
- fqdn = canonicalize(locs[j].getIpAddr());
- String blockUrl = HttpConfig.getSchemePrefix() + fqdn + ":" + locs[j].getInfoPort()
+ String blockUrl = "///" + JspHelper.Url.authority(req.getScheme(), locs[j])
+ "/browseBlock.jsp?blockId=" + blockidstring
+ "&blockSize=" + blockSize
+ "&filename=" + URLEncoder.encode(filename, "UTF-8")
@@ -380,7 +380,7 @@ public class DatanodeJspHelper {
+ JspHelper.getDelegationTokenUrlParam(tokenString)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
- String blockInfoUrl = HttpConfig.getSchemePrefix() + nnCanonicalName + ":"
+ String blockInfoUrl = "///" + nnCanonicalName + ":"
+ namenodeInfoPort
+ "/block_info_xml.jsp?blockId=" + blockidstring;
out.print("<td> </td><td><a href=\"" + blockUrl + "\">"
@@ -391,7 +391,7 @@ public class DatanodeJspHelper {
}
out.println("</table>");
out.print("<hr>");
- out.print("<br><a href=\"" + HttpConfig.getSchemePrefix()
+ out.print("<br><a href=\"///"
+ nnCanonicalName + ":"
+ namenodeInfoPort + "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
@@ -491,9 +491,7 @@ public class DatanodeJspHelper {
String parent = new File(filename).getParent();
JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, parent, nnAddr);
out.print("<hr>");
- out.print("<a href=\"" + HttpConfig.getSchemePrefix()
- + req.getServerName() + ":" + req.getServerPort()
- + "/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
+ out.print("<a href=\"/browseDirectory.jsp?dir=" + URLEncoder.encode(parent, "UTF-8")
+ "&namenodeInfoPort=" + namenodeInfoPort
+ JspHelper.getDelegationTokenUrlParam(tokenString)
+ JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
@@ -501,112 +499,23 @@ public class DatanodeJspHelper {
out.print("<a href=\"#viewOptions\">Advanced view/download options</a><br>");
out.print("<hr>");
- // Determine the prev & next blocks
- long nextStartOffset = 0;
- long nextBlockSize = 0;
- String nextBlockIdStr = null;
- String nextGenStamp = null;
- String nextHost = req.getServerName();
- int nextPort = req.getServerPort();
- int nextDatanodePort = datanodePort;
- // determine data for the next link
- if (startOffset + chunkSizeToView >= blockSize) {
- // we have to go to the next block from this point onwards
- List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
- Long.MAX_VALUE).getLocatedBlocks();
- for (int i = 0; i < blocks.size(); i++) {
- if (blocks.get(i).getBlock().getBlockId() == blockId) {
- if (i != blocks.size() - 1) {
- LocatedBlock nextBlock = blocks.get(i + 1);
- nextBlockIdStr = Long.toString(nextBlock.getBlock().getBlockId());
- nextGenStamp = Long.toString(nextBlock.getBlock()
- .getGenerationStamp());
- nextStartOffset = 0;
- nextBlockSize = nextBlock.getBlock().getNumBytes();
- DatanodeInfo d = JspHelper.bestNode(nextBlock, conf);
- nextDatanodePort = d.getXferPort();
- nextHost = d.getIpAddr();
- nextPort = d.getInfoPort();
- }
- }
- }
- } else {
- // we are in the same block
- nextBlockIdStr = blockId.toString();
- nextStartOffset = startOffset + chunkSizeToView;
- nextBlockSize = blockSize;
- nextGenStamp = genStamp.toString();
- }
- String nextUrl = null;
- if (nextBlockIdStr != null) {
- nextUrl = HttpConfig.getSchemePrefix() + canonicalize(nextHost) + ":" + nextPort
- + "/browseBlock.jsp?blockId=" + nextBlockIdStr
- + "&blockSize=" + nextBlockSize
- + "&startOffset=" + nextStartOffset
- + "&genstamp=" + nextGenStamp
- + "&filename=" + URLEncoder.encode(filename, "UTF-8")
- + "&chunkSizeToView=" + chunkSizeToView
- + "&datanodePort=" + nextDatanodePort
- + "&namenodeInfoPort=" + namenodeInfoPort
- + JspHelper.getDelegationTokenUrlParam(tokenString)
- + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ String authority = req.getServerName() + ":" + req.getServerPort();
+ String nextUrl = generateLinksForAdjacentBlock(NEXT_BLOCK, authority,
+ datanodePort, startOffset, chunkSizeToView, blockSize, blockId,
+ genStamp, dfs, filename, conf, req.getScheme(), tokenString,
+ namenodeInfoPort, nnAddr);
+ if (nextUrl != null) {
out.print("<a href=\"" + nextUrl + "\">View Next chunk</a> ");
}
- // determine data for the prev link
- String prevBlockIdStr = null;
- String prevGenStamp = null;
- long prevStartOffset = 0;
- long prevBlockSize = 0;
- String prevHost = req.getServerName();
- int prevPort = req.getServerPort();
- int prevDatanodePort = datanodePort;
- if (startOffset == 0) {
- List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
- Long.MAX_VALUE).getLocatedBlocks();
- for (int i = 0; i < blocks.size(); i++) {
- if (blocks.get(i).getBlock().getBlockId() == blockId) {
- if (i != 0) {
- LocatedBlock prevBlock = blocks.get(i - 1);
- prevBlockIdStr = Long.toString(prevBlock.getBlock().getBlockId());
- prevGenStamp = Long.toString(prevBlock.getBlock()
- .getGenerationStamp());
- prevStartOffset = prevBlock.getBlock().getNumBytes()
- - chunkSizeToView;
- if (prevStartOffset < 0)
- prevStartOffset = 0;
- prevBlockSize = prevBlock.getBlock().getNumBytes();
- DatanodeInfo d = JspHelper.bestNode(prevBlock, conf);
- prevDatanodePort = d.getXferPort();
- prevHost = d.getIpAddr();
- prevPort = d.getInfoPort();
- }
- }
- }
- } else {
- // we are in the same block
- prevBlockIdStr = blockId.toString();
- prevStartOffset = startOffset - chunkSizeToView;
- if (prevStartOffset < 0)
- prevStartOffset = 0;
- prevBlockSize = blockSize;
- prevGenStamp = genStamp.toString();
- }
- String prevUrl = null;
- if (prevBlockIdStr != null) {
- prevUrl = HttpConfig.getSchemePrefix() + canonicalize(prevHost) + ":" + prevPort
- + "/browseBlock.jsp?blockId=" + prevBlockIdStr
- + "&blockSize=" + prevBlockSize
- + "&startOffset=" + prevStartOffset
- + "&filename=" + URLEncoder.encode(filename, "UTF-8")
- + "&chunkSizeToView=" + chunkSizeToView
- + "&genstamp=" + prevGenStamp
- + "&datanodePort=" + prevDatanodePort
- + "&namenodeInfoPort=" + namenodeInfoPort
- + JspHelper.getDelegationTokenUrlParam(tokenString)
- + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ String prevUrl = generateLinksForAdjacentBlock(PREV_BLOCK, authority,
+ datanodePort, startOffset, chunkSizeToView, blockSize, blockId,
+ genStamp, dfs, filename, conf, req.getScheme(), tokenString,
+ namenodeInfoPort, nnAddr);
+ if (prevUrl != null) {
out.print("<a href=\"" + prevUrl + "\">View Prev chunk</a> ");
}
+
out.print("<hr>");
out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
try {
@@ -621,6 +530,71 @@ public class DatanodeJspHelper {
dfs.close();
}
+ private static String generateLinksForAdjacentBlock(final int direction,
+ String authority, int datanodePort, long startOffset,
+ int chunkSizeToView, long blockSize, long blockId, Long genStamp,
+ final DFSClient dfs, final String filename, final Configuration conf,
+ final String scheme, final String tokenString,
+ final int namenodeInfoPort, final String nnAddr)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+
+ boolean found = false;
+ if ((direction == NEXT_BLOCK && startOffset + chunkSizeToView < blockSize)
+ || (direction == PREV_BLOCK && startOffset != 0)) {
+ // we are in the same block
+ found = true;
+
+ if (direction == NEXT_BLOCK) {
+ startOffset = startOffset + chunkSizeToView;
+ } else {
+ startOffset = Math.max(0, startOffset - chunkSizeToView);
+ }
+ } else {
+ List<LocatedBlock> blocks = dfs.getNamenode().getBlockLocations(filename, 0,
+ Long.MAX_VALUE).getLocatedBlocks();
+
+ final long curBlockId = blockId;
+ int curBlockIdx = Iterables.indexOf(blocks, new Predicate<LocatedBlock>() {
+ @Override
+ public boolean apply(LocatedBlock b) {
+ return b.getBlock().getBlockId() == curBlockId;
+ }
+ });
+ found = curBlockIdx != -1 &&
+ ((direction == NEXT_BLOCK && curBlockIdx < blocks.size() - 1)
+ || (direction == PREV_BLOCK && curBlockIdx > 0));
+
+ if (found) {
+ LocatedBlock nextBlock = blocks.get(curBlockIdx + direction);
+
+ blockId = nextBlock.getBlock().getBlockId();
+ genStamp = nextBlock.getBlock().getGenerationStamp();
+ startOffset = 0;
+ blockSize = nextBlock.getBlock().getNumBytes();
+ DatanodeInfo d = JspHelper.bestNode(nextBlock, conf);
+ datanodePort = d.getXferPort();
+ authority = JspHelper.Url.authority(scheme, d);
+ }
+ }
+
+ if (found) {
+ return "///" + authority
+ + "/browseBlock.jsp?blockId=" + blockId
+ + "&blockSize=" + blockSize
+ + "&startOffset=" + startOffset
+ + "&genstamp=" + genStamp
+ + "&filename=" + URLEncoder.encode(filename, "UTF-8")
+ + "&chunkSizeToView=" + chunkSizeToView
+ + "&datanodePort=" + datanodePort
+ + "&namenodeInfoPort=" + namenodeInfoPort
+ + JspHelper.getDelegationTokenUrlParam(tokenString)
+ + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
+ } else {
+ return null;
+ }
+ }
+
static void generateFileChunksForTail(JspWriter out, HttpServletRequest req,
Configuration conf
) throws IOException,
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java?rev=1532910&r1=1532909&r2=1532910&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java Wed Oct 16 21:07:28 2013
@@ -827,7 +827,7 @@ class ClusterJspHelper {
doc.startTag("item");
doc.attribute("label", label);
doc.attribute("value", value);
- doc.attribute("link", HttpConfig.getSchemePrefix() + url);
+ doc.attribute("link", "///" + url);
doc.endTag(); // item
}
@@ -887,7 +887,16 @@ class ClusterJspHelper {
private static String queryMbean(String httpAddress, Configuration conf)
throws IOException {
- URL url = new URL(HttpConfig.getSchemePrefix() + httpAddress+JMX_QRY);
+ /**
+ * Although the other namenode might support HTTPS, it is fundamentally
+ * broken to get the JMX via an HTTPS connection inside the namenode,
+ * because in HTTPS set up the principal of the client and the one of
+ * the namenode differs. Therefore, there is no guarantees that the
+ * HTTPS connection can be set up.
+ *
+ * As a result, we just hard code the connection as an HTTP connection.
+ */
+ URL url = new URL("http://" + httpAddress + JMX_QRY);
return readOutput(url);
}
/**