You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2012/04/06 02:34:53 UTC
svn commit: r1310141 - in
/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/protocol/
src/main/java/org/apache/hadoop/hdfs/protocolPB/ src/mai...
Author: suresh
Date: Fri Apr 6 00:34:51 2012
New Revision: 1310141
URL: http://svn.apache.org/viewvc?rev=1310141&view=rev
Log:
Merging changes from trunk to this HDFS-3092 branch
Modified:
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1310118-1310140
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Apr 6 00:34:51 2012
@@ -62,6 +62,9 @@ Trunk (unreleased changes)
HDFS-3178. Add states and state handler for journal synchronization in
JournalService. (szetszwo)
+ HDFS-3204. Minor modification to JournalProtocol.proto to make
+ it generic. (suresh)
+
OPTIMIZATIONS
HDFS-2834. Add a ByteBuffer-based read API to DFSInputStream.
@@ -427,6 +430,12 @@ Release 2.0.0 - UNRELEASED
HDFS-3109. Remove hsqldf exclusions from pom.xml. (Ravi Prakash
via suresh)
+ HDFS-3210. JsonUtil#toJsonMap for for a DatanodeInfo should use
+ "ipAddr" instead of "name". (eli)
+
+ HDFS-3208. Bogus entries in hosts files are incorrectly displayed
+ in the report. (eli)
+
BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1310118-1310140
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Fri Apr 6 00:34:51 2012
@@ -211,7 +211,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth";
public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;
public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address";
- public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:50010";
+ public static final int DFS_DATANODE_DEFAULT_PORT = 50010;
+ public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_DEFAULT_PORT;
public static final String DFS_DATANODE_DATA_DIR_PERMISSION_KEY = "dfs.datanode.data.dir.perm";
public static final String DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT = "700";
public static final String DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY = "dfs.datanode.directoryscan.interval";
@@ -227,7 +228,8 @@ public class DFSConfigKeys extends Commo
public static final String DFS_DATANODE_HANDLER_COUNT_KEY = "dfs.datanode.handler.count";
public static final int DFS_DATANODE_HANDLER_COUNT_DEFAULT = 3;
public static final String DFS_DATANODE_HTTP_ADDRESS_KEY = "dfs.datanode.http.address";
- public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50075";
+ public static final int DFS_DATANODE_HTTP_DEFAULT_PORT = 50075;
+ public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTP_DEFAULT_PORT;
public static final String DFS_DATANODE_MAX_RECEIVER_THREADS_KEY = "dfs.datanode.max.transfer.threads";
public static final int DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096;
public static final String DFS_DATANODE_NUMBLOCKS_KEY = "dfs.datanode.numblocks";
@@ -251,13 +253,15 @@ public class DFSConfigKeys extends Commo
public static final String DFS_HTTPS_ENABLE_KEY = "dfs.https.enable";
public static final boolean DFS_HTTPS_ENABLE_DEFAULT = false;
public static final String DFS_HTTPS_PORT_KEY = "dfs.https.port";
- public static final int DFS_HTTPS_PORT_DEFAULT = 50470;
public static final String DFS_DEFAULT_CHUNK_VIEW_SIZE_KEY = "dfs.default.chunk.view.size";
public static final int DFS_DEFAULT_CHUNK_VIEW_SIZE_DEFAULT = 32*1024;
public static final String DFS_DATANODE_HTTPS_ADDRESS_KEY = "dfs.datanode.https.address";
- public static final String DFS_DATANODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:50475";
+ public static final String DFS_DATANODE_HTTPS_PORT_KEY = "datanode.https.port";
+ public static final int DFS_DATANODE_HTTPS_DEFAULT_PORT = 50475;
+ public static final String DFS_DATANODE_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTPS_DEFAULT_PORT;
public static final String DFS_DATANODE_IPC_ADDRESS_KEY = "dfs.datanode.ipc.address";
- public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0:50020";
+ public static final int DFS_DATANODE_IPC_DEFAULT_PORT = 50020;
+ public static final String DFS_DATANODE_IPC_ADDRESS_DEFAULT = "0.0.0.0" + DFS_DATANODE_IPC_DEFAULT_PORT;
public static final String DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY = "dfs.block.access.token.enable";
public static final boolean DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT = false;
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java Fri Apr 6 00:34:51 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
@@ -50,14 +51,21 @@ public class DatanodeID implements Writa
protected int infoPort; // info server port
protected int ipcPort; // IPC server port
- /** Equivalent to DatanodeID(""). */
- public DatanodeID() {this("");}
+ public DatanodeID() {
+ this("", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT);
+ }
- /** Equivalent to DatanodeID(ipAddr, "", -1, -1, -1). */
- public DatanodeID(String ipAddr) {this(ipAddr, "", "", -1, -1, -1);}
+ public DatanodeID(String ipAddr, int xferPort) {
+ this(ipAddr, "", "", xferPort,
+ DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT);
+ }
- /** Equivalent to DatanodeID(ipAddr, "", xferPort, -1, -1). */
- public DatanodeID(String ipAddr, int xferPort) {this(ipAddr, "", "", xferPort, -1, -1);}
+ public DatanodeID(String ipAddr, String hostName, int xferPort) {
+ this(ipAddr, hostName, "", xferPort,
+ DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT);
+ }
/**
* DatanodeID copy constructor
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java Fri Apr 6 00:34:51 2012
@@ -122,12 +122,12 @@ public class DatanodeInfo extends Datano
}
/** Constructor */
- public DatanodeInfo(final String name, final String hostName,
+ public DatanodeInfo(final String ipAddr, final String hostName,
final String storageID, final int xferPort, final int infoPort, final int ipcPort,
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long lastUpdate, final int xceiverCount,
final String networkLocation, final AdminStates adminState) {
- super(name, hostName, storageID, xferPort, infoPort, ipcPort);
+ super(ipAddr, hostName, storageID, xferPort, infoPort, ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
this.remaining = remaining;
@@ -265,7 +265,7 @@ public class DatanodeInfo extends Datano
long c = getCapacity();
long r = getRemaining();
long u = getDfsUsed();
- buffer.append(ipAddr);
+ buffer.append(getName());
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
buffer.append(" "+location);
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java Fri Apr 6 00:34:51 2012
@@ -48,7 +48,7 @@ public class JournalProtocolServerSideTr
public JournalResponseProto journal(RpcController unused,
JournalRequestProto req) throws ServiceException {
try {
- impl.journal(PBHelper.convert(req.getRegistration()),
+ impl.journal(PBHelper.convert(req.getJournalInfo()),
req.getFirstTxnId(), req.getNumTxns(), req.getRecords()
.toByteArray());
} catch (IOException e) {
@@ -62,7 +62,7 @@ public class JournalProtocolServerSideTr
public StartLogSegmentResponseProto startLogSegment(RpcController controller,
StartLogSegmentRequestProto req) throws ServiceException {
try {
- impl.startLogSegment(PBHelper.convert(req.getRegistration()),
+ impl.startLogSegment(PBHelper.convert(req.getJournalInfo()),
req.getTxid());
} catch (IOException e) {
throw new ServiceException(e);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolTranslatorPB.java Fri Apr 6 00:34:51 2012
@@ -24,12 +24,10 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.StartLogSegmentRequestProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;
@@ -63,7 +61,7 @@ public class JournalProtocolTranslatorPB
public void journal(NamenodeRegistration reg, long firstTxnId,
int numTxns, byte[] records) throws IOException {
JournalRequestProto req = JournalRequestProto.newBuilder()
- .setRegistration(PBHelper.convert(reg))
+ .setJournalInfo(PBHelper.convertToJournalInfo(reg))
.setFirstTxnId(firstTxnId)
.setNumTxns(numTxns)
.setRecords(PBHelper.getByteString(records))
@@ -79,7 +77,7 @@ public class JournalProtocolTranslatorPB
public void startLogSegment(NamenodeRegistration registration, long txid)
throws IOException {
StartLogSegmentRequestProto req = StartLogSegmentRequestProto.newBuilder()
- .setRegistration(PBHelper.convert(registration))
+ .setJournalInfo(PBHelper.convertToJournalInfo(registration))
.setTxid(txid)
.build();
try {
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Fri Apr 6 00:34:51 2012
@@ -95,6 +95,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReplicaStateProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.UpgradeStatusReportProto;
+import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
@@ -127,7 +128,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
-import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
@@ -1346,4 +1346,26 @@ public class PBHelper {
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageID(r.getStorageID()).build();
}
+
+ public static NamenodeRegistration convert(JournalInfoProto info) {
+ int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0;
+ int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0;
+ StorageInfo storage = new StorageInfo(lv, nsID, info.getClusterID(), 0);
+
+ // Note that the role is always {@link NamenodeRole#NAMENODE} as this
+ // conversion happens for messages from Namenode to Journal receivers.
+ // Addresses in the registration are unused.
+ return new NamenodeRegistration("", "", storage, NamenodeRole.NAMENODE);
+ }
+
+ /**
+ * Method used for converting {@link JournalInfoProto} sent from Namenode
+ * to Journal receivers to {@link NamenodeRegistration}.
+ */
+ public static JournalInfoProto convertToJournalInfo(
+ NamenodeRegistration reg) {
+ return JournalInfoProto.newBuilder().setClusterID(reg.getClusterID())
+ .setLayoutVersion(reg.getLayoutVersion())
+ .setNamespaceID(reg.getNamespaceID()).build();
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Fri Apr 6 00:34:51 2012
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import java.io.DataInput;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
@@ -26,13 +24,10 @@ import java.util.List;
import java.util.Queue;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
/**
* This class extends the DatanodeInfo class with ephemeral information (eg
@@ -158,18 +153,18 @@ public class DatanodeDescriptor extends
*/
private boolean disallowed = false;
- /** Default constructor */
public DatanodeDescriptor() {}
- /** DatanodeDescriptor constructor
+ /**
+ * DatanodeDescriptor constructor
* @param nodeID id of the data node
*/
public DatanodeDescriptor(DatanodeID nodeID) {
this(nodeID, 0L, 0L, 0L, 0L, 0, 0);
}
- /** DatanodeDescriptor constructor
- *
+ /**
+ * DatanodeDescriptor constructor
* @param nodeID id of the data node
* @param networkLocation location of the data node in network
*/
@@ -178,8 +173,8 @@ public class DatanodeDescriptor extends
this(nodeID, networkLocation, 0L, 0L, 0L, 0L, 0, 0);
}
- /** DatanodeDescriptor constructor
- *
+ /**
+ * DatanodeDescriptor constructor
* @param nodeID id of the data node
* @param capacity capacity of the data node
* @param dfsUsed space used by the data node
@@ -199,8 +194,8 @@ public class DatanodeDescriptor extends
failedVolumes);
}
- /** DatanodeDescriptor constructor
- *
+ /**
+ * DatanodeDescriptor constructor
* @param nodeID id of the data node
* @param networkLocation location of the data node in network
* @param capacity capacity of the data node, including space used by non-dfs
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java Fri Apr 6 00:34:51 2012
@@ -71,6 +71,8 @@ import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.util.ReflectionUtils;
+import com.google.common.net.InetAddresses;
+
/**
* Manage datanodes, include decommission and other activities.
*/
@@ -353,14 +355,9 @@ public class DatanodeManager {
private void resolveNetworkLocation (DatanodeDescriptor node) {
List<String> names = new ArrayList<String>(1);
if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
- // get the node's IP address
names.add(node.getIpAddr());
} else {
- // get the node's host name
- String hostName = node.getHostName();
- int colon = hostName.indexOf(":");
- hostName = (colon==-1)?hostName:hostName.substring(0,colon);
- names.add(hostName);
+ names.add(node.getHostName());
}
// resolve its network location
@@ -771,6 +768,40 @@ public class DatanodeManager {
}
}
+ /**
+ * Parse a DatanodeID from a hosts file entry
+ * @param hostLine of form [hostname|ip][:port]?
+ * @return DatanodeID constructed from the given string
+ */
+ private DatanodeID parseDNFromHostsEntry(String hostLine) {
+ DatanodeID dnId;
+ String hostStr;
+ int port;
+ int idx = hostLine.indexOf(':');
+
+ if (-1 == idx) {
+ hostStr = hostLine;
+ port = DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT;
+ } else {
+ hostStr = hostLine.substring(0, idx);
+ port = Integer.valueOf(hostLine.substring(idx));
+ }
+
+ if (InetAddresses.isInetAddress(hostStr)) {
+ // The IP:port is sufficient for listing in a report
+ dnId = new DatanodeID(hostStr, "", port);
+ } else {
+ String ipAddr = "";
+ try {
+ ipAddr = InetAddress.getByName(hostStr).getHostAddress();
+ } catch (UnknownHostException e) {
+ LOG.warn("Invalid hostname " + hostStr + " in hosts file");
+ }
+ dnId = new DatanodeID(ipAddr, hostStr, port);
+ }
+ return dnId;
+ }
+
/** For generating datanode reports */
public List<DatanodeDescriptor> getDatanodeListForReport(
final DatanodeReportType type) {
@@ -782,7 +813,7 @@ public class DatanodeManager {
HashMap<String, String> mustList = new HashMap<String, String>();
if (listDeadNodes) {
- //first load all the nodes listed in include and exclude files.
+ // Put all nodes referenced in the hosts files in the map
Iterator<String> it = hostsReader.getHosts().iterator();
while (it.hasNext()) {
mustList.put(it.next(), "");
@@ -805,7 +836,7 @@ public class DatanodeManager {
if ( (isDead && listDeadNodes) || (!isDead && listLiveNodes) ) {
nodes.add(dn);
}
- //Remove any form of the this datanode in include/exclude lists.
+ // Remove any nodes we know about from the map
try {
InetAddress inet = InetAddress.getByName(dn.getIpAddr());
// compare hostname(:port)
@@ -814,7 +845,7 @@ public class DatanodeManager {
// compare ipaddress(:port)
mustList.remove(inet.getHostAddress().toString());
mustList.remove(inet.getHostAddress().toString()+ ":" +dn.getXferPort());
- } catch ( UnknownHostException e ) {
+ } catch (UnknownHostException e) {
mustList.remove(dn.getName());
mustList.remove(dn.getIpAddr());
LOG.warn(e);
@@ -825,9 +856,13 @@ public class DatanodeManager {
if (listDeadNodes) {
Iterator<String> it = mustList.keySet().iterator();
while (it.hasNext()) {
- DatanodeDescriptor dn =
- new DatanodeDescriptor(new DatanodeID(it.next()));
- dn.setLastUpdate(0);
+ // The remaining nodes are ones that are referenced by the hosts
+ // files but that we do not know about, ie that we have never
+ // head from. Eg. a host that is no longer part of the cluster
+ // or a bogus entry was given in the hosts files
+ DatanodeID dnId = parseDNFromHostsEntry(it.next());
+ DatanodeDescriptor dn = new DatanodeDescriptor(dnId);
+ dn.setLastUpdate(0); // Consider this node dead for reporting
nodes.add(dn);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Apr 6 00:34:51 2012
@@ -668,8 +668,7 @@ public class DataNode extends Configured
*/
DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) {
final String xferIp = streamingAddr.getAddress().getHostAddress();
- DatanodeRegistration bpRegistration = new DatanodeRegistration(xferIp);
- bpRegistration.setXferPort(getXferPort());
+ DatanodeRegistration bpRegistration = new DatanodeRegistration(xferIp, getXferPort());
bpRegistration.setInfoPort(getInfoPort());
bpRegistration.setIpcPort(getIpcPort());
bpRegistration.setHostName(hostName);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java Fri Apr 6 00:34:51 2012
@@ -331,8 +331,7 @@ public class BackupImage extends FSImage
*/
private synchronized void setState(BNState newState) {
if (LOG.isDebugEnabled()) {
- LOG.debug("State transition " + bnState + " -> " + newState,
- new Exception("trace"));
+ LOG.debug("State transition " + bnState + " -> " + newState);
}
bnState = newState;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java Fri Apr 6 00:34:51 2012
@@ -27,17 +27,15 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.NameNodeProxies;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalProtocolService;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
import org.apache.hadoop.hdfs.protocolPB.JournalProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
-import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
@@ -225,8 +223,6 @@ public class BackupNode extends NameNode
static class BackupNodeRpcServer extends NameNodeRpcServer implements
JournalProtocol {
- private final String nnRpcAddress;
-
private BackupNodeRpcServer(Configuration conf, BackupNode nn)
throws IOException {
super(conf, nn);
@@ -236,8 +232,31 @@ public class BackupNode extends NameNode
.newReflectiveBlockingService(journalProtocolTranslator);
DFSUtil.addPBProtocol(conf, JournalProtocolPB.class, service,
this.clientRpcServer);
- nnRpcAddress = nn.nnRpcAddress;
}
+
+ /**
+ * Verifies a journal request
+ * @param nodeReg node registration
+ * @throws UnregisteredNodeException if the registration is invalid
+ */
+ void verifyJournalRequest(NamenodeRegistration reg) throws IOException {
+ verifyVersion(reg.getLayoutVersion());
+ String errorMsg = null;
+ int expectedNamespaceID = namesystem.getNamespaceInfo().getNamespaceID();
+ if (reg.getNamespaceID() != expectedNamespaceID) {
+ errorMsg = "Invalid namespaceID in journal request - expected " + expectedNamespaceID
+ + " actual " + reg.getNamespaceID();
+ LOG.warn(errorMsg);
+ throw new UnregisteredNodeException(reg);
+ }
+ if (!reg.getClusterID().equals(namesystem.getClusterId())) {
+ errorMsg = "Invalid clusterId in journal request - expected "
+ + reg.getClusterID() + " actual " + namesystem.getClusterId();
+ LOG.warn(errorMsg);
+ throw new UnregisteredNodeException(reg);
+ }
+ }
+
/////////////////////////////////////////////////////
// BackupNodeProtocol implementation for backup node.
@@ -246,8 +265,7 @@ public class BackupNode extends NameNode
public void startLogSegment(NamenodeRegistration registration, long txid)
throws IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
- verifyRequest(registration);
-
+ verifyJournalRequest(registration);
getBNImage().namenodeStartedLogSegment(txid);
}
@@ -256,10 +274,7 @@ public class BackupNode extends NameNode
long firstTxId, int numTxns,
byte[] records) throws IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
- verifyRequest(nnReg);
- if(!nnRpcAddress.equals(nnReg.getAddress()))
- throw new IOException("Journal request from unexpected name-node: "
- + nnReg.getAddress() + " expecting " + nnRpcAddress);
+ verifyJournalRequest(nnReg);
getBNImage().journal(firstTxId, numTxns, records);
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java Fri Apr 6 00:34:51 2012
@@ -62,7 +62,7 @@ public class FileChecksumServlets {
? ((DatanodeInfo)host).getHostName() : host.getIpAddr();
final String scheme = request.getScheme();
final int port = "https".equals(scheme)
- ? (Integer)getServletContext().getAttribute("datanode.https.port")
+ ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY)
: host.getInfoPort();
final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum");
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileDataServlet.java Fri Apr 6 00:34:51 2012
@@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -62,7 +63,7 @@ public class FileDataServlet extends Dfs
hostname = host.getIpAddr();
}
final int port = "https".equals(scheme)
- ? (Integer)getServletContext().getAttribute("datanode.https.port")
+ ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY)
: host.getInfoPort();
String dtParam = "";
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java Fri Apr 6 00:34:51 2012
@@ -165,10 +165,11 @@ public class NameNodeHttpServer {
httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth,
useKrb);
// assume same ssl port for all datanodes
- InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(conf
- .get(DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475));
- httpServer.setAttribute("datanode.https.port", datanodeSslPort
- .getPort());
+ InetSocketAddress datanodeSslPort = NetUtils.createSocketAddr(
+ conf.get(DFS_DATANODE_HTTPS_ADDRESS_KEY,
+ infoHost + ":" + DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT));
+ httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY,
+ datanodeSslPort.getPort());
}
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
httpServer.setAttribute(NAMENODE_ADDRESS_ATTRIBUTE_KEY,
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java Fri Apr 6 00:34:51 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -52,18 +53,9 @@ implements Writable, NodeRegistration {
private StorageInfo storageInfo;
private ExportedBlockKeys exportedKeys;
- /**
- * Default constructor.
- */
public DatanodeRegistration() {
- this("");
- }
-
- /**
- * Create DatanodeRegistration
- */
- public DatanodeRegistration(String ipAddr) {
- this(ipAddr, new StorageInfo(), new ExportedBlockKeys());
+ this("", DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
+ new StorageInfo(), new ExportedBlockKeys());
}
public DatanodeRegistration(DatanodeID dn, StorageInfo info,
@@ -72,10 +64,14 @@ implements Writable, NodeRegistration {
this.storageInfo = info;
this.exportedKeys = keys;
}
-
- public DatanodeRegistration(String ipAddr, StorageInfo info,
+
+ public DatanodeRegistration(String ipAddr, int xferPort) {
+ this(ipAddr, xferPort, new StorageInfo(), new ExportedBlockKeys());
+ }
+
+ public DatanodeRegistration(String ipAddr, int xferPort, StorageInfo info,
ExportedBlockKeys keys) {
- super(ipAddr);
+ super(ipAddr, xferPort);
this.storageInfo = info;
this.exportedKeys = keys;
}
@@ -114,7 +110,7 @@ implements Writable, NodeRegistration {
@Override
public String toString() {
return getClass().getSimpleName()
- + "(" + ipAddr
+ + "(" + getIpAddr()
+ ", storageID=" + storageID
+ ", infoPort=" + infoPort
+ ", ipcPort=" + ipcPort
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java Fri Apr 6 00:34:51 2012
@@ -305,7 +305,7 @@ public class JsonUtil {
}
return new DatanodeInfo(
- (String)m.get("name"),
+ (String)m.get("ipAddr"),
(String)m.get("hostName"),
(String)m.get("storageID"),
(int)(long)(Long)m.get("xferPort"),
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1310118-1310140
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/JournalProtocol.proto Fri Apr 6 00:34:51 2012
@@ -27,16 +27,25 @@ option java_generate_equals_and_hash = t
import "hdfs.proto";
/**
- * registration - the registration info of the active NameNode
- * firstTxnId - the first txid in the rolled edit log
+ * Journal information used by the journal receiver to identify a journal.
+ */
+message JournalInfoProto {
+ required string clusterID = 1; // ID of the cluster
+ optional uint32 layoutVersion = 2; // Layout version
+ optional uint32 namespaceID = 3; // Namespace ID
+}
+
+/**
+ * JournalInfo - the information about the journal
+ * firstTxnId - the first txid in the journal records
* numTxns - Number of transactions in editlog
* records - bytes containing serialized journal records
*/
message JournalRequestProto {
- required NamenodeRegistrationProto registration = 1; // Registration info
- required uint64 firstTxnId = 2; // Transaction ID
- required uint32 numTxns = 3; // Transaction ID
- required bytes records = 4; // Journal record
+ required JournalInfoProto journalInfo = 1;
+ required uint64 firstTxnId = 2;
+ required uint32 numTxns = 3;
+ required bytes records = 4;
}
/**
@@ -46,12 +55,12 @@ message JournalResponseProto {
}
/**
- * registration - the registration info of the active NameNode
+ * JournalInfo - the information about the journal
* txid - first txid in the new log
*/
message StartLogSegmentRequestProto {
- required NamenodeRegistrationProto registration = 1; // Registration info
- required uint64 txid = 2; // Transaction ID
+ required JournalInfoProto journalInfo = 1;
+ required uint64 txid = 2;
}
/**
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1310118-1310140
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1310118-1310140
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1310118-1310140
Propchange: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1310118-1310140
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Fri Apr 6 00:34:51 2012
@@ -395,9 +395,9 @@ public class TestDFSUtil {
conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
String httpsport = DFSUtil.getInfoServer(null, conf, true);
- assertEquals("0.0.0.0:50470", httpsport);
+ assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTPS_PORT_DEFAULT, httpsport);
String httpport = DFSUtil.getInfoServer(null, conf, false);
- assertEquals("0.0.0.0:50070", httpport);
+ assertEquals("0.0.0.0:"+DFS_NAMENODE_HTTP_PORT_DEFAULT, httpport);
}
@Test
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java Fri Apr 6 00:34:51 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -516,7 +517,8 @@ public class TestDecommission {
// Now empty hosts file and ensure the datanode is disallowed
// from talking to namenode, resulting in it's shutdown.
ArrayList<String>list = new ArrayList<String>();
- list.add("invalidhost");
+ final String badHostname = "BOGUSHOST";
+ list.add(badHostname);
writeConfigFile(hostsFile, list);
for (int j = 0; j < numNameNodes; j++) {
@@ -530,6 +532,17 @@ public class TestDecommission {
info = client.datanodeReport(DatanodeReportType.LIVE);
}
assertEquals("Number of live nodes should be 0", 0, info.length);
+
+ // Test that non-live and bogus hostnames are considered "dead".
+ // The dead report should have an entry for (1) the DN that is
+ // now considered dead because it is no longer allowed to connect
+ // and (2) the bogus entry in the hosts file (these entries are
+ // always added last)
+ info = client.datanodeReport(DatanodeReportType.DEAD);
+ assertEquals("There should be 2 dead nodes", 2, info.length);
+ DatanodeID id = cluster.getDataNodes().get(0).getDatanodeId();
+ assertEquals(id.getHostName(), info[0].getHostName());
+ assertEquals(badHostname, info[1].getHostName());
}
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java Fri Apr 6 00:34:51 2012
@@ -60,7 +60,7 @@ public class TestReplaceDatanodeOnFailur
final DatanodeInfo[][] datanodes = new DatanodeInfo[infos.length + 1][];
datanodes[0] = new DatanodeInfo[0];
for(int i = 0; i < infos.length; ) {
- infos[i] = new DatanodeInfo(new DatanodeID("dn" + i));
+ infos[i] = new DatanodeInfo(new DatanodeID("dn" + i, 100));
i++;
datanodes[i] = new DatanodeInfo[i];
System.arraycopy(infos, 0, datanodes[i], 0, datanodes[i].length);
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java Fri Apr 6 00:34:51 2012
@@ -115,7 +115,7 @@ public class TestBPOfferService {
0, HdfsConstants.LAYOUT_VERSION))
.when(mock).versionRequest();
- Mockito.doReturn(new DatanodeRegistration("fake-node"))
+ Mockito.doReturn(new DatanodeRegistration("fake-node", 100))
.when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class));
Mockito.doAnswer(new HeartbeatAnswer(nnIdx))
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java Fri Apr 6 00:34:51 2012
@@ -779,9 +779,9 @@ public class NNThroughputBenchmark {
}
TinyDatanode(int dnIdx, int blockCapacity) throws IOException {
+ String ipAddr = DNS.getDefaultIP("default");
String hostName = DNS.getDefaultHost("default", "default");
- dnRegistration = new DatanodeRegistration(hostName);
- dnRegistration.setXferPort(getNodePort(dnIdx));
+ dnRegistration = new DatanodeRegistration(ipAddr, getNodePort(dnIdx));
dnRegistration.setHostName(hostName);
this.blocks = new ArrayList<Block>(blockCapacity);
this.nrBlocks = 0;
@@ -894,10 +894,10 @@ public class NNThroughputBenchmark {
for(int t = 0; t < blockTargets.length; t++) {
DatanodeInfo dnInfo = blockTargets[t];
DatanodeRegistration receivedDNReg;
- receivedDNReg = new DatanodeRegistration(dnInfo.getIpAddr());
+ receivedDNReg =
+ new DatanodeRegistration(dnInfo.getIpAddr(), dnInfo.getXferPort());
receivedDNReg.setStorageInfo(
- new DataStorage(nsInfo, dnInfo.getStorageID()));
- receivedDNReg.setXferPort(dnInfo.getXferPort());
+ new DataStorage(nsInfo, dnInfo.getStorageID()));
receivedDNReg.setInfoPort(dnInfo.getInfoPort());
receivedDNReg.setIpcPort(dnInfo.getIpcPort());
ReceivedDeletedBlockInfo[] rdBlocks = {
Modified: hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1310141&r1=1310140&r2=1310141&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Fri Apr 6 00:34:51 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.Arrays;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
@@ -133,8 +134,20 @@ public class TestWebHdfsFileSystemContra
final BlockLocation[] expected = cluster.getFileSystem().getFileBlockLocations(
new Path(f), 0L, 1L);
assertEquals(expected.length, computed.length);
- for(int i = 0; i < computed.length; i++) {
+ for (int i = 0; i < computed.length; i++) {
assertEquals(expected[i].toString(), computed[i].toString());
+ // Check names
+ String names1[] = expected[i].getNames();
+ String names2[] = computed[i].getNames();
+ Arrays.sort(names1);
+ Arrays.sort(names2);
+ Assert.assertArrayEquals("Names differ", names1, names2);
+ // Check topology
+ String topos1[] = expected[i].getTopologyPaths();
+ String topos2[] = computed[i].getTopologyPaths();
+ Arrays.sort(topos1);
+ Arrays.sort(topos2);
+ Assert.assertArrayEquals("Topology differs", topos1, topos2);
}
}