You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by at...@apache.org on 2012/08/24 22:38:22 UTC
svn commit: r1377092 [2/4] - in
/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs...
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java Fri Aug 24 20:38:08 2012
@@ -33,8 +33,6 @@ import org.apache.hadoop.fs.UnresolvedLi
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.EnumSetWritable;
@@ -695,16 +693,6 @@ public interface ClientProtocol {
public void finalizeUpgrade() throws IOException;
/**
- * Report distributed upgrade progress or force current upgrade to proceed.
- *
- * @param action {@link HdfsConstants.UpgradeAction} to perform
- * @return upgrade status information or null if no upgrades are in progress
- * @throws IOException
- */
- public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action)
- throws IOException;
-
- /**
* @return CorruptFileBlocks, containing a list of corrupt files (with
* duplicates if there is more than one corrupt block in a file)
* and a cookie
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java Fri Aug 24 20:38:08 2012
@@ -60,7 +60,7 @@ public class HdfsConstants {
public static int MAX_PATH_LENGTH = 8000;
public static int MAX_PATH_DEPTH = 1000;
- // TODO mb@media-style.com: should be conf injected?
+ // TODO should be conf injected?
public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
public static final int IO_FILE_BUFFER_SIZE = new HdfsConfiguration().getInt(
DFSConfigKeys.IO_FILE_BUFFER_SIZE_KEY,
@@ -85,16 +85,6 @@ public class HdfsConstants {
public static final long INVALID_TXID = -12345;
/**
- * Distributed upgrade actions:
- *
- * 1. Get upgrade status. 2. Get detailed upgrade status. 3. Proceed with the
- * upgrade if it is stuck, no matter what the status is.
- */
- public static enum UpgradeAction {
- GET_STATUS, DETAILED_STATUS, FORCE_PROCEED;
- }
-
- /**
* URI Scheme for hdfs://namenode/ URIs.
*/
public static final String HDFS_URI_SCHEME = "hdfs";
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java Fri Aug 24 20:38:08 2012
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.util.ExactSizeInputStream;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.security.token.Token;
import com.google.common.collect.Lists;
@@ -155,6 +156,14 @@ public abstract class HdfsProtoUtil {
return ret;
}
+ public static DataChecksum.Type fromProto(HdfsProtos.ChecksumTypeProto type) {
+ return DataChecksum.Type.valueOf(type.name());
+ }
+
+ public static HdfsProtos.ChecksumTypeProto toProto(DataChecksum.Type type) {
+ return HdfsProtos.ChecksumTypeProto.valueOf(type.name());
+ }
+
public static InputStream vintPrefixed(final InputStream input)
throws IOException {
final int firstByte = input.read();
@@ -167,4 +176,4 @@ public abstract class HdfsProtoUtil {
return new ExactSizeInputStream(input, size);
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java Fri Aug 24 20:38:08 2012
@@ -31,9 +31,6 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
-
/**
* Static utilities for dealing with the protocol buffers used by the
@@ -42,19 +39,6 @@ import com.google.common.collect.Immutab
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class DataTransferProtoUtil {
-
- /**
- * Map between the internal DataChecksum identifiers and the protobuf-
- * generated identifiers on the wire.
- */
- static BiMap<Integer, ChecksumProto.ChecksumType> checksumTypeMap =
- ImmutableBiMap.<Integer, ChecksumProto.ChecksumType>builder()
- .put(DataChecksum.CHECKSUM_CRC32, ChecksumProto.ChecksumType.CRC32)
- .put(DataChecksum.CHECKSUM_CRC32C, ChecksumProto.ChecksumType.CRC32C)
- .put(DataChecksum.CHECKSUM_NULL, ChecksumProto.ChecksumType.NULL)
- .build();
-
-
static BlockConstructionStage fromProto(
OpWriteBlockProto.BlockConstructionStage stage) {
return BlockConstructionStage.valueOf(BlockConstructionStage.class,
@@ -68,7 +52,7 @@ public abstract class DataTransferProtoU
}
public static ChecksumProto toProto(DataChecksum checksum) {
- ChecksumType type = checksumTypeMap.get(checksum.getChecksumType());
+ ChecksumType type = ChecksumType.valueOf(checksum.getChecksumType().name());
if (type == null) {
throw new IllegalArgumentException(
"Can't convert checksum to protobuf: " + checksum);
@@ -84,7 +68,7 @@ public abstract class DataTransferProtoU
if (proto == null) return null;
int bytesPerChecksum = proto.getBytesPerChecksum();
- int type = checksumTypeMap.inverse().get(proto.getType());
+ DataChecksum.Type type = DataChecksum.Type.valueOf(proto.getType().name());
return DataChecksum.newDataChecksum(type, bytesPerChecksum);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -18,19 +18,31 @@
package org.apache.hadoop.hdfs.protocolPB;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenIdentifierProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import com.google.protobuf.ByteString;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -106,4 +118,38 @@ public class ClientDatanodeProtocolServe
.setLocalPath(resp.getBlockPath()).setLocalMetaPath(resp.getMetaPath())
.build();
}
+
+ @Override
+ public GetHdfsBlockLocationsResponseProto getHdfsBlockLocations(
+ RpcController controller, GetHdfsBlockLocationsRequestProto request)
+ throws ServiceException {
+ HdfsBlocksMetadata resp;
+ try {
+ // Construct the Lists to make the actual call
+ List<ExtendedBlock> blocks =
+ new ArrayList<ExtendedBlock>(request.getBlocksCount());
+ for (ExtendedBlockProto b : request.getBlocksList()) {
+ blocks.add(PBHelper.convert(b));
+ }
+ List<Token<BlockTokenIdentifier>> tokens =
+ new ArrayList<Token<BlockTokenIdentifier>>(request.getTokensCount());
+ for (BlockTokenIdentifierProto b : request.getTokensList()) {
+ tokens.add(PBHelper.convert(b));
+ }
+ // Call the real implementation
+ resp = impl.getHdfsBlocksMetadata(blocks, tokens);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ List<ByteString> volumeIdsByteStrings =
+ new ArrayList<ByteString>(resp.getVolumeIds().size());
+ for (byte[] b : resp.getVolumeIds()) {
+ volumeIdsByteStrings.add(ByteString.copyFrom(b));
+ }
+ // Build and return the response
+ Builder builder = GetHdfsBlockLocationsResponseProto.newBuilder();
+ builder.addAllVolumeIds(volumeIdsByteStrings);
+ builder.addAllVolumeIndexes(resp.getVolumeIndexes());
+ return builder.build();
+ }
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.protocolP
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
import javax.net.SocketFactory;
@@ -33,12 +35,17 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenIdentifierProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -50,6 +57,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import com.google.protobuf.ByteString;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -207,4 +215,44 @@ public class ClientDatanodeProtocolTrans
public Object getUnderlyingProxyObject() {
return rpcProxy;
}
-}
\ No newline at end of file
+
+ @Override
+ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks,
+ List<Token<BlockTokenIdentifier>> tokens) throws IOException {
+ // Convert to proto objects
+ List<ExtendedBlockProto> blocksProtos =
+ new ArrayList<ExtendedBlockProto>(blocks.size());
+ List<BlockTokenIdentifierProto> tokensProtos =
+ new ArrayList<BlockTokenIdentifierProto>(tokens.size());
+ for (ExtendedBlock b : blocks) {
+ blocksProtos.add(PBHelper.convert(b));
+ }
+ for (Token<BlockTokenIdentifier> t : tokens) {
+ tokensProtos.add(PBHelper.convert(t));
+ }
+ // Build the request
+ GetHdfsBlockLocationsRequestProto request =
+ GetHdfsBlockLocationsRequestProto.newBuilder()
+ .addAllBlocks(blocksProtos)
+ .addAllTokens(tokensProtos)
+ .build();
+ // Send the RPC
+ GetHdfsBlockLocationsResponseProto response;
+ try {
+ response = rpcProxy.getHdfsBlockLocations(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ // List of volumes in the response
+ List<ByteString> volumeIdsByteStrings = response.getVolumeIdsList();
+ List<byte[]> volumeIds = new ArrayList<byte[]>(volumeIdsByteStrings.size());
+ for (ByteString bs : volumeIdsByteStrings) {
+ volumeIds.add(bs.toByteArray());
+ }
+ // Array of indexes into the list of volumes, one per block
+ List<Integer> volumeIndexes = response.getVolumeIndexesList();
+ // Parsed HdfsVolumeId values, one per block
+ return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}),
+ volumeIds, volumeIndexes);
+ }
+}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -50,8 +50,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto;
@@ -130,7 +128,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.io.Text;
import com.google.protobuf.RpcController;
@@ -571,24 +568,6 @@ public class ClientNamenodeProtocolServe
}
@Override
- public DistributedUpgradeProgressResponseProto distributedUpgradeProgress(
- RpcController controller, DistributedUpgradeProgressRequestProto req)
- throws ServiceException {
- try {
- UpgradeStatusReport result = server.distributedUpgradeProgress(PBHelper
- .convert(req.getAction()));
- DistributedUpgradeProgressResponseProto.Builder builder =
- DistributedUpgradeProgressResponseProto.newBuilder();
- if (result != null) {
- builder.setReport(PBHelper.convert(result));
- }
- return builder.build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
public ListCorruptFileBlocksResponseProto listCorruptFileBlocks(
RpcController controller, ListCorruptFileBlocksRequestProto req)
throws ServiceException {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -58,8 +57,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetAdditionalDatanodeRequestProto;
@@ -102,7 +99,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.EnumSetWritable;
@@ -128,8 +124,7 @@ public class ClientNamenodeProtocolTrans
ProtocolMetaInterface, ClientProtocol, Closeable, ProtocolTranslator {
final private ClientNamenodeProtocolPB rpcProxy;
- public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy)
- throws IOException {
+ public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) {
rpcProxy = proxy;
}
@@ -565,21 +560,6 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action)
- throws IOException {
- DistributedUpgradeProgressRequestProto req =
- DistributedUpgradeProgressRequestProto.newBuilder().
- setAction(PBHelper.convert(action)).build();
- try {
- DistributedUpgradeProgressResponseProto res = rpcProxy
- .distributedUpgradeProgress(null, req);
- return res.hasReport() ? PBHelper.convert(res.getReport()) : null;
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
throws IOException {
ListCorruptFileBlocksRequestProto.Builder req =
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -41,8 +41,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ErrorReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlocksRequestProto;
@@ -59,7 +57,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
@@ -253,20 +250,6 @@ public class DatanodeProtocolClientSideT
}
@Override
- public UpgradeCommand processUpgradeCommand(UpgradeCommand comm)
- throws IOException {
- ProcessUpgradeRequestProto req = ProcessUpgradeRequestProto.newBuilder()
- .setCmd(PBHelper.convert(comm)).build();
- ProcessUpgradeResponseProto resp;
- try {
- resp = rpcProxy.processUpgrade(NULL_CONTROLLER, req);
- } catch (ServiceException se) {
- throw ProtobufHelper.getRemoteException(se);
- }
- return resp.hasCmd() ? PBHelper.convert(resp.getCmd()) : null;
- }
-
- @Override
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
ReportBadBlocksRequestProto.Builder builder = ReportBadBlocksRequestProto
.newBuilder();
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java Fri Aug 24 20:38:08 2012
@@ -33,8 +33,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ErrorReportResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeResponseProto;
@@ -56,7 +54,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -212,25 +209,6 @@ public class DatanodeProtocolServerSideT
}
@Override
- public ProcessUpgradeResponseProto processUpgrade(RpcController controller,
- ProcessUpgradeRequestProto request) throws ServiceException {
- UpgradeCommand ret;
- try {
- UpgradeCommand cmd = request.hasCmd() ? PBHelper
- .convert(request.getCmd()) : null;
- ret = impl.processUpgradeCommand(cmd);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- ProcessUpgradeResponseProto.Builder builder =
- ProcessUpgradeResponseProto.newBuilder();
- if (ret != null) {
- builder.setCmd(PBHelper.convert(ret));
- }
- return builder.build();
- }
-
- @Override
public ReportBadBlocksResponseProto reportBadBlocks(RpcController controller,
ReportBadBlocksRequestProto request) throws ServiceException {
List<LocatedBlockProto> lbps = request.getBlocksList();
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Fri Aug 24 20:38:08 2012
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -47,7 +46,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpgradeActionProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
@@ -61,7 +59,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.UpgradeCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DataEncryptionKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
@@ -70,6 +67,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ChecksumTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
@@ -95,7 +93,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReplicaStateProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.UpgradeStatusReportProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
@@ -105,7 +102,6 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
@@ -131,9 +127,9 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.security.token.Token;
import com.google.protobuf.ByteString;
@@ -390,8 +386,8 @@ public class PBHelper {
public static NamespaceInfo convert(NamespaceInfoProto info) {
StorageInfoProto storage = info.getStorageInfo();
return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(),
- info.getBlockPoolID(), storage.getCTime(), info.getDistUpgradeVersion(),
- info.getBuildVersion(), info.getSoftwareVersion());
+ info.getBlockPoolID(), storage.getCTime(), info.getBuildVersion(),
+ info.getSoftwareVersion());
}
public static NamenodeCommand convert(NamenodeCommandProto cmd) {
@@ -641,8 +637,6 @@ public class PBHelper {
return PBHelper.convert(proto.getKeyUpdateCmd());
case RegisterCommand:
return REG_CMD;
- case UpgradeCommand:
- return PBHelper.convert(proto.getUpgradeCmd());
}
return null;
}
@@ -739,11 +733,6 @@ public class PBHelper {
builder.setCmdType(DatanodeCommandProto.Type.BlockCommand).setBlkCmd(
PBHelper.convert((BlockCommand) datanodeCommand));
break;
- case DatanodeProtocol.DNA_UC_ACTION_REPORT_STATUS:
- case DatanodeProtocol.DNA_UC_ACTION_START_UPGRADE:
- builder.setCmdType(DatanodeCommandProto.Type.UpgradeCommand)
- .setUpgradeCmd(PBHelper.convert((UpgradeCommand) datanodeCommand));
- break;
case DatanodeProtocol.DNA_UNKNOWN: //Not expected
default:
builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand);
@@ -751,19 +740,6 @@ public class PBHelper {
return builder.build();
}
- public static UpgradeCommand convert(UpgradeCommandProto upgradeCmd) {
- int action = UpgradeCommand.UC_ACTION_UNKNOWN;
- switch (upgradeCmd.getAction()) {
- case REPORT_STATUS:
- action = UpgradeCommand.UC_ACTION_REPORT_STATUS;
- break;
- case START_UPGRADE:
- action = UpgradeCommand.UC_ACTION_START_UPGRADE;
- }
- return new UpgradeCommand(action, upgradeCmd.getVersion(),
- (short) upgradeCmd.getUpgradeStatus());
- }
-
public static KeyUpdateCommand convert(KeyUpdateCommandProto keyUpdateCmd) {
return new KeyUpdateCommand(PBHelper.convert(keyUpdateCmd.getKeys()));
}
@@ -853,28 +829,6 @@ public class PBHelper {
.build();
}
- public static UpgradeCommandProto convert(UpgradeCommand comm) {
- UpgradeCommandProto.Builder builder = UpgradeCommandProto.newBuilder();
- if (comm == null) {
- return builder.setAction(UpgradeCommandProto.Action.UNKNOWN)
- .setVersion(0).setUpgradeStatus(0).build();
- }
- builder.setVersion(comm.getVersion()).setUpgradeStatus(
- comm.getCurrentStatus());
- switch (comm.getAction()) {
- case UpgradeCommand.UC_ACTION_REPORT_STATUS:
- builder.setAction(UpgradeCommandProto.Action.REPORT_STATUS);
- break;
- case UpgradeCommand.UC_ACTION_START_UPGRADE:
- builder.setAction(UpgradeCommandProto.Action.START_UPGRADE);
- break;
- default:
- builder.setAction(UpgradeCommandProto.Action.UNKNOWN);
- break;
- }
- return builder.build();
- }
-
public static ReceivedDeletedBlockInfo convert(
ReceivedDeletedBlockInfoProto proto) {
ReceivedDeletedBlockInfo.BlockStatus status = null;
@@ -899,7 +853,7 @@ public class PBHelper {
return NamespaceInfoProto.newBuilder()
.setBlockPoolID(info.getBlockPoolID())
.setBuildVersion(info.getBuildVersion())
- .setDistUpgradeVersion(info.getDistributedUpgradeVersion())
+ .setUnused(0)
.setStorageInfo(PBHelper.convert((StorageInfo)info))
.setSoftwareVersion(info.getSoftwareVersion()).build();
}
@@ -1005,7 +959,9 @@ public class PBHelper {
fs.getBlockSize(), fs.getBytesPerChecksum(),
fs.getWritePacketSize(), (short) fs.getReplication(),
fs.getFileBufferSize(),
- fs.getEncryptDataTransfer());
+ fs.getEncryptDataTransfer(),
+ fs.getTrashInterval(),
+ DataChecksum.Type.valueOf(fs.getChecksumType().name()));
}
public static FsServerDefaultsProto convert(FsServerDefaults fs) {
@@ -1016,7 +972,10 @@ public class PBHelper {
setWritePacketSize(fs.getWritePacketSize())
.setReplication(fs.getReplication())
.setFileBufferSize(fs.getFileBufferSize())
- .setEncryptDataTransfer(fs.getEncryptDataTransfer()).build();
+ .setEncryptDataTransfer(fs.getEncryptDataTransfer())
+ .setTrashInterval(fs.getTrashInterval())
+ .setChecksumType(ChecksumTypeProto.valueOf(fs.getChecksumType().name()))
+ .build();
}
public static FsPermissionProto convert(FsPermission p) {
@@ -1234,51 +1193,6 @@ public class PBHelper {
}
}
- public static UpgradeActionProto convert(
- UpgradeAction a) {
- switch (a) {
- case GET_STATUS:
- return UpgradeActionProto.GET_STATUS;
- case DETAILED_STATUS:
- return UpgradeActionProto.DETAILED_STATUS;
- case FORCE_PROCEED:
- return UpgradeActionProto.FORCE_PROCEED;
- default:
- throw new IllegalArgumentException("Unexpected UpgradeAction :" + a);
- }
- }
-
-
- public static UpgradeAction convert(
- UpgradeActionProto a) {
- switch (a) {
- case GET_STATUS:
- return UpgradeAction.GET_STATUS;
- case DETAILED_STATUS:
- return UpgradeAction.DETAILED_STATUS;
- case FORCE_PROCEED:
- return UpgradeAction.FORCE_PROCEED;
- default:
- throw new IllegalArgumentException("Unexpected UpgradeAction :" + a);
- }
- }
-
- public static UpgradeStatusReportProto convert(UpgradeStatusReport r) {
- if (r == null)
- return null;
- return UpgradeStatusReportProto.newBuilder()
- .setVersion(r.getVersion())
- .setUpgradeStatus(r.getUpgradeStatus())
- .setFinalized(r.isFinalized())
- .build();
- }
-
- public static UpgradeStatusReport convert(UpgradeStatusReportProto r) {
- if (r == null) return null;
- return new UpgradeStatusReport(r.getVersion(),
- (short) r.getUpgradeStatus(), r.getFinalized());
- }
-
public static CorruptFileBlocks convert(CorruptFileBlocksProto c) {
if (c == null)
return null;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Fri Aug 24 20:38:08 2012
@@ -268,23 +268,8 @@ public class BlockManager {
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT);
this.shouldCheckForEnoughRacks = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) != null;
- this.blocksInvalidateWorkPct = conf.getFloat(
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
- Preconditions.checkArgument(
- (this.blocksInvalidateWorkPct > 0),
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
- " = '" + this.blocksInvalidateWorkPct + "' is invalid. " +
- "It should be a positive, non-zero float value " +
- "indicating a percentage.");
- this.blocksReplWorkMultiplier = conf.getInt(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
- Preconditions.checkArgument(
- (this.blocksReplWorkMultiplier > 0),
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
- " = '" + this.blocksReplWorkMultiplier + "' is invalid. " +
- "It should be a positive, non-zero integer value.");
+ this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
this.replicationRecheckInterval =
conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
@@ -373,10 +358,17 @@ public class BlockManager {
}
public void close() {
- if (pendingReplications != null) pendingReplications.stop();
- blocksMap.close();
- datanodeManager.close();
- if (replicationThread != null) replicationThread.interrupt();
+ try {
+ if (replicationThread != null) {
+ replicationThread.interrupt();
+ replicationThread.join(3000);
+ }
+ } catch (InterruptedException ie) {
+ } finally {
+ if (pendingReplications != null) pendingReplications.stop();
+ blocksMap.close();
+ datanodeManager.close();
+ }
}
/** @return the datanodeManager */
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java Fri Aug 24 20:38:08 2012
@@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -74,7 +73,6 @@ class BPOfferService {
*/
DatanodeRegistration bpRegistration;
- UpgradeManagerDatanode upgradeManager = null;
private final DataNode dn;
/**
@@ -260,33 +258,6 @@ class BPOfferService {
}
}
- synchronized UpgradeManagerDatanode getUpgradeManager() {
- if(upgradeManager == null)
- upgradeManager =
- new UpgradeManagerDatanode(dn, getBlockPoolId());
-
- return upgradeManager;
- }
-
- void processDistributedUpgradeCommand(UpgradeCommand comm)
- throws IOException {
- UpgradeManagerDatanode upgradeManager = getUpgradeManager();
- upgradeManager.processUpgradeCommand(comm);
- }
-
- /**
- * Start distributed upgrade if it should be initiated by the data-node.
- */
- synchronized void startDistributedUpgradeIfNeeded() throws IOException {
- UpgradeManagerDatanode um = getUpgradeManager();
-
- if(!um.getUpgradeState())
- return;
- um.setUpgradeState(false, um.getUpgradeVersion());
- um.startUpgrade();
- return;
- }
-
DataNode getDataNode() {
return dn;
}
@@ -374,9 +345,6 @@ class BPOfferService {
if (bpServices.isEmpty()) {
dn.shutdownBlockPool(this);
-
- if(upgradeManager != null)
- upgradeManager.shutdownUpgrade();
}
}
@@ -591,10 +559,6 @@ class BPOfferService {
dn.finalizeUpgradeForPool(bp);
break;
- case UpgradeCommand.UC_ACTION_START_UPGRADE:
- // start distributed upgrade here
- processDistributedUpgradeCommand((UpgradeCommand)cmd);
- break;
case DatanodeProtocol.DNA_RECOVERBLOCK:
String who = "NameNode at " + actor.getNNSocketAddress();
dn.recoverBlocks(who, ((BlockRecoveryCommand)cmd).getRecoveringBlocks());
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java Fri Aug 24 20:38:08 2012
@@ -324,7 +324,7 @@ class BPServiceActor implements Runnable
* Run an immediate block report on this thread. Used by tests.
*/
@VisibleForTesting
- void triggerBlockReportForTests() throws IOException {
+ void triggerBlockReportForTests() {
synchronized (pendingIncrementalBR) {
lastBlockReport = 0;
lastHeartbeat = 0;
@@ -340,7 +340,7 @@ class BPServiceActor implements Runnable
}
@VisibleForTesting
- void triggerHeartbeatForTests() throws IOException {
+ void triggerHeartbeatForTests() {
synchronized (pendingIncrementalBR) {
lastHeartbeat = 0;
pendingIncrementalBR.notifyAll();
@@ -355,7 +355,7 @@ class BPServiceActor implements Runnable
}
@VisibleForTesting
- void triggerDeletionReportForTests() throws IOException {
+ void triggerDeletionReportForTests() {
synchronized (pendingIncrementalBR) {
lastDeletedReport = 0;
pendingIncrementalBR.notifyAll();
@@ -670,7 +670,6 @@ class BPServiceActor implements Runnable
while (shouldRun()) {
try {
- bpos.startDistributedUpgradeIfNeeded();
offerService();
} catch (Exception ex) {
LOG.error("Exception in BPOfferService for " + this, ex);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java Fri Aug 24 20:38:08 2012
@@ -554,11 +554,9 @@ class BlockPoolSliceScanner {
}
private synchronized void startNewPeriod() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Starting a new period : work left in prev period : "
+ LOG.info("Starting a new period : work left in prev period : "
+ String.format("%.2f%%", totalBytesToScan == 0 ? 0
: (bytesLeft * 100.0) / totalBytesToScan));
- }
// reset the byte counts :
bytesLeft = totalBytesToScan;
@@ -566,7 +564,6 @@ class BlockPoolSliceScanner {
}
void scanBlockPoolSlice() {
- startNewPeriod();
// Create a new processedBlocks structure
processedBlocks = new HashMap<Long, Integer>();
if (!assignInitialVerificationTimes()) {
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java Fri Aug 24 20:38:08 2012
@@ -138,7 +138,7 @@ public class BlockPoolSliceStorage exten
// During startup some of them can upgrade or roll back
// while others could be up-to-date for the regular startup.
for (int idx = 0; idx < getNumStorageDirs(); idx++) {
- doTransition(datanode, getStorageDir(idx), nsInfo, startOpt);
+ doTransition(getStorageDir(idx), nsInfo, startOpt);
assert getLayoutVersion() == nsInfo.getLayoutVersion()
: "Data-node and name-node layout versions must be the same.";
assert getCTime() == nsInfo.getCTime()
@@ -232,7 +232,7 @@ public class BlockPoolSliceStorage exten
* @param startOpt startup option
* @throws IOException
*/
- private void doTransition(DataNode datanode, StorageDirectory sd,
+ private void doTransition(StorageDirectory sd,
NamespaceInfo nsInfo, StartupOption startOpt) throws IOException {
if (startOpt == StartupOption.ROLLBACK)
doRollback(sd, nsInfo); // rollback if applicable
@@ -254,13 +254,9 @@ public class BlockPoolSliceStorage exten
+ blockpoolID);
}
if (this.layoutVersion == HdfsConstants.LAYOUT_VERSION
- && this.cTime == nsInfo.getCTime())
+ && this.cTime == nsInfo.getCTime()) {
return; // regular startup
-
- // verify necessity of a distributed upgrade
- UpgradeManagerDatanode um =
- datanode.getUpgradeManagerDatanode(nsInfo.getBlockPoolID());
- verifyDistributedUpgradeProgress(um, nsInfo);
+ }
if (this.layoutVersion > HdfsConstants.LAYOUT_VERSION
|| this.cTime < nsInfo.getCTime()) {
doUpgrade(sd, nsInfo); // upgrade
@@ -476,13 +472,6 @@ public class BlockPoolSliceStorage exten
LOG.info( hardLink.linkStats.report() );
}
- private void verifyDistributedUpgradeProgress(UpgradeManagerDatanode um,
- NamespaceInfo nsInfo) throws IOException {
- assert um != null : "DataNode.upgradeManager is null.";
- um.setUpgradeState(false, getLayoutVersion());
- um.initializeUpgrade(nsInfo);
- }
-
/**
* gets the data node storage directory based on block pool storage
*
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java Fri Aug 24 20:38:08 2012
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.d
import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.SocketOutputStream;
@@ -236,8 +235,7 @@ class BlockSender implements java.io.Clo
} else {
LOG.warn("Could not find metadata file for " + block);
// This only decides the buffer size. Use BUFFER_SIZE?
- csum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL,
- 16 * 1024);
+ csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 16 * 1024);
}
/*
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java Fri Aug 24 20:38:08 2012
@@ -99,13 +99,8 @@ public class DataBlockScanner implements
}
// Wait for at least one block pool to be up
- private void waitForInit(String bpid) {
- UpgradeManagerDatanode um = null;
- if(bpid != null && !bpid.equals(""))
- um = datanode.getUpgradeManagerDatanode(bpid);
-
- while ((um != null && ! um.isUpgradeCompleted())
- || (getBlockPoolSetSize() < datanode.getAllBpOs().length)
+ private void waitForInit() {
+ while ((getBlockPoolSetSize() < datanode.getAllBpOs().length)
|| (getBlockPoolSetSize() < 1)) {
try {
Thread.sleep(5000);
@@ -129,7 +124,7 @@ public class DataBlockScanner implements
String nextBpId = null;
while ((nextBpId == null) && datanode.shouldRun
&& !blockScannerThread.isInterrupted()) {
- waitForInit(currentBpId);
+ waitForInit();
synchronized (this) {
if (getBlockPoolSetSize() > 0) {
// Find nextBpId by the minimum of the last scan time
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri Aug 24 20:38:08 2012
@@ -96,6 +96,7 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
@@ -279,6 +280,7 @@ public class DataNode extends Configured
private final String userWithLocalPathAccess;
private boolean connectToDnViaHostname;
ReadaheadPool readaheadPool;
+ private final boolean getHdfsBlockLocationsEnabled;
/**
* Create the DataNode given a configuration and an array of dataDirs.
@@ -303,6 +305,9 @@ public class DataNode extends Configured
this.connectToDnViaHostname = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
+ this.getHdfsBlockLocationsEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
+ DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
try {
hostName = getHostName(conf);
LOG.info("Configured hostname is " + hostName);
@@ -497,7 +502,7 @@ public class DataNode extends Configured
reason = "verifcation is not supported by SimulatedFSDataset";
}
if (reason == null) {
- directoryScanner = new DirectoryScanner(this, data, conf);
+ directoryScanner = new DirectoryScanner(data, conf);
directoryScanner.start();
} else {
LOG.info("Periodic Directory Tree Verification scan is disabled because " +
@@ -1033,6 +1038,25 @@ public class DataNode extends Configured
metrics.incrBlocksGetLocalPathInfo();
return info;
}
+
+ @Override
+ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks,
+ List<Token<BlockTokenIdentifier>> tokens) throws IOException,
+ UnsupportedOperationException {
+ if (!getHdfsBlockLocationsEnabled) {
+ throw new UnsupportedOperationException("Datanode#getHdfsBlocksMetadata "
+ + " is not enabled in datanode config");
+ }
+ if (blocks.size() != tokens.size()) {
+ throw new IOException("Differing number of blocks and tokens");
+ }
+ // Check access for each block
+ for (int i = 0; i < blocks.size(); i++) {
+ checkBlockToken(blocks.get(i), tokens.get(i),
+ BlockTokenSecretManager.AccessMode.READ);
+ }
+ return data.getHdfsBlocksMetadata(blocks);
+ }
private void checkBlockToken(ExtendedBlock block, Token<BlockTokenIdentifier> token,
AccessMode accessMode) throws IOException {
@@ -1194,17 +1218,8 @@ public class DataNode extends Configured
return xmitsInProgress.get();
}
- UpgradeManagerDatanode getUpgradeManagerDatanode(String bpid) {
- BPOfferService bpos = blockPoolManager.get(bpid);
- if(bpos==null) {
- return null;
- }
- return bpos.getUpgradeManager();
- }
-
- private void transferBlock( ExtendedBlock block,
- DatanodeInfo xferTargets[]
- ) throws IOException {
+ private void transferBlock(ExtendedBlock block, DatanodeInfo xferTargets[])
+ throws IOException {
BPOfferService bpos = getBPOSForBlock(block);
DatanodeRegistration bpReg = getDNRegistrationForBP(block.getBlockPoolId());
@@ -1842,8 +1857,7 @@ public class DataNode extends Configured
private void recoverBlock(RecoveringBlock rBlock) throws IOException {
ExtendedBlock block = rBlock.getBlock();
String blookPoolId = block.getBlockPoolId();
- DatanodeInfo[] targets = rBlock.getLocations();
- DatanodeID[] datanodeids = (DatanodeID[])targets;
+ DatanodeID[] datanodeids = rBlock.getLocations();
List<BlockRecord> syncList = new ArrayList<BlockRecord>(datanodeids.length);
int errorCount = 0;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Fri Aug 24 20:38:08 2012
@@ -396,10 +396,6 @@ public class DataStorage extends Storage
if (this.layoutVersion == HdfsConstants.LAYOUT_VERSION
&& this.cTime == nsInfo.getCTime())
return; // regular startup
- // verify necessity of a distributed upgrade
- UpgradeManagerDatanode um =
- datanode.getUpgradeManagerDatanode(nsInfo.getBlockPoolID());
- verifyDistributedUpgradeProgress(um, nsInfo);
// do upgrade
if (this.layoutVersion > HdfsConstants.LAYOUT_VERSION
@@ -708,14 +704,6 @@ public class DataStorage extends Storage
new File(to, otherNames[i]), oldLV, hl);
}
- private void verifyDistributedUpgradeProgress(UpgradeManagerDatanode um,
- NamespaceInfo nsInfo
- ) throws IOException {
- assert um != null : "DataNode.upgradeManager is null.";
- um.setUpgradeState(false, getLayoutVersion());
- um.initializeUpgrade(nsInfo);
- }
-
/**
* Add bpStorage into bpStorageMap
*/
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Fri Aug 24 20:38:08 2012
@@ -609,6 +609,7 @@ class DataXceiver extends Receiver imple
.setBytesPerCrc(bytesPerCRC)
.setCrcPerBlock(crcPerBlock)
.setMd5(ByteString.copyFrom(md5.getDigest()))
+ .setCrcType(HdfsProtoUtil.toProto(checksum.getChecksumType()))
)
.build()
.writeDelimitedTo(out);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java Fri Aug 24 20:38:08 2012
@@ -56,7 +56,6 @@ import org.apache.hadoop.util.Time;
public class DirectoryScanner implements Runnable {
private static final Log LOG = LogFactory.getLog(DirectoryScanner.class);
- private final DataNode datanode;
private final FsDatasetSpi<?> dataset;
private final ExecutorService reportCompileThreadPool;
private final ScheduledExecutorService masterThread;
@@ -222,8 +221,7 @@ public class DirectoryScanner implements
}
}
- DirectoryScanner(DataNode dn, FsDatasetSpi<?> dataset, Configuration conf) {
- this.datanode = dn;
+ DirectoryScanner(FsDatasetSpi<?> dataset, Configuration conf) {
this.dataset = dataset;
int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
@@ -271,17 +269,6 @@ public class DirectoryScanner implements
return;
}
- String[] bpids = dataset.getBlockPoolList();
- for(String bpid : bpids) {
- UpgradeManagerDatanode um =
- datanode.getUpgradeManagerDatanode(bpid);
- if (um != null && !um.isUpgradeCompleted()) {
- //If distributed upgrades underway, exit and wait for next cycle.
- LOG.warn("this cycle terminating immediately because Distributed Upgrade is in process");
- return;
- }
- }
-
//We're are okay to run - do it
reconcile();
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java Fri Aug 24 20:38:08 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataStorage;
import org.apache.hadoop.hdfs.server.datanode.Replica;
@@ -373,4 +374,16 @@ public interface FsDatasetSpi<V extends
*/
public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b
) throws IOException;
+
+ /**
+ * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in
+ * <code>blocks</code>.
+ *
+ * @param blocks List of blocks for which to return metadata
+ * @return metadata Metadata for the list of blocks
+ * @throws IOException
+ */
+ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
+ throws IOException;
+
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java Fri Aug 24 20:38:08 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -1667,6 +1668,43 @@ class FsDatasetImpl implements FsDataset
datafile.getAbsolutePath(), metafile.getAbsolutePath());
return info;
}
+
+ @Override // FsDatasetSpi
+ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
+ throws IOException {
+ // List of VolumeIds, one per volume on the datanode
+ List<byte[]> blocksVolumeIds = new ArrayList<byte[]>(volumes.volumes.size());
+ // List of indexes into the list of VolumeIds, pointing at the VolumeId of
+ // the volume that the block is on
+ List<Integer> blocksVolumendexes = new ArrayList<Integer>(blocks.size());
+ // Initialize the list of VolumeIds simply by enumerating the volumes
+ for (int i = 0; i < volumes.volumes.size(); i++) {
+ blocksVolumeIds.add(new byte[] { (byte) i });
+ }
+ // Determine the index of the VolumeId of each block's volume, by comparing
+ // the block's volume against the enumerated volumes
+ for (int i = 0; i < blocks.size(); i++) {
+ ExtendedBlock block = blocks.get(i);
+ FsVolumeSpi blockVolume = getReplicaInfo(block).getVolume();
+ boolean isValid = false;
+ int volumeIndex = 0;
+ for (FsVolumeImpl volume : volumes.volumes) {
+ // This comparison of references should be safe
+ if (blockVolume == volume) {
+ isValid = true;
+ break;
+ }
+ volumeIndex++;
+ }
+ // Indicates that the block is not present, or not found in a data dir
+ if (!isValid) {
+ volumeIndex = Integer.MAX_VALUE;
+ }
+ blocksVolumendexes.add(volumeIndex);
+ }
+ return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}),
+ blocksVolumeIds, blocksVolumendexes);
+ }
@Override
public RollingLogs createRollingLogs(String bpid, String prefix
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Fri Aug 24 20:38:08 2012
@@ -215,7 +215,7 @@ public class DatanodeWebHdfsMethods {
fullpath, permission.getFsPermission(),
overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
: EnumSet.of(CreateFlag.CREATE),
- replication.getValue(conf), blockSize.getValue(conf), null, b), null);
+ replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
IOUtils.copyBytes(in, out, b);
out.close();
out = null;
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Aug 24 20:38:08 2012
@@ -83,7 +83,7 @@ import com.google.common.collect.Lists;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class FSEditLog {
+public class FSEditLog implements LogsPurgeable {
static final Log LOG = LogFactory.getLog(FSEditLog.class);
@@ -1032,6 +1032,7 @@ public class FSEditLog {
/**
* Archive any log files that are older than the given txid.
*/
+ @Override
public synchronized void purgeLogsOlderThan(final long minTxIdToKeep) {
assert curSegmentTxId == HdfsConstants.INVALID_TXID || // on format this is no-op
minTxIdToKeep <= curSegmentTxId :
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Fri Aug 24 20:38:08 2012
@@ -91,13 +91,11 @@ public class FSImage implements Closeabl
final private Configuration conf;
- private final NNStorageRetentionManager archivalManager;
+ protected NNStorageRetentionManager archivalManager;
/**
* Construct an FSImage
* @param conf Configuration
- * @see #FSImage(Configuration conf,
- * Collection imageDirs, Collection editsDirs)
* @throws IOException if default directories are invalid.
*/
public FSImage(Configuration conf) throws IOException {
@@ -191,8 +189,6 @@ public class FSImage implements Closeabl
throw new IOException(
"All specified directories are not accessible or do not exist.");
- storage.setUpgradeManager(target.upgradeManager);
-
// 1. For each data directory calculate its state and
// check whether all is consistent before transitioning.
Map<StorageDirectory, StorageState> dataDirStates =
@@ -227,9 +223,6 @@ public class FSImage implements Closeabl
storage.processStartupOptionsForUpgrade(startOpt, layoutVersion);
- // check whether distributed upgrade is required and/or should be continued
- storage.verifyDistributedUpgradeProgress(startOpt);
-
// 2. Format unformatted dirs.
for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
StorageDirectory sd = it.next();
@@ -320,13 +313,6 @@ public class FSImage implements Closeabl
}
private void doUpgrade(FSNamesystem target) throws IOException {
- if(storage.getDistributedUpgradeState()) {
- // only distributed upgrade need to continue
- // don't do version upgrade
- this.loadFSImage(target, null);
- storage.initializeDistributedUpgrade();
- return;
- }
// Upgrade is allowed only if there are
// no previous fs states in any of the directories
for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
@@ -409,7 +395,6 @@ public class FSImage implements Closeabl
+ storage.getRemovedStorageDirs().size()
+ " storage directory(ies), previously logged.");
}
- storage.initializeDistributedUpgrade();
}
private void doRollback() throws IOException {
@@ -472,8 +457,6 @@ public class FSImage implements Closeabl
LOG.info("Rollback of " + sd.getRoot()+ " is complete.");
}
isUpgradeFinalized = true;
- // check whether name-node can start in regular mode
- storage.verifyDistributedUpgradeProgress(StartupOption.REGULAR);
}
private void doFinalize(StorageDirectory sd) throws IOException {
@@ -573,6 +556,7 @@ public class FSImage implements Closeabl
*/
void reloadFromImageFile(File file, FSNamesystem target) throws IOException {
target.dir.reset();
+ target.dtSecretManager.reset();
LOG.debug("Reloading namespace from " + file);
loadFSImage(file, target, null);
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1377092&r1=1377091&r2=1377092&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Aug 24 20:38:08 2012
@@ -19,10 +19,14 @@ package org.apache.hadoop.hdfs.server.na
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY;
@@ -131,7 +135,6 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -155,7 +158,6 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
@@ -174,7 +176,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Server;
@@ -192,6 +193,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.VersionInfo;
import org.mortbay.util.ajax.JSON;
@@ -296,7 +298,7 @@ public class FSNamesystem implements Nam
// Scan interval is not configurable.
private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL =
TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
- private final DelegationTokenSecretManager dtSecretManager;
+ final DelegationTokenSecretManager dtSecretManager;
private final boolean alwaysUseDelegationTokensForTests;
@@ -473,13 +475,25 @@ public class FSNamesystem implements Nam
"must not be specified if HA is not enabled.");
}
+ // Get the checksum type from config
+ String checksumTypeStr = conf.get(DFS_CHECKSUM_TYPE_KEY, DFS_CHECKSUM_TYPE_DEFAULT);
+ DataChecksum.Type checksumType;
+ try {
+ checksumType = DataChecksum.Type.valueOf(checksumTypeStr);
+ } catch (IllegalArgumentException iae) {
+ throw new IOException("Invalid checksum type in "
+ + DFS_CHECKSUM_TYPE_KEY + ": " + checksumTypeStr);
+ }
+
this.serverDefaults = new FsServerDefaults(
conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT),
conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT),
conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT),
(short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT),
conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT),
- conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT));
+ conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT),
+ conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT),
+ checksumType);
this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY,
DFS_NAMENODE_MAX_OBJECTS_DEFAULT);
@@ -924,8 +938,7 @@ public class FSNamesystem implements Nam
NamespaceInfo unprotectedGetNamespaceInfo() {
return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(),
getClusterId(), getBlockPoolId(),
- dir.fsImage.getStorage().getCTime(),
- upgradeManager.getUpgradeVersion());
+ dir.fsImage.getStorage().getCTime());
}
/**
@@ -3369,13 +3382,6 @@ public class FSNamesystem implements Nam
DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat(
nodeReg, blockPoolId, capacity, dfsUsed, remaining, blockPoolUsed,
xceiverCount, maxTransfer, failedVolumes);
- if (cmds == null || cmds.length == 0) {
- DatanodeCommand cmd = upgradeManager.getBroadcastCommand();
- if (cmd != null) {
- cmds = new DatanodeCommand[] {cmd};
- }
- }
-
return new HeartbeatResponse(cmds, createHaStatusHeartbeat());
} finally {
readUnlock();
@@ -3816,24 +3822,9 @@ public class FSNamesystem implements Nam
/**
* Leave safe mode.
* <p>
- * Switch to manual safe mode if distributed upgrade is required.<br>
* Check for invalid, under- & over-replicated blocks in the end of startup.
*/
- private synchronized void leave(boolean checkForUpgrades) {
- if(checkForUpgrades) {
- // verify whether a distributed upgrade needs to be started
- boolean needUpgrade = false;
- try {
- needUpgrade = upgradeManager.startUpgrade();
- } catch(IOException e) {
- FSNamesystem.LOG.error("IOException in startDistributedUpgradeIfNeeded", e);
- }
- if(needUpgrade) {
- // switch to manual safe mode
- safeMode = new SafeModeInfo(false);
- return;
- }
- }
+ private synchronized void leave() {
// if not done yet, initialize replication queues.
// In the standby, do not populate repl queues
if (!isPopulatingReplQueues() && !isInStandbyState()) {
@@ -3927,7 +3918,7 @@ public class FSNamesystem implements Nam
// the threshold is reached
if (!isOn() || // safe mode is off
extension <= 0 || threshold <= 0) { // don't need to wait
- this.leave(true); // leave safe mode
+ this.leave(); // leave safe mode
return;
}
if (reached > 0) { // threshold has already been reached before
@@ -4031,10 +4022,6 @@ public class FSNamesystem implements Nam
leaveMsg = "Safe mode will be turned off automatically";
}
if(isManual()) {
- if(upgradeManager.getUpgradeState())
- return leaveMsg + " upon completion of " +
- "the distributed upgrade: upgrade progress = " +
- upgradeManager.getUpgradeStatus() + "%";
leaveMsg = "Use \"hdfs dfsadmin -safemode leave\" to turn safe mode off";
}
@@ -4169,13 +4156,7 @@ public class FSNamesystem implements Nam
LOG.info("NameNode is being shutdown, exit SafeModeMonitor thread. ");
} else {
// leave safe mode and stop the monitor
- try {
- leaveSafeMode(true);
- } catch(SafeModeException es) { // should never happen
- String msg = "SafeModeMonitor may not run during distributed upgrade.";
- assert false : msg;
- throw new RuntimeException(msg, es);
- }
+ leaveSafeMode();
}
smmthread = null;
}
@@ -4186,7 +4167,7 @@ public class FSNamesystem implements Nam
checkSuperuserPrivilege();
switch(action) {
case SAFEMODE_LEAVE: // leave safe mode
- leaveSafeMode(false);
+ leaveSafeMode();
break;
case SAFEMODE_ENTER: // enter safe mode
enterSafeMode(false);
@@ -4371,17 +4352,14 @@ public class FSNamesystem implements Nam
* Leave safe mode.
* @throws IOException
*/
- void leaveSafeMode(boolean checkForUpgrades) throws SafeModeException {
+ void leaveSafeMode() {
writeLock();
try {
if (!isInSafeMode()) {
NameNode.stateChangeLog.info("STATE* Safe mode is already OFF.");
return;
}
- if(upgradeManager.getUpgradeState())
- throw new SafeModeException("Distributed upgrade is in progress",
- safeMode);
- safeMode.leave(checkForUpgrades);
+ safeMode.leave();
} finally {
writeUnlock();
}
@@ -4456,18 +4434,6 @@ public class FSNamesystem implements Nam
return (blockManager.getBlockCollection(b) != null);
}
- // Distributed upgrade manager
- final UpgradeManagerNamenode upgradeManager = new UpgradeManagerNamenode(this);
-
- UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action
- ) throws IOException {
- return upgradeManager.distributedUpgradeProgress(action);
- }
-
- UpgradeCommand processDistributedUpgradeCommand(UpgradeCommand comm) throws IOException {
- return upgradeManager.processUpgradeCommand(comm);
- }
-
PermissionStatus createFsOwnerPermissions(FsPermission permission) {
return new PermissionStatus(fsOwner.getShortUserName(), supergroup, permission);
}