You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:25 UTC
svn commit: r1619012 [9/35] - in
/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop-...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java Tue Aug 19 23:49:39 2014
@@ -35,8 +35,11 @@ import org.apache.hadoop.fs.FsServerDefa
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -92,6 +95,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto;
@@ -141,10 +145,16 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetTimesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CheckAccessRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.SetXAttrRequestProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufHelper;
@@ -329,11 +339,12 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public void abandonBlock(ExtendedBlock b, String src, String holder)
- throws AccessControlException, FileNotFoundException,
- UnresolvedLinkException, IOException {
+ public void abandonBlock(ExtendedBlock b, long fileId, String src,
+ String holder) throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
AbandonBlockRequestProto req = AbandonBlockRequestProto.newBuilder()
- .setB(PBHelper.convert(b)).setSrc(src).setHolder(holder).build();
+ .setB(PBHelper.convert(b)).setSrc(src).setHolder(holder)
+ .setFileId(fileId).build();
try {
rpcProxy.abandonBlock(null, req);
} catch (ServiceException e) {
@@ -365,8 +376,8 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public LocatedBlock getAdditionalDatanode(String src, ExtendedBlock blk,
- DatanodeInfo[] existings, String[] existingStorageIDs,
+ public LocatedBlock getAdditionalDatanode(String src, long fileId,
+ ExtendedBlock blk, DatanodeInfo[] existings, String[] existingStorageIDs,
DatanodeInfo[] excludes,
int numAdditionalNodes, String clientName) throws AccessControlException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
@@ -374,6 +385,7 @@ public class ClientNamenodeProtocolTrans
GetAdditionalDatanodeRequestProto req = GetAdditionalDatanodeRequestProto
.newBuilder()
.setSrc(src)
+ .setFileId(fileId)
.setBlk(PBHelper.convert(blk))
.addAllExistings(PBHelper.convert(existings))
.addAllExistingStorageUuids(Arrays.asList(existingStorageIDs))
@@ -573,6 +585,20 @@ public class ClientNamenodeProtocolTrans
}
@Override
+ public DatanodeStorageReport[] getDatanodeStorageReport(DatanodeReportType type)
+ throws IOException {
+ final GetDatanodeStorageReportRequestProto req
+ = GetDatanodeStorageReportRequestProto.newBuilder()
+ .setType(PBHelper.convert(type)).build();
+ try {
+ return PBHelper.convertDatanodeStorageReports(
+ rpcProxy.getDatanodeStorageReport(null, req).getDatanodeStorageReportsList());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public long getPreferredBlockSize(String filename) throws IOException,
UnresolvedLinkException {
GetPreferredBlockSizeRequestProto req = GetPreferredBlockSizeRequestProto
@@ -750,11 +776,13 @@ public class ClientNamenodeProtocolTrans
}
@Override
- public void fsync(String src, String client, long lastBlockLength)
+ public void fsync(String src, long fileId, String client,
+ long lastBlockLength)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException {
FsyncRequestProto req = FsyncRequestProto.newBuilder().setSrc(src)
- .setClient(client).setLastBlockLength(lastBlockLength).build();
+ .setClient(client).setLastBlockLength(lastBlockLength)
+ .setFileId(fileId).build();
try {
rpcProxy.fsync(null, req);
} catch (ServiceException e) {
@@ -1264,4 +1292,71 @@ public class ClientNamenodeProtocolTrans
throw ProtobufHelper.getRemoteException(e);
}
}
+
+ @Override
+ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder()
+ .setSrc(src)
+ .setXAttr(PBHelper.convertXAttrProto(xAttr))
+ .setFlag(PBHelper.convert(flag))
+ .build();
+ try {
+ rpcProxy.setXAttr(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
+ throws IOException {
+ GetXAttrsRequestProto.Builder builder = GetXAttrsRequestProto.newBuilder();
+ builder.setSrc(src);
+ if (xAttrs != null) {
+ builder.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ }
+ GetXAttrsRequestProto req = builder.build();
+ try {
+ return PBHelper.convert(rpcProxy.getXAttrs(null, req));
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public List<XAttr> listXAttrs(String src)
+ throws IOException {
+ ListXAttrsRequestProto.Builder builder = ListXAttrsRequestProto.newBuilder();
+ builder.setSrc(src);
+ ListXAttrsRequestProto req = builder.build();
+ try {
+ return PBHelper.convert(rpcProxy.listXAttrs(null, req));
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void removeXAttr(String src, XAttr xAttr) throws IOException {
+ RemoveXAttrRequestProto req = RemoveXAttrRequestProto
+ .newBuilder().setSrc(src)
+ .setXAttr(PBHelper.convertXAttrProto(xAttr)).build();
+ try {
+ rpcProxy.removeXAttr(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void checkAccess(String path, FsAction mode) throws IOException {
+ CheckAccessRequestProto req = CheckAccessRequestProto.newBuilder()
+ .setPath(path).setMode(PBHelper.convert(mode)).build();
+ try {
+ rpcProxy.checkAccess(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java Tue Aug 19 23:49:39 2014
@@ -21,18 +21,13 @@ package org.apache.hadoop.hdfs.protocolP
import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto;
@@ -51,7 +46,6 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageBlockReportProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -61,14 +55,10 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -97,7 +87,7 @@ public class DatanodeProtocolClientSideT
RPC.setProtocolEngine(conf, DatanodeProtocolPB.class,
ProtobufRpcEngine.class);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- rpcProxy = createNamenodeWithRetry(createNamenode(nameNodeAddr, conf, ugi));
+ rpcProxy = createNamenode(nameNodeAddr, conf, ugi);
}
private static DatanodeProtocolPB createNamenode(
@@ -109,33 +99,6 @@ public class DatanodeProtocolClientSideT
org.apache.hadoop.ipc.Client.getPingInterval(conf), null).getProxy();
}
- /** Create a {@link NameNode} proxy */
- static DatanodeProtocolPB createNamenodeWithRetry(
- DatanodeProtocolPB rpcNamenode) {
- RetryPolicy createPolicy = RetryPolicies
- .retryUpToMaximumCountWithFixedSleep(5,
- HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
-
- Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
- createPolicy);
-
- Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
- new HashMap<Class<? extends Exception>, RetryPolicy>();
- exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
- .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
- remoteExceptionToPolicyMap));
- RetryPolicy methodPolicy = RetryPolicies.retryByException(
- RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
- Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
-
- methodNameToPolicyMap.put("create", methodPolicy);
-
- return (DatanodeProtocolPB) RetryProxy.create(DatanodeProtocolPB.class,
- rpcNamenode, methodNameToPolicyMap);
- }
-
@Override
public void close() throws IOException {
RPC.stopProxy(rpcProxy);
@@ -164,9 +127,7 @@ public class DatanodeProtocolClientSideT
.setRegistration(PBHelper.convert(registration))
.setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount)
.setFailedVolumes(failedVolumes);
- for (StorageReport r : reports) {
- builder.addReports(PBHelper.convert(r));
- }
+ builder.addAllReports(PBHelper.convertStorageReports(reports));
if (cacheCapacity != 0) {
builder.setCacheCapacity(cacheCapacity);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java Tue Aug 19 23:49:39 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
+import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
@@ -61,7 +62,7 @@ import com.google.protobuf.ServiceExcept
@InterfaceAudience.Private
@InterfaceStability.Stable
public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
- ProtocolMetaInterface, Closeable {
+ ProtocolMetaInterface, Closeable, ProtocolTranslator {
/** RpcController is not used and hence is set to null */
private final static RpcController NULL_CONTROLLER = null;
@@ -89,6 +90,11 @@ public class NamenodeProtocolTranslatorP
}
@Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+
+ @Override
public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
throws IOException {
GetBlocksRequestProto req = GetBlocksRequestProto.newBuilder()
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Tue Aug 19 23:49:39 2014
@@ -32,6 +32,8 @@ import org.apache.hadoop.fs.ContentSumma
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclEntryScope;
import org.apache.hadoop.fs.permission.AclEntryType;
@@ -57,6 +59,7 @@ import org.apache.hadoop.hdfs.protocol.D
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.FsAclPermission;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
@@ -87,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolStatsProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeActionProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeInfoProto;
@@ -99,14 +103,11 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto.StorageState;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.FinalizeCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
@@ -122,6 +123,8 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto.AdminState;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfosProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeLocalInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.StorageState;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
@@ -146,9 +149,16 @@ import org.apache.hadoop.hdfs.protocol.p
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrProto.XAttrNamespaceProto;
+import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrSetFlagProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
@@ -173,6 +183,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
import org.apache.hadoop.hdfs.server.protocol.JournalInfo;
import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
@@ -220,6 +231,8 @@ public class PBHelper {
AclEntryType.values();
private static final FsAction[] FSACTION_VALUES =
FsAction.values();
+ private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
+ XAttr.NameSpace.values();
private PBHelper() {
/** Hidden constructor */
@@ -339,15 +352,19 @@ public class PBHelper {
return BlockWithLocationsProto.newBuilder()
.setBlock(convert(blk.getBlock()))
.addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids()))
- .addAllStorageUuids(Arrays.asList(blk.getStorageIDs())).build();
+ .addAllStorageUuids(Arrays.asList(blk.getStorageIDs()))
+ .addAllStorageTypes(convertStorageTypes(blk.getStorageTypes()))
+ .build();
}
public static BlockWithLocations convert(BlockWithLocationsProto b) {
final List<String> datanodeUuids = b.getDatanodeUuidsList();
final List<String> storageUuids = b.getStorageUuidsList();
+ final List<StorageTypeProto> storageTypes = b.getStorageTypesList();
return new BlockWithLocations(convert(b.getBlock()),
datanodeUuids.toArray(new String[datanodeUuids.size()]),
- storageUuids.toArray(new String[storageUuids.size()]));
+ storageUuids.toArray(new String[storageUuids.size()]),
+ convertStorageTypes(storageTypes, storageUuids.size()));
}
public static BlocksWithLocationsProto convert(BlocksWithLocations blks) {
@@ -609,6 +626,41 @@ public class PBHelper {
return builder.build();
}
+ public static DatanodeStorageReportProto convertDatanodeStorageReport(
+ DatanodeStorageReport report) {
+ return DatanodeStorageReportProto.newBuilder()
+ .setDatanodeInfo(convert(report.getDatanodeInfo()))
+ .addAllStorageReports(convertStorageReports(report.getStorageReports()))
+ .build();
+ }
+
+ public static List<DatanodeStorageReportProto> convertDatanodeStorageReports(
+ DatanodeStorageReport[] reports) {
+ final List<DatanodeStorageReportProto> protos
+ = new ArrayList<DatanodeStorageReportProto>(reports.length);
+ for(int i = 0; i < reports.length; i++) {
+ protos.add(convertDatanodeStorageReport(reports[i]));
+ }
+ return protos;
+ }
+
+ public static DatanodeStorageReport convertDatanodeStorageReport(
+ DatanodeStorageReportProto proto) {
+ return new DatanodeStorageReport(
+ convert(proto.getDatanodeInfo()),
+ convertStorageReports(proto.getStorageReportsList()));
+ }
+
+ public static DatanodeStorageReport[] convertDatanodeStorageReports(
+ List<DatanodeStorageReportProto> protos) {
+ final DatanodeStorageReport[] reports
+ = new DatanodeStorageReport[protos.size()];
+ for(int i = 0; i < reports.length; i++) {
+ reports[i] = convertDatanodeStorageReport(protos.get(i));
+ }
+ return reports;
+ }
+
public static AdminStates convert(AdminState adminState) {
switch(adminState) {
case DECOMMISSION_INPROGRESS:
@@ -664,14 +716,8 @@ public class PBHelper {
targets[i] = PBHelper.convert(locs.get(i));
}
- final int storageTypesCount = proto.getStorageTypesCount();
- final StorageType[] storageTypes;
- if (storageTypesCount == 0) {
- storageTypes = null;
- } else {
- Preconditions.checkState(storageTypesCount == locs.size());
- storageTypes = convertStorageTypeProtos(proto.getStorageTypesList());
- }
+ final StorageType[] storageTypes = convertStorageTypes(
+ proto.getStorageTypesList(), locs.size());
final int storageIDsCount = proto.getStorageIDsCount();
final String[] storageIDs;
@@ -959,6 +1005,20 @@ public class PBHelper {
targets[i] = PBHelper.convert(targetList.get(i));
}
+ StorageType[][] targetStorageTypes = new StorageType[targetList.size()][];
+ List<StorageTypesProto> targetStorageTypesList = blkCmd.getTargetStorageTypesList();
+ if (targetStorageTypesList.isEmpty()) { // missing storage types
+ for(int i = 0; i < targetStorageTypes.length; i++) {
+ targetStorageTypes[i] = new StorageType[targets[i].length];
+ Arrays.fill(targetStorageTypes[i], StorageType.DEFAULT);
+ }
+ } else {
+ for(int i = 0; i < targetStorageTypes.length; i++) {
+ List<StorageTypeProto> p = targetStorageTypesList.get(i).getStorageTypesList();
+ targetStorageTypes[i] = p.toArray(new StorageType[p.size()]);
+ }
+ }
+
List<StorageUuidsProto> targetStorageUuidsList = blkCmd.getTargetStorageUuidsList();
String[][] targetStorageIDs = new String[targetStorageUuidsList.size()][];
for(int i = 0; i < targetStorageIDs.length; i++) {
@@ -981,7 +1041,7 @@ public class PBHelper {
throw new AssertionError("Unknown action type: " + blkCmd.getAction());
}
return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets,
- targetStorageIDs);
+ targetStorageTypes, targetStorageIDs);
}
public static BlockIdCommand convert(BlockIdCommandProto blkIdCmd) {
@@ -1186,13 +1246,11 @@ public class PBHelper {
}
public static FsPermissionProto convert(FsPermission p) {
- if (p == null) return null;
- return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
+ return FsPermissionProto.newBuilder().setPerm(p.toExtendedShort()).build();
}
public static FsPermission convert(FsPermissionProto p) {
- if (p == null) return null;
- return new FsPermission((short)p.getPerm());
+ return new FsAclPermission((short)p.getPerm());
}
@@ -1414,6 +1472,7 @@ public class PBHelper {
case ALL: return DatanodeReportTypeProto.ALL;
case LIVE: return DatanodeReportTypeProto.LIVE;
case DEAD: return DatanodeReportTypeProto.DEAD;
+ case DECOMMISSIONING: return DatanodeReportTypeProto.DECOMMISSIONING;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
@@ -1425,6 +1484,7 @@ public class PBHelper {
case ALL: return DatanodeReportType.ALL;
case LIVE: return DatanodeReportType.LIVE;
case DEAD: return DatanodeReportType.DEAD;
+ case DECOMMISSIONING: return DatanodeReportType.DECOMMISSIONING;
default:
throw new IllegalArgumentException("Unexpected data type report:" + t);
}
@@ -1595,8 +1655,25 @@ public class PBHelper {
}
}
- private static StorageTypeProto convertStorageType(
- StorageType type) {
+ public static List<StorageTypeProto> convertStorageTypes(
+ StorageType[] types) {
+ return convertStorageTypes(types, 0);
+ }
+
+ public static List<StorageTypeProto> convertStorageTypes(
+ StorageType[] types, int startIdx) {
+ if (types == null) {
+ return null;
+ }
+ final List<StorageTypeProto> protos = new ArrayList<StorageTypeProto>(
+ types.length);
+ for (int i = startIdx; i < types.length; ++i) {
+ protos.add(convertStorageType(types[i]));
+ }
+ return protos;
+ }
+
+ public static StorageTypeProto convertStorageType(StorageType type) {
switch(type) {
case DISK:
return StorageTypeProto.DISK;
@@ -1611,7 +1688,7 @@ public class PBHelper {
public static DatanodeStorage convert(DatanodeStorageProto s) {
return new DatanodeStorage(s.getStorageUuid(),
PBHelper.convertState(s.getState()),
- PBHelper.convertType(s.getStorageType()));
+ PBHelper.convertStorageType(s.getStorageType()));
}
private static State convertState(StorageState state) {
@@ -1624,7 +1701,7 @@ public class PBHelper {
}
}
- private static StorageType convertType(StorageTypeProto type) {
+ public static StorageType convertStorageType(StorageTypeProto type) {
switch(type) {
case DISK:
return StorageType.DISK;
@@ -1636,11 +1713,16 @@ public class PBHelper {
}
}
- private static StorageType[] convertStorageTypeProtos(
- List<StorageTypeProto> storageTypesList) {
- final StorageType[] storageTypes = new StorageType[storageTypesList.size()];
- for (int i = 0; i < storageTypes.length; ++i) {
- storageTypes[i] = PBHelper.convertType(storageTypesList.get(i));
+ public static StorageType[] convertStorageTypes(
+ List<StorageTypeProto> storageTypesList, int expectedSize) {
+ final StorageType[] storageTypes = new StorageType[expectedSize];
+ if (storageTypesList.size() != expectedSize) { // missing storage types
+ Preconditions.checkState(storageTypesList.isEmpty());
+ Arrays.fill(storageTypes, StorageType.DEFAULT);
+ } else {
+ for (int i = 0; i < storageTypes.length; ++i) {
+ storageTypes[i] = convertStorageType(storageTypesList.get(i));
+ }
}
return storageTypes;
}
@@ -1672,6 +1754,15 @@ public class PBHelper {
return report;
}
+ public static List<StorageReportProto> convertStorageReports(StorageReport[] storages) {
+ final List<StorageReportProto> protos = new ArrayList<StorageReportProto>(
+ storages.length);
+ for(int i = 0; i < storages.length; i++) {
+ protos.add(convert(storages[i]));
+ }
+ return protos;
+ }
+
public static JournalInfo convert(JournalInfoProto info) {
int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0;
int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0;
@@ -1726,24 +1817,29 @@ public class PBHelper {
}
DiffType type = DiffType.getTypeFromLabel(entry
.getModificationLabel());
- return type == null ? null :
- new DiffReportEntry(type, entry.getFullpath().toByteArray());
+ return type == null ? null : new DiffReportEntry(type, entry.getFullpath()
+ .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath()
+ .toByteArray() : null);
}
public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) {
if (entry == null) {
return null;
}
- byte[] fullPath = entry.getRelativePath();
- ByteString fullPathString = ByteString
- .copyFrom(fullPath == null ? DFSUtil.EMPTY_BYTES : fullPath);
-
+ ByteString sourcePath = ByteString
+ .copyFrom(entry.getSourcePath() == null ? DFSUtil.EMPTY_BYTES : entry
+ .getSourcePath());
String modification = entry.getType().getLabel();
-
- SnapshotDiffReportEntryProto entryProto = SnapshotDiffReportEntryProto
- .newBuilder().setFullpath(fullPathString)
- .setModificationLabel(modification).build();
- return entryProto;
+ SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
+ .newBuilder().setFullpath(sourcePath)
+ .setModificationLabel(modification);
+ if (entry.getType() == DiffType.RENAME) {
+ ByteString targetPath = ByteString
+ .copyFrom(entry.getTargetPath() == null ? DFSUtil.EMPTY_BYTES : entry
+ .getTargetPath());
+ builder.setTargetPath(targetPath);
+ }
+ return builder.build();
}
public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) {
@@ -2006,12 +2102,20 @@ public class PBHelper {
private static AclEntryType convert(AclEntryTypeProto v) {
return castEnum(v, ACL_ENTRY_TYPE_VALUES);
}
+
+ private static XAttrNamespaceProto convert(XAttr.NameSpace v) {
+ return XAttrNamespaceProto.valueOf(v.ordinal());
+ }
+
+ private static XAttr.NameSpace convert(XAttrNamespaceProto v) {
+ return castEnum(v, XATTR_NAMESPACE_VALUES);
+ }
- private static FsActionProto convert(FsAction v) {
+ public static FsActionProto convert(FsAction v) {
return FsActionProto.valueOf(v != null ? v.ordinal() : 0);
}
- private static FsAction convert(FsActionProto v) {
+ public static FsAction convert(FsActionProto v) {
return castEnum(v, FSACTION_VALUES);
}
@@ -2059,6 +2163,126 @@ public class PBHelper {
.addAllEntries(convertAclEntryProto(e.getEntries())).build();
return GetAclStatusResponseProto.newBuilder().setResult(r).build();
}
+
+ public static XAttrProto convertXAttrProto(XAttr a) {
+ XAttrProto.Builder builder = XAttrProto.newBuilder();
+ builder.setNamespace(convert(a.getNameSpace()));
+ if (a.getName() != null) {
+ builder.setName(a.getName());
+ }
+ if (a.getValue() != null) {
+ builder.setValue(getByteString(a.getValue()));
+ }
+ return builder.build();
+ }
+
+ public static List<XAttrProto> convertXAttrProto(
+ List<XAttr> xAttrSpec) {
+ if (xAttrSpec == null) {
+ return Lists.newArrayListWithCapacity(0);
+ }
+ ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
+ xAttrSpec.size());
+ for (XAttr a : xAttrSpec) {
+ XAttrProto.Builder builder = XAttrProto.newBuilder();
+ builder.setNamespace(convert(a.getNameSpace()));
+ if (a.getName() != null) {
+ builder.setName(a.getName());
+ }
+ if (a.getValue() != null) {
+ builder.setValue(getByteString(a.getValue()));
+ }
+ xAttrs.add(builder.build());
+ }
+ return xAttrs;
+ }
+
+ /**
+ * The flag field in PB is a bitmask whose values are the same a the
+ * emum values of XAttrSetFlag
+ */
+ public static int convert(EnumSet<XAttrSetFlag> flag) {
+ int value = 0;
+ if (flag.contains(XAttrSetFlag.CREATE)) {
+ value |= XAttrSetFlagProto.XATTR_CREATE.getNumber();
+ }
+ if (flag.contains(XAttrSetFlag.REPLACE)) {
+ value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber();
+ }
+ return value;
+ }
+
+ public static EnumSet<XAttrSetFlag> convert(int flag) {
+ EnumSet<XAttrSetFlag> result =
+ EnumSet.noneOf(XAttrSetFlag.class);
+ if ((flag & XAttrSetFlagProto.XATTR_CREATE_VALUE) ==
+ XAttrSetFlagProto.XATTR_CREATE_VALUE) {
+ result.add(XAttrSetFlag.CREATE);
+ }
+ if ((flag & XAttrSetFlagProto.XATTR_REPLACE_VALUE) ==
+ XAttrSetFlagProto.XATTR_REPLACE_VALUE) {
+ result.add(XAttrSetFlag.REPLACE);
+ }
+ return result;
+ }
+
+ public static XAttr convertXAttr(XAttrProto a) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(convert(a.getNamespace()));
+ if (a.hasName()) {
+ builder.setName(a.getName());
+ }
+ if (a.hasValue()) {
+ builder.setValue(a.getValue().toByteArray());
+ }
+ return builder.build();
+ }
+
+ public static List<XAttr> convertXAttrs(List<XAttrProto> xAttrSpec) {
+ ArrayList<XAttr> xAttrs = Lists.newArrayListWithCapacity(xAttrSpec.size());
+ for (XAttrProto a : xAttrSpec) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(convert(a.getNamespace()));
+ if (a.hasName()) {
+ builder.setName(a.getName());
+ }
+ if (a.hasValue()) {
+ builder.setValue(a.getValue().toByteArray());
+ }
+ xAttrs.add(builder.build());
+ }
+ return xAttrs;
+ }
+
+ public static List<XAttr> convert(GetXAttrsResponseProto a) {
+ List<XAttrProto> xAttrs = a.getXAttrsList();
+ return convertXAttrs(xAttrs);
+ }
+
+ public static GetXAttrsResponseProto convertXAttrsResponse(
+ List<XAttr> xAttrs) {
+ GetXAttrsResponseProto.Builder builder = GetXAttrsResponseProto
+ .newBuilder();
+ if (xAttrs != null) {
+ builder.addAllXAttrs(convertXAttrProto(xAttrs));
+ }
+ return builder.build();
+ }
+
+ public static List<XAttr> convert(ListXAttrsResponseProto a) {
+ final List<XAttrProto> xAttrs = a.getXAttrsList();
+ return convertXAttrs(xAttrs);
+ }
+
+ public static ListXAttrsResponseProto convertListXAttrsResponse(
+ List<XAttr> names) {
+ ListXAttrsResponseProto.Builder builder =
+ ListXAttrsResponseProto.newBuilder();
+ if (names != null) {
+ builder.addAllXAttrs(convertXAttrProto(names));
+ }
+ return builder.build();
+ }
public static ShortCircuitShmSlotProto convert(SlotId slotId) {
return ShortCircuitShmSlotProto.newBuilder().
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java Tue Aug 19 23:49:39 2014
@@ -101,9 +101,9 @@ public class BlockTokenSecretManager ext
*
* @param keyUpdateInterval how often a new key will be generated
* @param tokenLifetime how long an individual token is valid
- * @param isHaEnabled whether or not HA is enabled
- * @param thisNnId the NN ID of this NN in an HA setup
- * @param otherNnId the NN ID of the other NN in an HA setup
+ * @param nnIndex namenode index
+ * @param blockPoolId block pool ID
+ * @param encryptionAlgorithm encryption algorithm to use
*/
public BlockTokenSecretManager(long keyUpdateInterval,
long tokenLifetime, int nnIndex, String blockPoolId,
@@ -412,8 +412,7 @@ public class BlockTokenSecretManager ext
* @param keyId identifier of the secret key used to generate the encryption key.
* @param nonce random value used to create the encryption key
* @return the encryption key which corresponds to this (keyId, blockPoolId, nonce)
- * @throws InvalidToken
- * @throws InvalidEncryptionKeyException
+ * @throws InvalidEncryptionKeyException
*/
public byte[] retrieveDataEncryptionKey(int keyId, byte[] nonce)
throws InvalidEncryptionKeyException {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java Tue Aug 19 23:49:39 2014
@@ -23,6 +23,8 @@ import java.io.DataInputStream;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
@@ -75,4 +77,25 @@ public class DelegationTokenIdentifier
return ident.toString();
}
}
+
+ public static class WebHdfsDelegationTokenIdentifier
+ extends DelegationTokenIdentifier {
+ public WebHdfsDelegationTokenIdentifier() {
+ super();
+ }
+ @Override
+ public Text getKind() {
+ return WebHdfsFileSystem.TOKEN_KIND;
+ }
+ }
+
+ public static class SWebHdfsDelegationTokenIdentifier extends WebHdfsDelegationTokenIdentifier {
+ public SWebHdfsDelegationTokenIdentifier() {
+ super();
+ }
+ @Override
+ public Text getKind() {
+ return SWebHdfsFileSystem.TOKEN_KIND;
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Tue Aug 19 23:49:39 2014
@@ -18,9 +18,16 @@
package org.apache.hadoop.hdfs.security.token.delegation;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.protobuf.ByteString;
+import java.io.DataInput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -43,13 +50,9 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.security.token.delegation.DelegationKey;
-import java.io.DataInput;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
/**
* A HDFS specific delegation token secret manager.
@@ -211,6 +214,18 @@ public class DelegationTokenSecretManage
}
}
+ /**
+ * Store the current state of the SecretManager for persistence
+ *
+ * @param out Output stream for writing into fsimage.
+ * @param sdPath String storage directory path
+ * @throws IOException
+ */
+ public synchronized void saveSecretManagerStateCompat(DataOutputStream out,
+ String sdPath) throws IOException {
+ serializerCompat.save(out, sdPath);
+ }
+
public synchronized SecretManagerState saveSecretManagerState() {
SecretManagerSection s = SecretManagerSection.newBuilder()
.setCurrentId(currentId)
@@ -299,7 +314,7 @@ public class DelegationTokenSecretManage
* Update the token cache with renewal record in edit logs.
*
* @param identifier DelegationTokenIdentifier of the renewed token
- * @param expiryTime
+ * @param expiryTime expirty time in milliseconds
* @throws IOException
*/
public synchronized void updatePersistedTokenRenewal(
@@ -406,6 +421,56 @@ public class DelegationTokenSecretManage
loadCurrentTokens(in);
}
+ private void save(DataOutputStream out, String sdPath) throws IOException {
+ out.writeInt(currentId);
+ saveAllKeys(out, sdPath);
+ out.writeInt(delegationTokenSequenceNumber);
+ saveCurrentTokens(out, sdPath);
+ }
+
+ /**
+ * Private helper methods to save delegation keys and tokens in fsimage
+ */
+ private synchronized void saveCurrentTokens(DataOutputStream out,
+ String sdPath) throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_TOKENS, sdPath);
+ prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+ prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+ Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
+ out.writeInt(currentTokens.size());
+ Iterator<DelegationTokenIdentifier> iter = currentTokens.keySet()
+ .iterator();
+ while (iter.hasNext()) {
+ DelegationTokenIdentifier id = iter.next();
+ id.write(out);
+ DelegationTokenInformation info = currentTokens.get(id);
+ out.writeLong(info.getRenewDate());
+ counter.increment();
+ }
+ prog.endStep(Phase.SAVING_CHECKPOINT, step);
+ }
+
+ /*
+ * Save the current state of allKeys
+ */
+ private synchronized void saveAllKeys(DataOutputStream out, String sdPath)
+ throws IOException {
+ StartupProgress prog = NameNode.getStartupProgress();
+ Step step = new Step(StepType.DELEGATION_KEYS, sdPath);
+ prog.beginStep(Phase.SAVING_CHECKPOINT, step);
+ prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size());
+ Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step);
+ out.writeInt(allKeys.size());
+ Iterator<Integer> iter = allKeys.keySet().iterator();
+ while (iter.hasNext()) {
+ Integer key = iter.next();
+ allKeys.get(key).write(out);
+ counter.increment();
+ }
+ prog.endStep(Phase.SAVING_CHECKPOINT, step);
+ }
+
/**
* Private helper methods to load Delegation tokens from fsimage
*/
@@ -429,8 +494,7 @@ public class DelegationTokenSecretManage
/**
* Private helper method to load delegation keys from fsimage.
- * @param in
- * @throws IOException
+ * @throws IOException on error
*/
private synchronized void loadAllKeys(DataInput in) throws IOException {
StartupProgress prog = NameNode.getStartupProgress();