You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:44:57 UTC
[45/50] [abbrv] hadoop git commit: Merge commit
'b67ae6d9d741e79ccf2bd6f08a37fce070e6ad77' into HDFS-12943
Merge commit 'b67ae6d9d741e79ccf2bd6f08a37fce070e6ad77' into HDFS-12943
# Conflicts:
# hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
# hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
# hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
# hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/066afcf2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/066afcf2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/066afcf2
Branch: refs/heads/HDFS-12943
Commit: 066afcf291d84d4215b7b447937d9e55f052239e
Parents: 90e61bf 3ac07b7
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Sat Aug 25 17:13:21 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sat Aug 25 17:13:21 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 19 +-
.../hadoop/hdfs/DistributedFileSystem.java | 34 +
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 14 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 32 +
.../ClientNamenodeProtocolTranslatorPB.java | 13 +
.../src/main/proto/ClientNamenodeProtocol.proto | 11 +-
.../router/RouterNamenodeProtocol.java | 7 +
.../federation/router/RouterRpcServer.java | 12 +
.../hadoop-hdfs/src/main/bin/hdfs | 7 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 44 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 2 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 20 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 4 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 18 +
.../NamenodeProtocolTranslatorPB.java | 15 +
.../hdfs/server/balancer/NameNodeConnector.java | 36 +-
.../blockmanagement/BlockInfoStriped.java | 10 +
.../server/blockmanagement/BlockManager.java | 72 +
.../blockmanagement/DatanodeDescriptor.java | 1 -
.../server/blockmanagement/DatanodeManager.java | 24 +-
.../hdfs/server/common/HdfsServerConstants.java | 5 +
.../hdfs/server/common/sps/BlockDispatcher.java | 186 ++
.../sps/BlockMovementAttemptFinished.java | 100 ++
.../server/common/sps/BlockMovementStatus.java | 53 +
.../common/sps/BlockStorageMovementTracker.java | 91 +
.../sps/BlocksMovementsStatusHandler.java | 39 +
.../hdfs/server/common/sps/package-info.java | 27 +
.../hdfs/server/datanode/BPServiceActor.java | 2 +
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 7 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 157 ++
.../hdfs/server/namenode/FSDirXAttrOp.java | 7 +
.../hdfs/server/namenode/FSDirectory.java | 19 +
.../hdfs/server/namenode/FSNamesystem.java | 97 +-
.../hdfs/server/namenode/FSTreeTraverser.java | 4 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 48 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 47 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 10 +
.../server/namenode/ReencryptionHandler.java | 3 +-
.../namenode/sps/BlockMoveTaskHandler.java | 43 +
.../namenode/sps/BlockMovementListener.java | 40 +
.../sps/BlockStorageMovementAttemptedItems.java | 313 ++++
.../sps/BlockStorageMovementNeeded.java | 346 ++++
.../hdfs/server/namenode/sps/Context.java | 141 ++
.../namenode/sps/DatanodeCacheManager.java | 121 ++
.../hdfs/server/namenode/sps/FileCollector.java | 43 +
.../hdfs/server/namenode/sps/ItemInfo.java | 82 +
.../hdfs/server/namenode/sps/SPSService.java | 127 ++
.../namenode/sps/StoragePolicySatisfier.java | 1226 +++++++++++++
.../sps/StoragePolicySatisfyManager.java | 293 ++++
.../hdfs/server/namenode/sps/package-info.java | 28 +
.../protocol/BlockStorageMovementCommand.java | 136 ++
.../BlocksStorageMoveAttemptFinished.java | 48 +
.../hdfs/server/protocol/DatanodeProtocol.java | 2 +
.../server/protocol/DropSPSWorkCommand.java | 36 +
.../hdfs/server/protocol/NamenodeProtocol.java | 10 +-
.../sps/ExternalSPSBlockMoveTaskHandler.java | 223 +++
.../hdfs/server/sps/ExternalSPSContext.java | 211 +++
.../sps/ExternalSPSFilePathCollector.java | 174 ++
.../sps/ExternalStoragePolicySatisfier.java | 117 ++
.../hadoop/hdfs/server/sps/package-info.java | 28 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 +-
.../src/main/proto/DatanodeProtocol.proto | 1 +
.../src/main/proto/NamenodeProtocol.proto | 21 +
.../src/main/resources/hdfs-default.xml | 125 ++
.../src/site/markdown/ArchivalStorage.md | 71 +-
.../src/site/markdown/HDFSCommands.md | 2 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 112 ++
.../SimpleBlocksMovementsStatusHandler.java | 88 +
.../hadoop/hdfs/server/mover/TestMover.java | 73 +-
.../hdfs/server/mover/TestStorageMover.java | 2 +
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestNameNodeReconfigure.java | 130 ++
.../TestPersistentStoragePolicySatisfier.java | 531 ++++++
.../TestStoragePolicySatisfierWithHA.java | 112 ++
.../TestBlockStorageMovementAttemptedItems.java | 220 +++
...stStoragePolicySatisfierWithStripedFile.java | 581 ++++++
.../sps/TestExternalStoragePolicySatisfier.java | 1650 ++++++++++++++++++
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../hdfs/tools/TestStoragePolicyCommands.java | 10 +-
.../TestStoragePolicySatisfyAdminCommands.java | 112 ++
82 files changed, 8953 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 84a875a,e8c881b..9e456a5
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@@ -1789,13 -1744,16 +1789,27 @@@ public interface ClientProtocol
EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException;
/**
+ * Called by client to wait until the server has reached the state id of the
+ * client. The client and server state id are given by client side and server
+ * side alignment context respectively. This can be a blocking call.
+ *
+ * @throws IOException
+ */
+ @Idempotent
+ @ReadOnly
+ void msync() throws IOException;
++
++ /**
+ * Satisfy the storage policy for a file/directory.
+ * @param path Path of an existing file/directory.
+ * @throws AccessControlException If access is denied.
+ * @throws org.apache.hadoop.fs.UnresolvedLinkException if <code>src</code>
+ * contains a symlink.
+ * @throws java.io.FileNotFoundException If file/dir <code>src</code> is not
+ * found.
+ * @throws org.apache.hadoop.hdfs.server.namenode.SafeModeException append not
+ * allowed in safemode.
+ */
+ @AtMostOnce
+ void satisfyStoragePolicy(String path) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 088a9aa,e4bca51..8ca199f
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@@ -1946,12 -1948,13 +1949,22 @@@ public class ClientNamenodeProtocolTran
}
@Override
+ public void msync() throws IOException {
+ MsyncRequestProto.Builder req = MsyncRequestProto.newBuilder();
+ try {
+ rpcProxy.msync(null, req.build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
++
+ public void satisfyStoragePolicy(String src) throws IOException {
+ SatisfyStoragePolicyRequestProto req =
+ SatisfyStoragePolicyRequestProto.newBuilder().setSrc(src).build();
+ try {
+ rpcProxy.satisfyStoragePolicy(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 247e54b,49ea3f3..55113a1
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@@ -831,12 -830,14 +830,20 @@@ message ListOpenFilesResponseProto
repeated OpenFilesTypeProto types = 3;
}
+message MsyncRequestProto {
+}
+
+message MsyncResponseProto {
+}
+
+ message SatisfyStoragePolicyRequestProto {
+ required string src = 1;
+ }
+
+ message SatisfyStoragePolicyResponseProto {
+
+ }
+
service ClientNamenodeProtocol {
rpc getBlockLocations(GetBlockLocationsRequestProto)
returns(GetBlockLocationsResponseProto);
@@@ -1023,6 -1024,6 +1030,8 @@@
returns(GetQuotaUsageResponseProto);
rpc listOpenFiles(ListOpenFilesRequestProto)
returns(ListOpenFilesResponseProto);
+ rpc msync(MsyncRequestProto)
+ returns(MsyncResponseProto);
+ rpc satisfyStoragePolicy(SatisfyStoragePolicyRequestProto)
+ returns(SatisfyStoragePolicyResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 81dc0fa,e51529e..8a4d4a1
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@@ -1890,13 -1894,14 +1896,25 @@@ public class ClientNamenodeProtocolServ
}
@Override
+ public MsyncResponseProto msync(RpcController controller,
+ MsyncRequestProto req) throws ServiceException {
+ try {
+ server.msync();
+ return MsyncResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
++
++ @Override
+ public SatisfyStoragePolicyResponseProto satisfyStoragePolicy(
+ RpcController controller,
+ SatisfyStoragePolicyRequestProto request) throws ServiceException {
+ try {
+ server.satisfyStoragePolicy(request.getSrc());
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return VOID_SATISFYSTORAGEPOLICY_RESPONSE;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org