You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/07/09 20:33:30 UTC
[21/37] hadoop git commit: Merge remote-tracking branch
'apache-commit/trunk' into HDDS-48
Merge remote-tracking branch 'apache-commit/trunk' into HDDS-48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1045251
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1045251
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1045251
Branch: refs/heads/trunk
Commit: c10452516804eed793e575dedcd1ee7758ec1f4c
Parents: 52d1d96 d3fa83a
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Jun 28 14:28:23 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Jun 28 14:28:23 2018 -0700
----------------------------------------------------------------------
NOTICE.txt | 8 +
dev-support/bin/ozone-dist-layout-stitching | 4 +-
.../hadoop-client-minicluster/pom.xml | 8 -
hadoop-common-project/hadoop-common/pom.xml | 5 -
.../org/apache/hadoop/conf/Configuration.java | 8 +-
.../hadoop/conf/ReconfigurationServlet.java | 42 +-
.../apache/hadoop/crypto/key/KeyProvider.java | 4 +-
.../org/apache/hadoop/crypto/key/KeyShell.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 16 +-
.../java/org/apache/hadoop/fs/BBPartHandle.java | 58 ++
.../org/apache/hadoop/fs/BBUploadHandle.java | 57 ++
.../fs/CommonConfigurationKeysPublic.java | 4 +-
.../hadoop/fs/FSDataOutputStreamBuilder.java | 22 +
.../java/org/apache/hadoop/fs/FileContext.java | 66 ++
.../hadoop/fs/FileSystemMultipartUploader.java | 132 ++++
.../main/java/org/apache/hadoop/fs/FsShell.java | 3 +-
.../hadoop/fs/LocalFileSystemPathHandle.java | 100 +++
.../org/apache/hadoop/fs/MultipartUploader.java | 90 +++
.../hadoop/fs/MultipartUploaderFactory.java | 65 ++
.../main/java/org/apache/hadoop/fs/Options.java | 3 +
.../java/org/apache/hadoop/fs/PartHandle.java | 45 ++
.../main/java/org/apache/hadoop/fs/Path.java | 2 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 61 +-
.../UnsupportedMultipartUploaderException.java | 41 ++
.../java/org/apache/hadoop/fs/UploadHandle.java | 47 ++
.../java/org/apache/hadoop/fs/shell/Count.java | 2 +-
.../apache/hadoop/io/ElasticByteBufferPool.java | 2 +-
.../apache/hadoop/io/FastByteComparisons.java | 44 +-
.../apache/hadoop/io/erasurecode/ECSchema.java | 4 +-
.../io/erasurecode/ErasureCodeNative.java | 2 +-
.../nativeio/SharedFileDescriptorFactory.java | 2 +-
.../org/apache/hadoop/ipc/CallerContext.java | 4 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 4 +-
.../org/apache/hadoop/ipc/FairCallQueue.java | 4 +-
.../hadoop/metrics2/MetricsJsonBuilder.java | 2 +-
.../hadoop/metrics2/impl/MetricsConfig.java | 15 +-
.../hadoop/metrics2/lib/MethodMetric.java | 2 +-
.../metrics2/lib/MutableMetricsFactory.java | 2 +-
.../hadoop/metrics2/lib/MutableQuantiles.java | 2 +-
.../metrics2/lib/MutableRollingAverages.java | 2 +-
.../apache/hadoop/metrics2/lib/MutableStat.java | 2 +-
.../metrics2/sink/RollingFileSystemSink.java | 2 +-
.../org/apache/hadoop/net/TableMapping.java | 2 +-
.../apache/hadoop/net/unix/DomainSocket.java | 2 +-
.../hadoop/net/unix/DomainSocketWatcher.java | 2 +-
.../security/ShellBasedUnixGroupsMapping.java | 12 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../hadoop/security/alias/CredentialShell.java | 2 +-
.../hadoop/security/http/CrossOriginFilter.java | 2 +-
.../hadoop/security/token/DtFileOperations.java | 2 +-
.../web/DelegationTokenAuthenticator.java | 8 +-
.../org/apache/hadoop/tools/TableListing.java | 6 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 12 +-
.../apache/hadoop/util/JsonSerialization.java | 24 +
.../main/java/org/apache/hadoop/util/Shell.java | 2 +-
.../org/apache/hadoop/util/StringUtils.java | 64 +-
.../org/apache/hadoop/util/SysInfoLinux.java | 19 +-
.../hadoop-common/src/main/proto/FSProtos.proto | 8 +
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 +
.../apache/hadoop/conf/TestConfiguration.java | 2 +-
.../conf/TestConfigurationFieldsBase.java | 2 +-
.../crypto/random/TestOsSecureRandom.java | 2 +-
.../fs/AbstractSystemMultipartUploaderTest.java | 143 ++++
.../org/apache/hadoop/fs/FSTestWrapper.java | 2 +-
.../fs/FileContextMainOperationsBaseTest.java | 44 +-
.../apache/hadoop/fs/FileContextTestHelper.java | 1 -
.../hadoop/fs/TestDFCachingGetSpaceUsed.java | 2 +-
.../fs/TestFileSystemStorageStatistics.java | 18 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 23 +-
.../TestLocalFileSystemMultipartUploader.java | 65 ++
.../AbstractContractPathHandleTest.java | 6 +
.../TestRawlocalContractPathHandle.java | 40 ++
.../hadoop/fs/shell/TestCopyFromLocal.java | 10 +-
.../TestSharedFileDescriptorFactory.java | 2 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 2 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 2 +-
.../org/apache/hadoop/net/TestNetUtils.java | 2 +-
.../hadoop/net/unix/TestDomainSocket.java | 2 +-
.../TestShellBasedUnixGroupsMapping.java | 39 +-
...tionTokenAuthenticationHandlerWithMocks.java | 2 +-
.../AbstractServiceLauncherTestBase.java | 2 +-
.../apache/hadoop/test/GenericTestUtils.java | 4 +-
.../hadoop/util/TestShutdownHookManager.java | 2 +-
.../apache/hadoop/util/TestSignalLogger.java | 2 +-
.../apache/hadoop/util/TestSysInfoLinux.java | 60 ++
.../src/test/resources/contract/rawlocal.xml | 5 +
.../test/scripts/process_with_sigterm_trap.sh | 24 +
.../crypto/key/kms/server/KMSJSONWriter.java | 6 +-
hadoop-dist/pom.xml | 5 +
.../src/main/compose/ozone/docker-compose.yaml | 12 -
.../src/main/compose/ozone/docker-config | 5 -
.../main/compose/ozoneperf/docker-compose.yaml | 13 -
.../src/main/compose/ozoneperf/docker-config | 5 -
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 -
.../container/common/helpers/ContainerInfo.java | 32 +
.../scm/container/common/helpers/Pipeline.java | 142 ++--
.../common/helpers/PipelineChannel.java | 124 ----
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 -
.../apache/hadoop/utils/BackgroundService.java | 2 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 8 +-
.../common/src/main/resources/ozone-default.xml | 47 --
.../hadoop/ozone/HddsDatanodeService.java | 13 +-
.../statemachine/DatanodeStateMachine.java | 3 +
.../background/BlockDeletingService.java | 8 +-
.../ReplicateContainerCommandHandler.java | 67 ++
.../states/endpoint/HeartbeatEndpointTask.java | 12 +
.../container/common/utils/HddsVolumeUtil.java | 2 +-
.../container/replication/ReplicationQueue.java | 76 ++
.../replication/ReplicationRequest.java | 106 +++
.../container/replication/package-info.java | 23 +
.../protocol/commands/CommandForDatanode.java | 45 ++
.../commands/ReplicateContainerCommand.java | 94 +++
.../StorageContainerDatanodeProtocol.proto | 12 +-
.../replication/TestReplicationQueue.java | 134 ++++
.../container/replication/package-info.java | 23 +
.../testutils/BlockDeletingServiceTestImpl.java | 4 +-
.../hadoop/hdds/server/events/EventWatcher.java | 157 +++++
.../server/events/IdentifiableEventPayload.java | 30 +
.../hdds/server/events/EventHandlerStub.java | 38 +
.../hdds/server/events/TestEventWatcher.java | 220 ++++++
.../hdds/scm/container/ContainerMapping.java | 10 +-
.../scm/container/closer/ContainerCloser.java | 6 +-
.../replication/ContainerSupervisor.java | 340 ---------
.../container/replication/InProgressPool.java | 255 -------
.../scm/container/replication/PeriodicPool.java | 119 ----
.../scm/container/replication/package-info.java | 23 -
.../hadoop/hdds/scm/node/NodeManager.java | 6 -
.../hadoop/hdds/scm/node/NodePoolManager.java | 71 --
.../hadoop/hdds/scm/node/SCMNodeManager.java | 41 +-
.../hdds/scm/node/SCMNodePoolManager.java | 269 --------
.../hdds/scm/pipelines/PipelineManager.java | 67 +-
.../hdds/scm/pipelines/PipelineSelector.java | 11 +-
.../scm/pipelines/ratis/RatisManagerImpl.java | 13 +-
.../standalone/StandaloneManagerImpl.java | 8 +-
.../server/SCMDatanodeHeartbeatDispatcher.java | 126 ++++
.../scm/server/SCMDatanodeProtocolServer.java | 25 +-
.../scm/server/StorageContainerManager.java | 10 +-
.../SCMDatanodeContainerReportHandler.java | 76 --
.../report/SCMDatanodeHeartbeatDispatcher.java | 189 -----
.../report/SCMDatanodeNodeReportHandler.java | 43 --
.../server/report/SCMDatanodeReportHandler.java | 83 ---
.../report/SCMDatanodeReportHandlerFactory.java | 82 ---
.../hdds/scm/server/report/package-info.java | 57 --
.../hdds/scm/block/TestDeletedBlockLog.java | 8 +-
.../hdds/scm/container/MockNodeManager.java | 6 -
.../hadoop/hdds/scm/node/TestNodeManager.java | 39 ++
.../hdds/scm/node/TestSCMNodePoolManager.java | 160 -----
.../TestSCMDatanodeHeartbeatDispatcher.java | 119 ++++
.../TestSCMDatanodeContainerReportHandler.java | 34 -
.../TestSCMDatanodeHeartbeatDispatcher.java | 138 ----
.../TestSCMDatanodeNodeReportHandler.java | 36 -
.../TestSCMDatanodeReportHandlerFactory.java | 51 --
.../hdds/scm/server/report/package-info.java | 21 -
.../testutils/ReplicationNodeManagerMock.java | 5 -
.../ReplicationNodePoolManagerMock.java | 133 ----
.../main/java/org/apache/hadoop/fs/XAttr.java | 4 +-
.../hdfs/DFSMultipartUploaderFactory.java | 40 ++
.../org/apache/hadoop/hdfs/ExtendedBlockId.java | 4 +-
.../hdfs/client/impl/BlockReaderFactory.java | 2 +-
.../AddErasureCodingPolicyResponse.java | 4 +-
.../hdfs/protocol/CacheDirectiveInfo.java | 4 +-
.../hadoop/hdfs/protocol/CachePoolInfo.java | 4 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 4 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 4 +-
.../hdfs/protocol/ErasureCodingPolicyInfo.java | 4 +-
.../hadoop/hdfs/protocol/ExtendedBlock.java | 2 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 2 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 2 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 4 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 +-
...onfRefreshTokenBasedAccessTokenProvider.java | 8 +-
.../CredentialBasedAccessTokenProvider.java | 8 +-
...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 +
.../ha/TestRequestHedgingProxyProvider.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 -
.../federation/router/ConnectionPoolId.java | 2 +-
.../router/RemoteLocationContext.java | 2 +-
.../store/driver/impl/StateStoreFileImpl.java | 2 +-
.../federation/store/records/MountTable.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +-
.../hadoop/hdfs/protocol/CacheDirective.java | 2 +-
.../qjournal/server/GetJournalEditServlet.java | 4 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 12 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 30 +-
.../server/blockmanagement/BlockManager.java | 2 +-
.../hdfs/server/datanode/DirectoryScanner.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetCache.java | 2 +-
.../server/diskbalancer/command/Command.java | 4 +-
.../diskbalancer/command/PlanCommand.java | 4 +-
.../diskbalancer/command/ReportCommand.java | 4 +-
.../server/namenode/EncryptionZoneManager.java | 4 +-
.../server/namenode/FSDirErasureCodingOp.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 6 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 17 +-
.../server/namenode/INodeAttributeProvider.java | 2 +-
.../server/namenode/startupprogress/Step.java | 6 +-
.../server/namenode/top/metrics/TopMetrics.java | 2 +-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 2 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 2 +-
.../src/main/resources/hdfs-default.xml | 10 +
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 2 +-
.../hadoop/fs/TestHDFSMultipartUploader.java | 76 ++
.../TestWebHdfsFileContextMainOperations.java | 2 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +-
.../apache/hadoop/hdfs/TestDecommission.java | 2 +-
.../hadoop/hdfs/TestFsShellPermission.java | 2 +-
.../hadoop/hdfs/TestHDFSPolicyProvider.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hdfs/TestSecureEncryptionZoneWithKMS.java | 34 +-
.../sasl/SaslDataTransferTestCase.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 81 ++-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../command/TestDiskBalancerCommand.java | 2 +-
.../hadoop/hdfs/server/mover/TestMover.java | 46 ++
.../hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hdfs/server/namenode/TestAuditLogger.java | 2 +-
.../server/namenode/TestCacheDirectives.java | 2 +-
.../namenode/TestEditLogJournalFailures.java | 2 +-
.../hdfs/server/namenode/TestEditLogRace.java | 6 +-
.../server/namenode/TestFSNamesystemLock.java | 10 +-
.../hdfs/server/namenode/TestStartup.java | 9 +-
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
.../shortcircuit/TestShortCircuitCache.java | 2 +-
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +-
.../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 12 +
.../org/apache/hadoop/tracing/TestTracing.java | 2 +-
.../v2/app/job/impl/TestTaskAttempt.java | 4 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 8 +-
.../lib/input/CombineFileInputFormat.java | 4 +-
.../mapred/uploader/FrameworkUploader.java | 2 +-
.../mapred/uploader/TestFrameworkUploader.java | 13 +
hadoop-maven-plugins/pom.xml | 38 +-
.../maven/plugin/resourcegz/ResourceGzMojo.java | 2 +-
.../dev-support/bin/robot-all.sh | 2 +-
.../dev-support/bin/robot-dnd-all.sh | 63 ++
.../acceptance-test/dev-support/bin/robot.sh | 9 +-
.../dev-support/docker/Dockerfile | 21 +
.../dev-support/docker/docker-compose.yaml | 23 +
hadoop-ozone/acceptance-test/pom.xml | 1 +
.../src/test/acceptance/basic/.env | 17 +
.../src/test/acceptance/basic/basic.robot | 50 ++
.../test/acceptance/basic/docker-compose.yaml | 50 ++
.../src/test/acceptance/basic/docker-config | 33 +
.../src/test/acceptance/basic/ozone-shell.robot | 85 +++
.../src/test/acceptance/commonlib.robot | 78 +++
.../src/test/acceptance/ozonefs/.env | 17 +
.../test/acceptance/ozonefs/docker-compose.yaml | 59 ++
.../src/test/acceptance/ozonefs/docker-config | 34 +
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 ++
.../acceptance-test/src/test/compose/.env | 17 -
.../src/test/compose/docker-compose.yaml | 62 --
.../src/test/compose/docker-config | 36 -
.../robotframework/acceptance/ozone-shell.robot | 256 -------
.../test/robotframework/acceptance/ozone.robot | 104 ---
hadoop-ozone/common/src/main/bin/ozone | 9 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 8 +-
.../ozone/container/ContainerTestHelper.java | 19 +-
.../common/TestBlockDeletingService.java | 11 +-
.../TestReplicateContainerHandler.java | 71 ++
...TestGenerateOzoneRequiredConfigurations.java | 2 +-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 31 -
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 253 -------
.../ozone-manager/src/main/webapps/ksm/ksm.js | 4 +-
hadoop-ozone/ozonefs/pom.xml | 211 ++++++
.../org/apache/hadoop/fs/ozone/Constants.java | 42 ++
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 ++
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 +++
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
.../apache/hadoop/fs/ozone/package-info.java | 30 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
.../fs/ozone/TestOzoneFileInterfaces.java | 231 +++++++
.../contract/ITestOzoneContractCreate.java | 48 ++
.../contract/ITestOzoneContractDelete.java | 48 ++
.../contract/ITestOzoneContractDistCp.java | 50 ++
.../ITestOzoneContractGetFileStatus.java | 61 ++
.../ozone/contract/ITestOzoneContractMkdir.java | 48 ++
.../ozone/contract/ITestOzoneContractOpen.java | 47 ++
.../contract/ITestOzoneContractRename.java | 49 ++
.../contract/ITestOzoneContractRootDir.java | 51 ++
.../ozone/contract/ITestOzoneContractSeek.java | 47 ++
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
.../src/test/resources/contract/ozone.xml | 113 +++
.../ozonefs/src/test/resources/log4j.properties | 23 +
hadoop-ozone/pom.xml | 1 +
.../genesis/BenchMarkContainerStateMap.java | 11 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 6 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 78 +--
hadoop-project/pom.xml | 16 +-
.../aliyun/oss/AliyunCredentialsProvider.java | 2 +-
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 2 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 2 +-
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 2 +-
.../fs/aliyun/oss/AliyunOSSTestUtils.java | 2 +-
hadoop-tools/hadoop-aws/pom.xml | 42 +-
.../fs/s3a/AWSCredentialProviderList.java | 2 +-
.../fs/s3a/BasicAWSCredentialsProvider.java | 2 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 11 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +-
.../apache/hadoop/fs/s3a/S3AInputStream.java | 2 +-
.../hadoop/fs/s3a/S3AMultipartUploader.java | 150 ++++
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 2 +-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 2 +-
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 2 +-
.../s3a/auth/AssumedRoleCredentialProvider.java | 2 +-
.../s3a/commit/files/SinglePendingCommit.java | 2 +-
.../hadoop/fs/s3a/commit/files/SuccessData.java | 2 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 2 +-
.../hadoop/fs/s3a/commit/staging/Paths.java | 2 +-
.../fs/s3a/s3guard/DynamoDBClientFactory.java | 2 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 3 +-
.../fs/s3a/s3guard/LocalMetadataEntry.java | 81 +++
.../fs/s3a/s3guard/LocalMetadataStore.java | 249 ++++---
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 6 +-
.../hadoop/fs/s3native/S3xLoginHelper.java | 4 +-
...rg.apache.hadoop.fs.MultipartUploaderFactory | 15 +
.../org.apache.hadoop.fs.MultipartUploader | 16 +
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 21 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 4 +-
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 2 +-
...onSSEKMSUserDefinedKeyBlockOutputStream.java | 2 +-
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 1 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 36 +-
.../fs/s3a/commit/AbstractCommitITest.java | 2 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 30 +-
.../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 -----
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 6 +-
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 589 ----------------
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 33 +-
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +-
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 2 +-
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +-
.../fs/azure/ClientThrottlingAnalyzer.java | 2 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 4 +-
.../hadoop/fs/azure/PageBlobOutputStream.java | 2 +-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 2 +-
.../fs/azure/SecureWasbRemoteCallHelper.java | 2 +-
.../hadoop/fs/azure/security/JsonUtils.java | 4 +-
.../fs/azure/ITestBlobDataValidation.java | 2 +-
.../azure/ITestNativeAzureFileSystemAppend.java | 2 +-
.../hadoop/fs/azure/MockStorageInterface.java | 12 +-
...estNativeAzureFileSystemBlockCompaction.java | 2 +-
.../fs/azure/integration/AzureTestUtils.java | 2 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 2 +-
.../org/apache/hadoop/tools/OptionsParser.java | 2 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 2 +-
.../mapred/gridmix/ClusterSummarizer.java | 2 +-
.../mapred/gridmix/ExecutionSummarizer.java | 4 +-
.../hadoop/mapred/gridmix/JobFactory.java | 2 +-
.../mapred/gridmix/RandomTextDataGenerator.java | 2 +-
hadoop-tools/hadoop-ozone/pom.xml | 174 -----
.../org/apache/hadoop/fs/ozone/Constants.java | 42 --
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 --
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ---
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 --
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
.../apache/hadoop/fs/ozone/package-info.java | 30 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
.../fs/ozone/TestOzoneFileInterfaces.java | 231 -------
.../contract/ITestOzoneContractCreate.java | 48 --
.../contract/ITestOzoneContractDelete.java | 48 --
.../contract/ITestOzoneContractDistCp.java | 50 --
.../ITestOzoneContractGetFileStatus.java | 61 --
.../ozone/contract/ITestOzoneContractMkdir.java | 48 --
.../ozone/contract/ITestOzoneContractOpen.java | 47 --
.../contract/ITestOzoneContractRename.java | 49 --
.../contract/ITestOzoneContractRootDir.java | 51 --
.../ozone/contract/ITestOzoneContractSeek.java | 47 --
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
.../src/test/resources/contract/ozone.xml | 113 ---
.../src/test/resources/log4j.properties | 23 -
.../translator/impl/BaseLogParser.java | 1 -
.../WordListAnonymizerUtility.java | 2 +-
.../util/MapReduceJobPropertiesParser.java | 2 +-
.../hadoop/streaming/TestUnconsumedInput.java | 2 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 15 -
hadoop-tools/pom.xml | 11 -
.../yarn/api/records/ResourceRequest.java | 20 +
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../exceptions/RestApiErrorMessages.java | 2 +
.../yarn/service/utils/ServiceApiUtil.java | 5 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 18 +
.../yarn/client/api/impl/AMRMClientImpl.java | 11 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 6 +-
.../yarn/client/api/impl/TestYarnClient.java | 337 ++-------
.../client/api/impl/TestYarnClientImpl.java | 254 +++++++
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 14 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 2 +-
.../jquery/jquery-ui-1.12.1.custom.min.js | 13 +
.../static/jquery/jquery-ui-1.9.1.custom.min.js | 6 -
.../registry/client/api/RegistryConstants.java | 2 +-
.../hadoop/registry/server/dns/RegistryDNS.java | 29 +-
.../registry/server/dns/TestRegistryDNS.java | 23 +-
.../hadoop/yarn/server/AMRMClientRelayer.java | 8 +-
.../LocalityMulticastAMRMProxyPolicy.java | 10 +-
.../server/scheduler/ResourceRequestSet.java | 14 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 12 -
.../src/CMakeLists.txt | 4 +-
.../server/nodemanager/ContainerExecutor.java | 4 +-
.../containermanager/container/Container.java | 6 +
.../container/ContainerImpl.java | 11 +
.../launcher/ContainerLaunch.java | 46 +-
.../launcher/ContainerRelaunch.java | 1 -
.../CGroupsMemoryResourceHandlerImpl.java | 24 +
.../linux/resources/MemoryResourceHandler.java | 10 +
.../resources/gpu/GpuResourceAllocator.java | 68 +-
.../resources/gpu/GpuResourceHandlerImpl.java | 1 -
.../localizer/ContainerLocalizer.java | 7 +-
.../localizer/ResourceLocalizationService.java | 5 +-
.../monitor/ContainersMonitorImpl.java | 108 +--
.../oom-listener/test/oom_listener_test_main.cc | 33 +-
.../container/TestContainer.java | 8 +-
.../launcher/TestContainerLaunch.java | 2 -
.../TestCGroupsMemoryResourceHandlerImpl.java | 43 ++
.../nodemanager/webapp/MockContainer.java | 3 +
.../server/resourcemanager/ResourceManager.java | 2 +-
.../scheduler/capacity/CSQueueUtils.java | 18 +-
.../scheduler/capacity/CapacityScheduler.java | 18 +-
.../fair/AllocationFileLoaderService.java | 4 +-
.../LocalityAppPlacementAllocator.java | 10 +-
.../webapp/dao/ClusterMetricsInfo.java | 2 +-
.../server/resourcemanager/Application.java | 10 +-
.../server/resourcemanager/TestAppManager.java | 18 +-
.../src/site/markdown/FairScheduler.md | 2 +-
.../site/markdown/yarn-service/RegistryDNS.md | 10 +-
.../markdown/yarn-service/ServiceDiscovery.md | 6 +-
.../hadoop-yarn-ui/src/main/webapp/.bowerrc | 3 +-
.../src/main/webapp/WEB-INF/web.xml | 6 +-
pom.xml | 2 +-
start-build-env.sh | 8 +-
437 files changed, 7828 insertions(+), 7627 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index b6a9bb9,b073d7b..245d76f
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@@ -93,8 -95,9 +95,9 @@@ public class DatanodeStateMachine imple
// trick.
commandDispatcher = CommandDispatcher.newBuilder()
.addHandler(new CloseContainerCommandHandler())
- .addHandler(new DeleteBlocksCommandHandler(
- container.getContainerManager(), conf))
+ .addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(),
+ conf))
+ .addHandler(new ReplicateContainerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 6809d57,0000000..5d6fc0a
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@@ -1,163 -1,0 +1,163 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.utils;
+
+import com.google.common.annotations.VisibleForTesting;
- import org.apache.commons.lang.StringUtils;
++import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
+import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.util.Time;
+
+import java.io.File;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * A util class for {@link HddsVolume}.
+ */
+public final class HddsVolumeUtil {
+
+ // Private constructor for Utility class. Unused.
+ private HddsVolumeUtil() {
+ }
+
+ private static final String VERSION_FILE = "VERSION";
+ private static final String STORAGE_ID_PREFIX = "DS-";
+
+ public static File getVersionFile(File rootDir) {
+ return new File(rootDir, VERSION_FILE);
+ }
+
+ public static String generateUuid() {
+ return STORAGE_ID_PREFIX + UUID.randomUUID();
+ }
+
+ /**
+ * Get hddsRoot from volume root. If volumeRoot points to hddsRoot, it is
+ * returned as is.
+ * For a volumeRoot /data/disk1, the hddsRoot is /data/disk1/hdds.
+ * @param volumeRoot root of the volume.
+ * @return hddsRoot of the volume.
+ */
+ public static String getHddsRoot(String volumeRoot) {
+ if (volumeRoot.endsWith(HddsVolume.HDDS_VOLUME_DIR)) {
+ return volumeRoot;
+ } else {
+ File hddsRoot = new File(volumeRoot, HddsVolume.HDDS_VOLUME_DIR);
+ return hddsRoot.getPath();
+ }
+ }
+
+ /**
+ * Returns storageID if it is valid. Throws an exception otherwise.
+ */
+ @VisibleForTesting
+ public static String getStorageID(Properties props, File versionFile)
+ throws InconsistentStorageStateException {
+ return getProperty(props, OzoneConsts.STORAGE_ID, versionFile);
+ }
+
+ /**
+ * Returns clusterID if it is valid. It should match the clusterID from the
+ * Datanode. Throws an exception otherwise.
+ */
+ @VisibleForTesting
+ public static String getClusterID(Properties props, File versionFile,
+ String clusterID) throws InconsistentStorageStateException {
+ String cid = getProperty(props, OzoneConsts.CLUSTER_ID, versionFile);
+
+ if (clusterID == null) {
+ return cid;
+ }
+ if (!clusterID.equals(cid)) {
+ throw new InconsistentStorageStateException("Mismatched " +
+ "ClusterIDs. Version File : " + versionFile + " has clusterID: " +
+ cid + " and Datanode has clusterID: " + clusterID);
+ }
+ return cid;
+ }
+
+ /**
+ * Returns datanodeUuid if it is valid. It should match the UUID of the
+ * Datanode. Throws an exception otherwise.
+ */
+ @VisibleForTesting
+ public static String getDatanodeUUID(Properties props, File versionFile,
+ String datanodeUuid)
+ throws InconsistentStorageStateException {
+ String datanodeID = getProperty(props, OzoneConsts.DATANODE_UUID,
+ versionFile);
+
+ if (datanodeUuid != null && !datanodeUuid.equals(datanodeID)) {
+ throw new InconsistentStorageStateException("Mismatched " +
+ "DatanodeUUIDs. Version File : " + versionFile + " has datanodeUuid: "
+ + datanodeID + " and Datanode has datanodeUuid: " + datanodeUuid);
+ }
+ return datanodeID;
+ }
+
+ /**
+ * Returns creationTime if it is valid. Throws an exception otherwise.
+ */
+ @VisibleForTesting
+ public static long getCreationTime(Properties props, File versionFile)
+ throws InconsistentStorageStateException {
+ String cTimeStr = getProperty(props, OzoneConsts.CTIME, versionFile);
+
+ long cTime = Long.parseLong(cTimeStr);
+ long currentTime = Time.now();
+ if (cTime > currentTime || cTime < 0) {
+ throw new InconsistentStorageStateException("Invalid Creation time in " +
+ "Version File : " + versionFile + " - " + cTime + ". Current system" +
+ " time is " + currentTime);
+ }
+ return cTime;
+ }
+
+ /**
+ * Returns layOutVersion if it is valid. Throws an exception otherwise.
+ */
+ @VisibleForTesting
+ public static int getLayOutVersion(Properties props, File versionFile) throws
+ InconsistentStorageStateException {
+ String lvStr = getProperty(props, OzoneConsts.LAYOUTVERSION, versionFile);
+
+ int lv = Integer.parseInt(lvStr);
+ if(DataNodeLayoutVersion.getLatestVersion().getVersion() != lv) {
+ throw new InconsistentStorageStateException("Invalid layOutVersion. " +
+ "Version file has layOutVersion as " + lv + " and latest Datanode " +
+ "layOutVersion is " +
+ DataNodeLayoutVersion.getLatestVersion().getVersion());
+ }
+ return lv;
+ }
+
+ private static String getProperty(Properties props, String propName, File
+ versionFile)
+ throws InconsistentStorageStateException {
+ String value = props.getProperty(propName);
+ if (StringUtils.isBlank(value)) {
+ throw new InconsistentStorageStateException("Invalid " + propName +
+ ". Version File : " + versionFile + " has null or empty " + propName);
+ }
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 9ac9930,adca8ea..ff5b9f1
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@@ -40,9 -44,9 +44,10 @@@ import org.apache.hadoop.hdds.protocol.
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
+ import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org