You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:44:58 UTC
[46/50] [abbrv] hadoop git commit: Merge commit
'4aacbfff605262aaf3dbd926258afcadc86c72c0' into HDFS-12943
Merge commit '4aacbfff605262aaf3dbd926258afcadc86c72c0' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d53c5997
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d53c5997
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d53c5997
Branch: refs/heads/HDFS-12943
Commit: d53c5997ac0bb08e1b108529faf428e7b7ccbb87
Parents: 066afcf 4aacbff
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Sat Aug 25 17:15:15 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sat Aug 25 17:15:15 2018 -0700
----------------------------------------------------------------------
README.txt | 2 +-
dev-support/bin/ozone-dist-layout-stitching | 2 +
.../main/resources/checkstyle/suppressions.xml | 1 -
.../hadoop-common/HadoopJNI.cmake | 2 +
.../key/kms/LoadBalancingKMSClientProvider.java | 43 +-
.../org/apache/hadoop/fs/shell/FsCommand.java | 2 +-
.../java/org/apache/hadoop/fs/shell/Touch.java | 85 ----
.../apache/hadoop/fs/shell/TouchCommands.java | 198 ++++++++
.../org/apache/hadoop/util/ServletUtil.java | 2 +-
.../src/site/markdown/FileSystemShell.md | 32 ++
.../kms/TestLoadBalancingKMSClientProvider.java | 181 +++++++-
.../org/apache/hadoop/fs/TestFsShellTouch.java | 103 +++++
.../fs/contract/AbstractContractRenameTest.java | 4 +-
.../fs/contract/AbstractContractSeekTest.java | 2 +-
.../fs/contract/AbstractFSContractTestBase.java | 6 +-
.../apache/hadoop/http/TestSSLHttpServer.java | 54 ++-
.../apache/hadoop/test/GenericTestUtils.java | 8 +-
.../src/test/resources/testConf.xml | 51 +++
.../src/main/resources/kms-default.xml | 8 +
hadoop-dist/pom.xml | 17 -
hadoop-dist/src/main/compose/README.md | 51 +++
.../src/main/compose/ozone/docker-compose.yaml | 6 +-
.../main/compose/ozoneperf/docker-compose.yaml | 6 +-
.../apache/hadoop/hdds/scm/XceiverClient.java | 5 +-
.../hadoop/hdds/scm/XceiverClientGrpc.java | 5 +-
.../hadoop/hdds/scm/XceiverClientRatis.java | 14 +-
.../scm/client/ContainerOperationClient.java | 5 +-
hadoop-hdds/common/pom.xml | 2 +-
.../hadoop/hdds/scm/XceiverClientSpi.java | 6 +-
.../container/common/helpers/ContainerInfo.java | 25 +-
.../scm/container/common/helpers/Pipeline.java | 38 +-
.../container/common/helpers/PipelineID.java | 88 ++++
.../org/apache/hadoop/utils/RocksDBStore.java | 2 +-
.../main/java/org/apache/ratis/RatisHelper.java | 19 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 8 +-
.../container/common/impl/ContainerSet.java | 3 +-
.../CloseContainerCommandHandler.java | 3 +-
.../common/transport/server/XceiverServer.java | 7 +-
.../transport/server/XceiverServerGrpc.java | 7 +-
.../transport/server/XceiverServerSpi.java | 6 +-
.../server/ratis/ContainerStateMachine.java | 323 ++++++++++---
.../server/ratis/XceiverServerRatis.java | 36 +-
.../container/ozoneimpl/OzoneContainer.java | 8 +-
.../commands/CloseContainerCommand.java | 19 +-
.../StorageContainerDatanodeProtocol.proto | 1 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 4 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 6 +-
.../hdds/scm/block/DeletedBlockLogImpl.java | 38 +-
.../hdds/scm/block/PendingDeleteHandler.java | 38 ++
.../hdds/scm/block/PendingDeleteStatusList.java | 79 ++++
.../hdds/scm/block/SCMBlockDeletingService.java | 20 +-
.../container/CloseContainerEventHandler.java | 2 +-
.../hdds/scm/container/ContainerMapping.java | 75 ++-
.../scm/container/ContainerStateManager.java | 28 +-
.../scm/container/closer/ContainerCloser.java | 4 +-
.../scm/container/states/ContainerStateMap.java | 18 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 9 +
.../hdds/scm/exceptions/SCMException.java | 3 +-
.../hadoop/hdds/scm/node/DatanodeInfo.java | 8 +
.../hdds/scm/pipelines/PipelineManager.java | 36 +-
.../hdds/scm/pipelines/PipelineSelector.java | 31 +-
.../scm/pipelines/ratis/RatisManagerImpl.java | 18 +-
.../standalone/StandaloneManagerImpl.java | 14 +-
.../scm/server/StorageContainerManager.java | 5 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 25 -
.../hdds/scm/block/TestDeletedBlockLog.java | 6 +-
.../scm/container/TestContainerMapping.java | 50 ++
.../container/TestContainerReportHandler.java | 4 +-
.../replication/TestReplicationManager.java | 5 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 4 +-
.../ozone/container/common/TestEndPoint.java | 2 +
.../hadoop/hdfs/DistributedFileSystem.java | 6 +-
.../hadoop-hdfs-native-client/pom.xml | 105 +++--
.../src/CMakeLists.txt | 48 ++
.../src/main/native/libhdfs/CMakeLists.txt | 6 +-
.../main/native/libhdfspp/tests/CMakeLists.txt | 4 +
.../hdfs/server/blockmanagement/BlockInfo.java | 4 +
.../server/blockmanagement/BlockManager.java | 4 +
.../server/blockmanagement/DatanodeManager.java | 7 +-
.../hdfs/server/datanode/BPServiceActor.java | 3 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 15 +
.../hdfs/server/datanode/DataNodeMXBean.java | 7 +
.../hdfs/server/datanode/DirectoryScanner.java | 2 +-
.../server/namenode/FSImageFormatPBINode.java | 28 +-
.../server/namenode/FSImageFormatProtobuf.java | 9 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../server/namenode/FSPermissionChecker.java | 2 +-
.../hdfs/server/namenode/SecondaryNameNode.java | 5 +
.../namenode/SecondaryNameNodeInfoMXBean.java | 7 +
.../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 12 +-
.../src/main/native/docs/libhdfs_footer.html | 2 +-
.../src/main/webapps/hdfs/explorer.js | 2 +-
.../src/site/markdown/HDFSErasureCoding.md | 8 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 6 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 7 +
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 6 +-
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 65 +++
.../org/apache/hadoop/hdfs/TestReplication.java | 6 +-
.../qjournal/client/TestIPCLoggerChannel.java | 6 +-
.../server/datanode/BlockReportTestBase.java | 8 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 3 +
.../server/datanode/TestDataNodeMXBean.java | 47 +-
.../server/datanode/TestDirectoryScanner.java | 42 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 13 +
.../hdfs/server/namenode/TestCheckpoint.java | 4 +-
.../hdfs/server/namenode/TestDeleteRace.java | 10 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 18 +
.../namenode/TestINodeAttributeProvider.java | 43 +-
.../TestNameNodeMetadataConsistency.java | 2 +-
.../hdfs/server/namenode/TestSaveNamespace.java | 10 +-
.../server/namenode/TestSecureNameNode.java | 52 ++-
.../hdfs/server/namenode/ha/TestDNFencing.java | 6 +-
.../namenode/ha/TestPipelinesFailover.java | 10 +-
.../namenode/ha/TestStandbyCheckpoints.java | 6 +-
.../security/TestRefreshUserMappings.java | 51 ++-
.../test/resources/testErasureCodingConf.xml | 6 +-
hadoop-mapreduce-project/pom.xml | 2 +-
hadoop-ozone/docs/content/GettingStarted.md | 2 +-
hadoop-ozone/docs/pom.xml | 17 +
hadoop-ozone/docs/static/OzoneOverview.svg | 13 +
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 8 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 10 +-
.../hadoop/ozone/TestMiniOzoneCluster.java | 3 +-
.../ozone/container/ContainerTestHelper.java | 5 +-
.../commandhandler/TestBlockDeletion.java | 57 ++-
.../TestCloseContainerByPipeline.java | 41 +-
.../TestCloseContainerHandler.java | 7 +-
.../server/TestContainerStateMachine.java | 201 ++++++++
.../hadoop/ozone/om/TestOzoneManager.java | 11 +-
.../apache/hadoop/ozone/om/package-info.java | 22 +
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 37 +-
.../web/ozShell/volume/UpdateVolumeHandler.java | 5 +
.../ITestOzoneContractGetFileStatus.java | 2 +-
hadoop-ozone/pom.xml | 10 -
.../genesis/BenchMarkContainerStateMap.java | 13 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 3 +-
hadoop-project/pom.xml | 4 +-
.../fs/aliyun/oss/AliyunOSSInputStream.java | 6 +-
.../apache/hadoop/tools/HadoopArchiveLogs.java | 6 +-
.../hadoop/tools/HadoopArchiveLogsRunner.java | 8 +-
.../org/apache/hadoop/tools/HadoopArchives.java | 6 +-
.../hadoop/fs/s3a/s3guard/DDBPathMetadata.java | 77 ++++
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 130 ++++--
.../PathMetadataDynamoDBTranslation.java | 71 ++-
.../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 4 +
.../site/markdown/tools/hadoop-aws/s3guard.md | 5 +-
.../site/markdown/tools/hadoop-aws/testing.md | 157 +++++++
.../s3a/ITestS3AContractGetFileStatus.java | 2 +-
.../ITestS3AContractGetFileStatusV1List.java | 2 +-
.../s3guard/AbstractS3GuardToolTestBase.java | 13 +-
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 7 +-
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 11 +
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 49 ++
.../TestPathMetadataDynamoDBTranslation.java | 47 +-
.../hadoop/contrib/utils/join/JobBase.java | 6 +-
.../org/apache/hadoop/tools/CopyListing.java | 6 +-
.../java/org/apache/hadoop/tools/DistCp.java | 6 +-
.../apache/hadoop/tools/GlobbedCopyListing.java | 6 +-
.../org/apache/hadoop/tools/OptionsParser.java | 6 +-
.../apache/hadoop/tools/RegexCopyFilter.java | 8 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 10 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 6 +-
.../tools/mapred/RetriableFileCopyCommand.java | 8 +-
.../tools/mapred/UniformSizeInputFormat.java | 8 +-
.../tools/mapred/lib/DynamicInputChunk.java | 8 +-
.../mapred/lib/DynamicInputChunkContext.java | 6 +-
.../tools/mapred/lib/DynamicInputFormat.java | 6 +-
.../tools/mapred/lib/DynamicRecordReader.java | 6 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 6 +-
.../hadoop/tools/util/ProducerConsumer.java | 6 +-
.../hadoop/tools/util/RetriableCommand.java | 6 +-
.../apache/hadoop/tools/TestCopyListing.java | 6 +-
.../apache/hadoop/tools/TestDistCpSystem.java | 8 +-
.../apache/hadoop/tools/TestDistCpViewFs.java | 6 +-
.../apache/hadoop/tools/TestExternalCall.java | 6 +-
.../hadoop/tools/TestFileBasedCopyListing.java | 6 +-
.../apache/hadoop/tools/TestIntegration.java | 6 +-
.../contract/AbstractContractDistCpTest.java | 2 +-
.../hadoop/tools/mapred/TestCopyCommitter.java | 6 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 6 +-
.../tools/mapred/TestCopyOutputFormat.java | 6 +-
.../mapred/lib/TestDynamicInputFormat.java | 6 +-
.../hadoop/tools/util/TestDistCpUtils.java | 6 +-
.../tools/util/TestThrottledInputStream.java | 8 +-
.../java/org/apache/hadoop/tools/DistTool.java | 6 +-
.../mapred/gridmix/ClusterSummarizer.java | 6 +-
.../gridmix/CompressionEmulationUtil.java | 6 +-
.../gridmix/DistributedCacheEmulator.java | 8 +-
.../hadoop/mapred/gridmix/EchoUserResolver.java | 6 +-
.../mapred/gridmix/ExecutionSummarizer.java | 6 +-
.../apache/hadoop/mapred/gridmix/FilePool.java | 6 +-
.../hadoop/mapred/gridmix/GridmixJob.java | 6 +-
.../hadoop/mapred/gridmix/InputStriper.java | 6 +-
.../hadoop/mapred/gridmix/JobFactory.java | 6 +-
.../hadoop/mapred/gridmix/JobMonitor.java | 6 +-
.../hadoop/mapred/gridmix/JobSubmitter.java | 6 +-
.../apache/hadoop/mapred/gridmix/LoadJob.java | 6 +-
.../mapred/gridmix/RandomTextDataGenerator.java | 6 +-
.../hadoop/mapred/gridmix/ReplayJobFactory.java | 6 +-
.../mapred/gridmix/RoundRobinUserResolver.java | 6 +-
.../hadoop/mapred/gridmix/SerialJobFactory.java | 6 +-
.../apache/hadoop/mapred/gridmix/SleepJob.java | 6 +-
.../hadoop/mapred/gridmix/Statistics.java | 6 +-
.../hadoop/mapred/gridmix/StressJobFactory.java | 6 +-
.../mapred/gridmix/SubmitterUserResolver.java | 6 +-
.../hadoop/mapred/gridmix/CommonJobTest.java | 6 +-
.../hadoop/mapred/gridmix/DebugJobProducer.java | 6 +-
.../hadoop/mapred/gridmix/GridmixTestUtils.java | 6 +-
.../hadoop/mapred/gridmix/TestFilePool.java | 6 +-
.../hadoop/mapred/gridmix/TestFileQueue.java | 6 +-
.../mapred/gridmix/TestGridMixClasses.java | 6 +-
.../mapred/gridmix/TestGridmixRecord.java | 6 +-
.../mapred/gridmix/TestRecordFactory.java | 6 +-
.../tools/rumen/DeskewedJobTraceReader.java | 8 +-
.../org/apache/hadoop/tools/rumen/Folder.java | 6 +-
.../hadoop/tools/rumen/HadoopLogsAnalyzer.java | 6 +-
.../hadoop/tools/rumen/HistoryEventEmitter.java | 6 +-
.../apache/hadoop/tools/rumen/JobBuilder.java | 6 +-
.../apache/hadoop/tools/rumen/ParsedJob.java | 6 +-
.../apache/hadoop/tools/rumen/ParsedTask.java | 6 +-
.../hadoop/tools/rumen/ParsedTaskAttempt.java | 6 +-
.../hadoop/tools/rumen/RandomSeedGenerator.java | 6 +-
.../apache/hadoop/tools/rumen/TraceBuilder.java | 8 +-
.../apache/hadoop/tools/rumen/ZombieJob.java | 6 +-
.../sls/resourcemanager/MockAMLauncher.java | 6 +-
.../hadoop/yarn/sls/synthetic/SynthJob.java | 6 +-
.../sls/synthetic/SynthTraceJobProducer.java | 6 +-
.../org/apache/hadoop/streaming/PipeMapRed.java | 17 +-
.../streaming/StreamBaseRecordReader.java | 5 +-
.../org/apache/hadoop/streaming/StreamJob.java | 6 +-
.../mapreduce/StreamBaseRecordReader.java | 8 +-
.../streaming/TestMultipleArchiveFiles.java | 6 +-
.../streaming/TestStreamXmlMultipleRecords.java | 6 +-
.../streaming/TestStreamingBadRecords.java | 8 +-
.../org/apache/hadoop/streaming/UtilTest.java | 6 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +-
.../hadoop-yarn-services-api/pom.xml | 57 +++
.../yarn/service/client/ApiServiceClient.java | 83 +++-
.../client/TestSecureApiServiceClient.java | 83 ++++
.../yarn/service/component/Component.java | 3 +-
.../component/instance/ComponentInstance.java | 2 +-
.../containerlaunch/AbstractLauncher.java | 5 +
.../containerlaunch/ContainerLaunchService.java | 11 +
.../provider/AbstractProviderService.java | 5 +-
.../defaultImpl/DefaultProviderService.java | 4 +-
.../provider/docker/DockerProviderService.java | 30 +-
.../tarball/TarballProviderService.java | 7 +-
.../instance/TestComponentInstance.java | 4 +-
.../provider/TestAbstractProviderService.java | 138 ++++++
.../hadoop-yarn-submarine/README.md | 53 +++
.../hadoop-yarn-submarine/pom.xml | 213 +++++++++
.../yarn/submarine/client/cli/AbstractCli.java | 47 ++
.../hadoop/yarn/submarine/client/cli/Cli.java | 106 +++++
.../yarn/submarine/client/cli/CliConstants.java | 48 ++
.../yarn/submarine/client/cli/CliUtils.java | 174 +++++++
.../yarn/submarine/client/cli/RunJobCli.java | 204 +++++++++
.../yarn/submarine/client/cli/ShowJobCli.java | 125 +++++
.../client/cli/param/BaseParameters.java | 56 +++
.../client/cli/param/RunJobParameters.java | 222 +++++++++
.../client/cli/param/RunParameters.java | 103 +++++
.../client/cli/param/ShowJobParameters.java | 18 +
.../yarn/submarine/common/ClientContext.java | 77 ++++
.../hadoop/yarn/submarine/common/Envs.java | 27 ++
.../common/api/JobComponentStatus.java | 73 +++
.../yarn/submarine/common/api/JobState.java | 52 +++
.../yarn/submarine/common/api/JobStatus.java | 87 ++++
.../yarn/submarine/common/api/TaskType.java | 32 ++
.../api/builder/JobComponentStatusBuilder.java | 44 ++
.../common/api/builder/JobStatusBuilder.java | 64 +++
.../common/conf/SubmarineConfiguration.java | 51 +++
.../submarine/common/conf/SubmarineLogs.java | 31 ++
.../common/exception/SubmarineException.java | 21 +
.../exception/SubmarineRuntimeException.java | 25 +
.../fs/DefaultRemoteDirectoryManager.java | 84 ++++
.../common/fs/RemoteDirectoryManager.java | 30 ++
.../yarn/submarine/runtimes/RuntimeFactory.java | 106 +++++
.../common/FSBasedSubmarineStorageImpl.java | 106 +++++
.../submarine/runtimes/common/JobMonitor.java | 84 ++++
.../submarine/runtimes/common/JobSubmitter.java | 36 ++
.../runtimes/common/StorageKeyConstants.java | 24 +
.../runtimes/common/SubmarineStorage.java | 57 +++
.../yarnservice/YarnServiceJobMonitor.java | 46 ++
.../yarnservice/YarnServiceJobSubmitter.java | 458 +++++++++++++++++++
.../yarnservice/YarnServiceRuntimeFactory.java | 44 ++
.../runtimes/yarnservice/YarnServiceUtils.java | 78 ++++
.../src/site/DeveloperGuide.md | 26 ++
.../src/site/QuickStart.md | 134 ++++++
.../client/cli/TestRunJobCliParsing.java | 229 ++++++++++
.../client/cli/TestShowJobCliParsing.java | 104 +++++
.../yarnservice/TestYarnServiceRunJobCli.java | 167 +++++++
.../yarnservice/YarnServiceCliTestUtils.java | 35 ++
.../submarine/common/MockClientContext.java | 56 +++
.../common/fs/MockRemoteDirectoryManager.java | 83 ++++
.../runtimes/common/MemorySubmarineStorage.java | 74 +++
.../common/TestFSBasedSubmarineStorage.java | 73 +++
.../yarnservice/TestTFConfigGenerator.java | 42 ++
.../src/test/resources/core-site.xml | 21 +
.../src/test/resources/hdfs-site.xml | 21 +
.../hadoop-yarn-applications/pom.xml | 1 +
.../logaggregation/AggregatedLogFormat.java | 3 +-
.../filecontroller/LogAggregationHtmlBlock.java | 61 +++
.../ifile/IndexedFileAggregatedLogsBlock.java | 136 ++----
.../tfile/TFileAggregatedLogsBlock.java | 56 +--
.../server/nodemanager/ContainerExecutor.java | 62 ++-
.../yarn/server/nodemanager/NodeManager.java | 20 +-
.../server/nodemanager/NodeManagerMXBean.java | 38 ++
.../launcher/ContainerLaunch.java | 51 ++-
.../runtime/DockerLinuxContainerRuntime.java | 4 +-
.../impl/container-executor.c | 92 ++--
.../impl/container-executor.h | 3 +-
.../main/native/container-executor/impl/main.c | 26 +-
.../nodemanager/TestContainerExecutor.java | 31 +-
.../nodemanager/TestNodeManagerMXBean.java | 56 +++
.../runtime/TestDockerContainerRuntime.java | 8 +-
.../server/resourcemanager/ResourceManager.java | 18 +-
.../resourcemanager/ResourceManagerMXBean.java | 38 ++
.../TestResourceManagerMXBean.java | 56 +++
.../TestAMLaunchFailure.java | 2 +-
.../TestSchedulerNegotiator.java | 2 +-
.../TestTimelineServiceClientIntegration.java | 3 +-
.../security/TestTimelineAuthFilterForV2.java | 2 +-
.../PerNodeTimelineCollectorsAuxService.java | 13 +-
323 files changed, 8429 insertions(+), 1311 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d53c5997/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d53c5997/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org