You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/07/09 20:33:20 UTC
[11/37] hadoop git commit: Merge remote-tracking branch
'apache-commit/trunk' into HDDS-48
Merge remote-tracking branch 'apache-commit/trunk' into HDDS-48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/418cff48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/418cff48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/418cff48
Branch: refs/heads/trunk
Commit: 418cff4820ba73cdbfd09fc5879b8b3aa4e62d5f
Parents: 7e228e5 9119b3c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jun 14 09:38:20 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jun 14 09:38:20 2018 -0700
----------------------------------------------------------------------
LICENSE.txt | 4 +-
.../main/resources/checkstyle/suppressions.xml | 1 +
.../hadoop-client-minicluster/pom.xml | 154 +-
.../src/main/bin/hadoop-functions.sh | 34 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 34 +
.../hadoop/fs/FileSystemStorageStatistics.java | 5 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 13 +
.../org/apache/hadoop/fs/FsUrlConnection.java | 2 +-
.../org/apache/hadoop/http/HttpServer2.java | 7 +-
.../rawcoder/AbstractNativeRawDecoder.java | 51 +-
.../rawcoder/AbstractNativeRawEncoder.java | 49 +-
.../rawcoder/NativeRSRawDecoder.java | 19 +-
.../rawcoder/NativeRSRawEncoder.java | 19 +-
.../rawcoder/NativeXORRawDecoder.java | 19 +-
.../rawcoder/NativeXORRawEncoder.java | 19 +-
.../security/IngressPortBasedResolver.java | 100 +
.../hadoop/security/SaslPropertiesResolver.java | 47 +-
.../hadoop/security/WhitelistBasedResolver.java | 20 +-
.../main/java/org/apache/hadoop/util/Shell.java | 18 +-
.../hadoop/util/curator/ZKCuratorManager.java | 14 +-
.../hadoop-common/src/site/markdown/Metrics.md | 12 +
.../markdown/release/3.0.3/CHANGES.3.0.3.md | 309 +
.../release/3.0.3/RELEASENOTES.3.0.3.md | 31 +
.../fs/TestFileSystemStorageStatistics.java | 6 +-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 206 +
.../rawcoder/RawErasureCoderBenchmark.java | 6 +
.../security/TestIngressPortBasedResolver.java | 59 +
.../org/apache/hadoop/test/MetricsAsserts.java | 22 +-
.../util/TestCloseableReferenceCount.java | 91 +
.../org/apache/hadoop/util/TestDiskChecker.java | 6 +-
.../hadoop/util/TestIntrusiveCollection.java | 193 +
.../hadoop/util/TestLimitInputStream.java | 74 +
.../java/org/apache/hadoop/util/TestShell.java | 8 +
.../org/apache/hadoop/util/TestStringUtils.java | 27 +
.../hadoop/util/TestUTF8ByteArrayUtils.java | 57 +
.../src/test/scripts/hadoop_stop_daemon.bats | 24 +-
.../src/main/resources/kms-default.xml | 35 +
.../apache/hadoop/oncrpc/security/Verifier.java | 12 +-
hadoop-dist/src/main/compose/ozoneperf/.env | 17 +
.../src/main/compose/ozoneperf/README.md | 73 +
.../src/main/compose/ozoneperf/compose-all.sh | 18 +
.../compose/ozoneperf/docker-compose-freon.yaml | 26 +
.../main/compose/ozoneperf/docker-compose.yaml | 77 +
.../src/main/compose/ozoneperf/docker-config | 37 +
hadoop-dist/src/main/compose/ozoneperf/init.sh | 21 +
.../src/main/compose/ozoneperf/prometheus.yml | 24 +
.../hadoop/hdds/scm/XceiverClientGrpc.java | 23 +
.../hadoop/hdds/scm/XceiverClientHandler.java | 2 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 2 +-
.../hadoop/hdds/conf/HddsConfServlet.java | 5 +-
.../container/common/helpers/ContainerInfo.java | 27 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 2 +
.../org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../apache/hadoop/utils/MetadataKeyFilters.java | 119 +-
.../main/java/org/apache/ratis/RatisHelper.java | 4 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 1 +
.../apache/hadoop/utils/TestMetadataStore.java | 78 +-
.../container/common/helpers/ContainerData.java | 24 +-
.../common/helpers/ContainerReport.java | 12 +
.../common/impl/ContainerManagerImpl.java | 31 +-
.../common/impl/ContainerReportManagerImpl.java | 67 -
.../container/common/impl/KeyManagerImpl.java | 12 +-
.../interfaces/ContainerReportManager.java | 30 -
.../common/report/ContainerReportPublisher.java | 70 +
.../common/report/NodeReportPublisher.java | 40 +
.../container/common/report/ReportManager.java | 148 +
.../common/report/ReportPublisher.java | 96 +
.../common/report/ReportPublisherFactory.java | 71 +
.../container/common/report/package-info.java | 80 +
.../statemachine/DatanodeStateMachine.java | 27 +-
.../common/statemachine/StateContext.java | 59 +-
.../background/BlockDeletingService.java | 16 +-
.../CloseContainerCommandHandler.java | 21 +-
.../commandhandler/CommandDispatcher.java | 4 +
.../DeleteBlocksCommandHandler.java | 3 +
.../states/endpoint/HeartbeatEndpointTask.java | 30 +-
.../states/endpoint/RegisterEndpointTask.java | 2 +-
.../common/transport/server/XceiverServer.java | 7 +
.../transport/server/XceiverServerGrpc.java | 9 +
.../transport/server/XceiverServerSpi.java | 7 +
.../server/ratis/ContainerStateMachine.java | 3 +-
.../server/ratis/XceiverServerRatis.java | 56 +-
.../container/common/utils/ContainerCache.java | 6 +-
.../container/ozoneimpl/OzoneContainer.java | 62 +-
.../commands/CloseContainerCommand.java | 12 +-
.../StorageContainerDatanodeProtocol.proto | 2 +
.../ozone/container/common/ScmTestMock.java | 32 +
.../ozone/container/common/package-info.java | 22 +
.../common/report/TestReportManager.java | 52 +
.../common/report/TestReportPublisher.java | 185 +
.../report/TestReportPublisherFactory.java | 68 +
.../container/common/report/package-info.java | 22 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 9 +-
.../hadoop/hdds/scm/block/DeletedBlockLog.java | 3 +-
.../hdds/scm/block/DeletedBlockLogImpl.java | 8 +-
.../container/CloseContainerEventHandler.java | 84 +
.../hdds/scm/container/ContainerMapping.java | 40 +
.../scm/container/ContainerStateManager.java | 21 +
.../hadoop/hdds/scm/container/Mapping.java | 17 +
.../scm/container/closer/ContainerCloser.java | 7 +-
.../scm/server/SCMDatanodeProtocolServer.java | 119 +-
.../SCMDatanodeContainerReportHandler.java | 76 +
.../report/SCMDatanodeHeartbeatDispatcher.java | 189 +
.../report/SCMDatanodeNodeReportHandler.java | 43 +
.../server/report/SCMDatanodeReportHandler.java | 83 +
.../report/SCMDatanodeReportHandlerFactory.java | 82 +
.../hdds/scm/server/report/package-info.java | 57 +
.../server-scm/src/main/webapps/scm/index.html | 6 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 16 +
.../TestCloseContainerEventHandler.java | 177 +
.../scm/container/TestContainerMapping.java | 6 +-
.../container/closer/TestContainerCloser.java | 3 +-
.../TestSCMDatanodeContainerReportHandler.java | 34 +
.../TestSCMDatanodeHeartbeatDispatcher.java | 138 +
.../TestSCMDatanodeNodeReportHandler.java | 36 +
.../TestSCMDatanodeReportHandlerFactory.java | 51 +
.../hdds/scm/server/report/package-info.java | 21 +
.../hdfs/CannotObtainBlockLengthException.java | 55 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 6 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 10 +-
.../org/apache/hadoop/hdfs/ReaderStrategy.java | 8 +
.../ha/ConfiguredFailoverProxyProvider.java | 39 +-
...apache.hadoop.security.token.TokenIdentifier | 17 +
...rg.apache.hadoop.security.token.TokenRenewer | 16 +
.../hadoop/fs/TestUrlStreamHandlerFactory.java | 36 +-
.../ha/TestConfiguredFailoverProxyProvider.java | 264 +
.../native/libhdfspp/include/hdfspp/ioservice.h | 1 +
.../native/libhdfspp/lib/reader/datatransfer.h | 2 +-
.../native/libhdfspp/lib/rpc/sasl_protocol.cc | 15 +-
.../native/libhdfspp/tests/mock_connection.h | 2 +-
.../libhdfspp/tests/remote_block_reader_test.cc | 2 +-
.../federation/router/ConnectionPool.java | 12 +-
.../driver/impl/StateStoreZooKeeperImpl.java | 6 +-
.../main/webapps/router/federationhealth.html | 6 +-
.../router/TestConnectionManager.java | 13 +
.../jdiff/Apache_Hadoop_HDFS_3.0.3.xml | 322 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +-
.../BlockPlacementPolicyDefault.java | 10 +-
.../server/blockmanagement/DatanodeManager.java | 2 +-
.../datanode/checker/DatasetVolumeChecker.java | 13 +-
.../server/namenode/FSDirErasureCodingOp.java | 23 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 10 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 23 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 16 +-
.../hdfs/server/namenode/FSNamesystem.java | 35 +-
.../server/namenode/FSPermissionChecker.java | 2 +-
.../hdfs/server/namenode/ha/EditLogTailer.java | 38 +-
.../namenode/metrics/NameNodeMetrics.java | 59 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +
.../OfflineImageViewerPB.java | 3 +-
.../offlineImageViewer/WebImageViewer.java | 17 +
...apache.hadoop.security.token.TokenIdentifier | 17 -
...rg.apache.hadoop.security.token.TokenRenewer | 16 -
.../src/main/resources/hdfs-default.xml | 12 +
.../src/main/webapps/datanode/datanode.html | 6 +-
.../src/main/webapps/hdfs/dfshealth.html | 6 +-
.../src/main/webapps/hdfs/dfshealth.js | 8 +-
.../src/main/webapps/hdfs/explorer.html | 10 +-
.../src/main/webapps/hdfs/explorer.js | 34 +-
.../src/main/webapps/journal/index.html | 6 +-
.../src/main/webapps/secondary/status.html | 6 +-
.../bootstrap-3.0.2/css/bootstrap-editable.css | 655 -
.../bootstrap-3.0.2/css/bootstrap.min.css | 9 -
.../fonts/glyphicons-halflings-regular.eot | Bin 20290 -> 0 bytes
.../fonts/glyphicons-halflings-regular.svg | 229 -
.../fonts/glyphicons-halflings-regular.ttf | Bin 41236 -> 0 bytes
.../fonts/glyphicons-halflings-regular.woff | Bin 23292 -> 0 bytes
.../static/bootstrap-3.0.2/img/clear.png | Bin 509 -> 0 bytes
.../static/bootstrap-3.0.2/img/loading.gif | Bin 1849 -> 0 bytes
.../js/bootstrap-editable.min.js | 7 -
.../static/bootstrap-3.0.2/js/bootstrap.min.js | 9 -
.../bootstrap-3.3.7/css/bootstrap-editable.css | 655 +
.../bootstrap-3.3.7/css/bootstrap-theme.css | 587 +
.../bootstrap-3.3.7/css/bootstrap-theme.css.map | 1 +
.../bootstrap-3.3.7/css/bootstrap-theme.min.css | 6 +
.../css/bootstrap-theme.min.css.map | 1 +
.../static/bootstrap-3.3.7/css/bootstrap.css | 6757 ++++++++
.../bootstrap-3.3.7/css/bootstrap.css.map | 1 +
.../bootstrap-3.3.7/css/bootstrap.min.css | 6 +
.../bootstrap-3.3.7/css/bootstrap.min.css.map | 1 +
.../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes
.../fonts/glyphicons-halflings-regular.svg | 288 +
.../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes
.../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes
.../fonts/glyphicons-halflings-regular.woff2 | Bin 0 -> 18028 bytes
.../js/bootstrap-editable.min.js | 7 +
.../static/bootstrap-3.3.7/js/bootstrap.js | 2377 +++
.../static/bootstrap-3.3.7/js/bootstrap.min.js | 7 +
.../webapps/static/bootstrap-3.3.7/js/npm.js | 13 +
.../src/main/webapps/static/dfs-dust.js | 2 +-
.../main/webapps/static/jquery-1.10.2.min.js | 6 -
.../src/main/webapps/static/jquery-3.3.1.min.js | 2 +
.../src/main/webapps/static/moment.min.js | 6 +-
.../src/site/markdown/HdfsImageViewer.md | 3 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 5 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 8 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 16 +
.../apache/hadoop/hdfs/TestEncryptionZones.java | 29 +
.../hdfs/TestErasureCodingExerciseAPIs.java | 2 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
.../hdfs/server/datanode/TestBlockScanner.java | 3 +-
.../checker/TestDatasetVolumeChecker.java | 3 +
.../namenode/TestReencryptionHandler.java | 7 +-
.../server/namenode/ha/TestEditLogTailer.java | 13 +-
.../namenode/metrics/TestNameNodeMetrics.java | 41 +
.../TestGetContentSummaryWithSnapshot.java | 26 +
.../apache/hadoop/hdfs/tools/TestGetConf.java | 6 +-
.../TestOfflineImageViewer.java | 20 +
.../hadoop/tools/TestHdfsConfigFields.java | 1 +
.../hadoop-hdfs/src/test/resources/editsStored | Bin 7909 -> 7909 bytes
.../src/test/resources/editsStored.xml | 2 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 9 +-
.../java/org/apache/hadoop/mapred/Task.java | 14 +-
.../hadoop/mapreduce/FileSystemCounter.java | 1 +
.../src/main/resources/mapred-default.xml | 9 +
.../mapreduce/FileSystemCounter.properties | 1 +
.../jobhistory/TestHistoryViewerPrinter.java | 378 +-
.../lib/output/TestFileOutputCommitter.java | 16 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 8 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 18 +-
.../mapreduce/v2/hs/webapp/HsJobsBlock.java | 40 +-
.../mapreduce/v2/hs/webapp/TestBlocks.java | 17 +-
.../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 20 +-
.../hadoop/mapred/TestJavaSerialization.java | 23 +-
.../TestNativeCollectorOnlyHandler.java | 11 +-
.../robotframework/acceptance/ozone-shell.robot | 39 +-
.../test/robotframework/acceptance/ozone.robot | 4 +-
.../hadoop/ozone/client/OzoneClientUtils.java | 18 +-
.../hadoop/ozone/client/rest/RestClient.java | 113 +-
.../ozone/client/rest/response/KeyInfo.java | 4 +-
.../hadoop/ozone/web/response/KeyInfo.java | 4 +-
hadoop-ozone/docs/content/CommandShell.md | 141 +-
hadoop-ozone/docs/content/GettingStarted.md | 353 +-
.../apache/hadoop/ozone/RatisTestHelper.java | 8 +-
.../ozone/TestOzoneConfigurationFields.java | 1 +
.../TestStorageContainerManagerHelper.java | 8 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 2 +-
.../common/TestBlockDeletingService.java | 29 +-
.../TestCloseContainerByPipeline.java | 221 +
.../TestCloseContainerHandler.java | 7 +-
...TestGenerateOzoneRequiredConfigurations.java | 72 +-
.../ozone/ksm/TestContainerReportWithKeys.java | 2 +-
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 16 +-
.../ozone/ksm/TestKsmBlockVersioning.java | 2 +-
.../ksm/TestMultipleContainerReadWrite.java | 2 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 35 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 20 +-
.../ozone/scm/TestXceiverClientManager.java | 8 +-
.../ozone/web/TestOzoneRestWithMiniCluster.java | 2 +-
.../hadoop/ozone/web/client/TestBuckets.java | 37 +-
.../ozone/web/client/TestBucketsRatis.java | 32 +-
.../hadoop/ozone/web/client/TestKeys.java | 37 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 +-
.../hadoop/ozone/web/client/TestVolume.java | 38 +-
.../ozone/web/client/TestVolumeRatis.java | 29 +-
.../web/storage/DistributedStorageHandler.java | 21 +-
.../ozone/ksm/KSMMetadataManagerImpl.java | 9 +-
.../hadoop/ozone/web/ozShell/Handler.java | 49 +-
.../src/main/webapps/ksm/index.html | 6 +-
.../hadoop/ozone/ksm/TestChunkStreams.java | 2 +-
hadoop-ozone/pom.xml | 14 +-
.../org/apache/hadoop/ozone/freon/Freon.java | 6 +-
.../genesis/BenchMarkContainerStateMap.java | 44 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 2 +-
.../genesis/BenchMarkMetadataStoreReads.java | 2 +-
.../genesis/BenchMarkMetadataStoreWrites.java | 2 +-
.../ozone/genesis/BenchMarkRocksDbStore.java | 2 +-
.../hadoop/ozone/genesis/GenesisUtil.java | 2 +-
hadoop-project-dist/pom.xml | 2 +-
hadoop-project/pom.xml | 11 +-
.../fs/azure/NativeAzureFileSystemHelper.java | 4 +-
.../hadoop/fs/azure/ITestContainerChecks.java | 9 +-
.../src/main/html/js/thirdparty/jquery.js | 13607 +++++++++--------
.../yarn/sls/appmaster/TestAMSimulator.java | 13 +-
.../hadoop-yarn/conf/yarn-env.sh | 1 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 -
.../protocolrecords/GetApplicationsRequest.java | 22 +-
.../hadoop/yarn/api/records/Resource.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +
.../yarn/util/resource/ResourceUtils.java | 4 +-
.../pom.xml | 4 -
.../distributedshell/ApplicationMaster.java | 24 +-
.../applications/distributedshell/Client.java | 16 +-
.../hadoop-yarn-services-api/pom.xml | 16 +
.../yarn/service/client/ApiServiceClient.java | 13 +-
.../client/SystemServiceManagerImpl.java | 29 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 3 +-
.../hadoop/yarn/service/ServiceClientTest.java | 4 +-
.../yarn/service/TestCleanupAfterKill.java | 94 +
.../client/TestSystemServiceManagerImpl.java | 40 +-
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop-yarn-services-core/pom.xml | 5 -
.../hadoop/yarn/service/ServiceMaster.java | 56 +-
.../yarn/service/api/records/Configuration.java | 2 +-
.../yarn/service/client/ServiceClient.java | 51 +-
.../yarn/service/monitor/probe/Probe.java | 2 +-
.../provider/AbstractClientProvider.java | 2 +-
.../provider/AbstractProviderService.java | 2 +-
.../provider/docker/DockerClientProvider.java | 2 +-
.../provider/docker/DockerProviderService.java | 26 +-
.../provider/tarball/TarballClientProvider.java | 2 +-
.../yarn/service/utils/ServiceApiUtil.java | 2 +-
.../hadoop/yarn/service/utils/ServiceUtils.java | 4 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 135 +
.../yarn/service/TestYarnNativeServices.java | 129 -
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 -
.../hadoop/yarn/client/api/AppAdminClient.java | 273 -
.../hadoop/yarn/client/cli/ClusterCLI.java | 2 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 4 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 2 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 6 +-
.../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 6 +-
.../impl/pb/GetApplicationsRequestPBImpl.java | 30 +-
.../hadoop/yarn/client/api/AppAdminClient.java | 285 +
.../yarn/logaggregation/LogCLIHelpers.java | 2 +-
.../yarn/logaggregation/LogToolUtils.java | 2 +-
.../LogAggregationFileController.java | 2 +-
.../LogAggregationFileControllerFactory.java | 2 +-
.../LogAggregationIndexedFileController.java | 2 +-
.../nodelabels/CommonNodeLabelsManager.java | 2 +-
.../hadoop/yarn/nodelabels/RMNodeLabel.java | 2 +-
.../org/apache/hadoop/yarn/state/Graph.java | 2 +-
.../util/Log4jWarningErrorMetricsAppender.java | 2 +-
.../yarn/util/ProcfsBasedProcessTree.java | 2 +-
.../org/apache/hadoop/yarn/webapp/Router.java | 2 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 2 +-
.../hadoop/yarn/webapp/hamlet/HamletImpl.java | 6 +-
.../hadoop/yarn/webapp/hamlet2/HamletImpl.java | 6 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 8 +-
.../hadoop/yarn/webapp/view/TextView.java | 6 +-
.../webapps/static/jquery/jquery-1.8.2.min.js | 2 -
.../webapps/static/jquery/jquery-3.3.1.min.js | 2 +
.../webapps/static/jt/jquery.jstree.js | 42 +-
.../src/main/resources/yarn-default.xml | 9 +
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 8 +-
.../yarn/api/TestGetApplicationsRequest.java | 7 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 4 +-
.../nodelabels/TestCommonNodeLabelsManager.java | 2 +-
.../apache/hadoop/yarn/webapp/TestWebApp.java | 2 +-
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 5 -
.../client/api/RegistryOperationsFactory.java | 2 +-
.../registry/client/binding/JsonSerDeser.java | 2 +-
.../registry/client/binding/RegistryUtils.java | 12 +-
.../impl/FSRegistryOperationsService.java | 6 +-
.../client/impl/zk/RegistrySecurity.java | 2 +-
.../hadoop/registry/server/dns/RegistryDNS.java | 9 +-
.../registry/server/dns/ReverseZoneUtils.java | 2 +-
.../integration/SelectByYarnPersistence.java | 2 +-
.../server/services/MicroZookeeperService.java | 2 +-
.../server/services/RegistryAdminService.java | 2 +-
.../hadoop/registry/RegistryTestHelper.java | 2 +-
.../registry/server/dns/TestRegistryDNS.java | 9 +-
.../ApplicationHistoryClientService.java | 4 +-
.../yarn/server/timeline/RollingLevelDB.java | 2 +-
.../timeline/RollingLevelDBTimelineStore.java | 2 +-
.../webapp/TestAHSWebServices.java | 2 +-
.../store/impl/SQLFederationStateStore.java | 6 +-
.../impl/ZookeeperFederationStateStore.java | 8 +-
.../utils/FederationStateStoreFacade.java | 4 +-
.../OpportunisticContainerAllocator.java | 2 +-
.../server/uam/UnmanagedApplicationManager.java | 2 +-
.../util/timeline/TimelineServerUtils.java | 2 +-
.../yarn/server/webapp/AppAttemptBlock.java | 6 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 4 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 14 +-
.../hadoop/yarn/server/webapp/WebServices.java | 4 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 25 +-
.../nodemanager/DefaultContainerExecutor.java | 2 +-
.../WindowsSecureContainerExecutor.java | 2 +-
.../containermanager/AuxServices.java | 6 +-
.../containermanager/container/Container.java | 8 +
.../container/ContainerImpl.java | 7 +-
.../privileged/PrivilegedOperationExecutor.java | 2 +-
.../linux/resources/CGroupsHandlerImpl.java | 4 +-
.../linux/resources/DefaultOOMHandler.java | 249 +-
.../runtime/DockerLinuxContainerRuntime.java | 49 +-
.../runtime/LinuxContainerRuntimeConstants.java | 2 +
.../executor/ContainerSignalContext.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 42 +-
.../nodemanager/webapp/NMWebServices.java | 2 +-
.../impl/container-executor.c | 58 +-
.../impl/container-executor.h | 4 +
.../main/native/container-executor/impl/main.c | 19 +-
.../container-executor/impl/utils/docker-util.c | 7 +-
.../test/utils/test_docker_util.cc | 48 +-
.../TestLocalDirsHandlerService.java | 4 +-
.../containermanager/TestAuxServices.java | 48 +
.../launcher/TestContainerLaunch.java | 60 +-
.../TestCGroupElasticMemoryController.java | 76 +-
.../linux/resources/TestDefaultOOMHandler.java | 922 +-
.../runtime/TestDockerContainerRuntime.java | 34 +-
.../TestAppLogAggregatorImpl.java | 2 +-
.../TestLogAggregationService.java | 2 +-
.../TestNonAggregatingLogHandler.java | 4 +-
.../nodemanager/webapp/MockContainer.java | 5 +
.../conf/capacity-scheduler.xml | 2 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 -
.../server/resourcemanager/ClientRMService.java | 12 +-
.../metrics/TimelineServiceV1Publisher.java | 6 +-
.../metrics/TimelineServiceV2Publisher.java | 8 +-
.../ProportionalCapacityPreemptionPolicy.java | 2 +-
.../placement/QueuePlacementRuleUtils.java | 2 +-
.../UserGroupMappingPlacementRule.java | 2 +-
.../recovery/ZKRMStateStore.java | 2 +-
.../reservation/NoOverCommitPolicy.java | 5 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 32 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 19 +-
.../scheduler/SchedulerApplicationAttempt.java | 6 +-
.../scheduler/SchedulerUtils.java | 2 +-
.../scheduler/capacity/AbstractCSQueue.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 2 +-
.../scheduler/capacity/ParentQueue.java | 2 +-
.../scheduler/capacity/QueueCapacities.java | 2 +-
.../allocator/RegularContainerAllocator.java | 2 +-
.../capacity/conf/ZKConfigurationStore.java | 2 +-
.../constraint/AllocationTagsManager.java | 2 +-
.../algorithm/LocalAllocationTagsManager.java | 2 +-
.../fair/AllocationFileLoaderService.java | 23 +-
.../scheduler/fair/FairScheduler.java | 8 +-
.../scheduler/fair/QueueManager.java | 41 +-
.../scheduler/fifo/FifoScheduler.java | 6 +-
.../security/DelegationTokenRenewer.java | 10 +-
.../webapp/FairSchedulerAppsBlock.java | 10 +-
.../webapp/RMAppAttemptBlock.java | 8 +-
.../resourcemanager/webapp/RMAppBlock.java | 4 +-
.../resourcemanager/webapp/RMAppsBlock.java | 14 +-
.../webapp/dao/AppAttemptInfo.java | 2 +-
.../TestContainerResourceUsage.java | 2 +-
.../recovery/TestZKRMStateStore.java | 29 +-
.../TestZKRMStateStoreZKClientConnections.java | 22 +-
.../TestRMAppAttemptImplDiagnostics.java | 2 +-
.../capacity/TestCapacityScheduler.java | 44 +-
...stCapacitySchedulerAutoCreatedQueueBase.java | 4 +-
.../fair/TestAllocationFileLoaderService.java | 96 +-
.../scheduler/fair/TestFairScheduler.java | 43 +-
.../scheduler/fifo/TestFifoScheduler.java | 44 +-
.../security/TestDelegationTokenRenewer.java | 24 +
.../clientrm/FederationClientInterceptor.java | 58 +-
.../yarn/server/router/webapp/AppsBlock.java | 6 +-
.../webapp/FederationInterceptorREST.java | 70 +-
.../pom.xml | 5 -
.../storage/TimelineSchemaCreator.java | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 5 -
.../AppLevelTimelineCollectorWithAgg.java | 14 +-
.../collector/TimelineCollectorWebService.java | 19 +-
.../reader/TimelineReaderManager.java | 21 +
.../reader/TimelineReaderUtils.java | 2 +-
.../reader/TimelineReaderWebServices.java | 18 +-
.../reader/TimelineReaderWebServicesUtils.java | 2 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 13 +
.../server/webproxy/amfilter/AmIpFilter.java | 19 +-
.../webproxy/amfilter/TestSecureAmFilter.java | 159 +
.../src/test/resources/krb5.conf | 33 +
.../src/site/markdown/CapacityScheduler.md | 5 +
.../src/site/markdown/DockerContainers.md | 48 +-
.../src/site/markdown/Federation.md | 19 +-
.../src/site/markdown/NodeManagerCgroups.md | 2 +-
.../src/site/markdown/yarn-service/Examples.md | 37 +-
.../markdown/yarn-service/YarnServiceAPI.md | 2 +-
.../main/webapp/app/adapters/yarn-app-log.js | 68 +
.../src/main/webapp/app/adapters/yarn-log.js | 6 -
.../webapp/app/components/deploy-service.js | 5 +-
.../webapp/app/controllers/yarn-app/logs.js | 17 +-
.../src/main/webapp/app/initializers/loader.js | 2 +-
.../src/main/webapp/app/models/yarn-app-log.js | 25 +
.../main/webapp/app/models/yarn-servicedef.js | 8 +-
.../webapp/app/routes/yarn-node-container.js | 2 +-
.../main/webapp/app/serializers/yarn-app-log.js | 38 +
.../app/templates/components/deploy-service.hbs | 11 +
.../webapp/app/templates/yarn-container-log.hbs | 2 +-
.../src/main/webapp/app/utils/converter.js | 10 +
.../src/main/webapp/app/utils/info-seeder.js | 1 +
.../tests/unit/adapters/yarn-app-log-test.js | 30 +
480 files changed, 28447 insertions(+), 9955 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/418cff48/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
index 06db1e1,0000000..d697bbf
mode 100644,000000..100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
@@@ -1,463 -1,0 +1,533 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.utils;
+
++import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
++
+import com.google.common.collect.Lists;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
+import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.event.Level;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Test class for ozone metadata store.
+ */
+@RunWith(Parameterized.class)
+public class TestMetadataStore {
+
+ private final String storeImpl;
+
+ public TestMetadataStore(String metadataImpl) {
+ this.storeImpl = metadataImpl;
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private MetadataStore store;
+ private File testDir;
+ private final static int MAX_GETRANGE_LENGTH = 100;
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Before
+ public void init() throws IOException {
++ if (OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(storeImpl)) {
++ // The initialization of RocksDB fails on Windows
++ assumeNotWindows();
++ }
++
+ testDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase());
+
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ store = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(testDir)
+ .build();
+
+ // Add 20 entries.
+ // {a0 : a-value0} to {a9 : a-value9}
+ // {b0 : b-value0} to {b9 : b-value9}
+ for (int i=0; i<10; i++) {
+ store.put(getBytes("a" + i), getBytes("a-value" + i));
+ store.put(getBytes("b" + i), getBytes("b-value" + i));
+ }
+ }
+
+ @Test
+ public void testMetaStoreConfigDifferentFromType() throws IOException {
+
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+ String dbType;
+ GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
+ GenericTestUtils.LogCapturer logCapturer =
+ GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);
+ if(storeImpl.equals(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB)) {
+ dbType = "RocksDB";
+ } else {
+ dbType = "LevelDB";
+ }
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + dbType.toLowerCase() + "-test");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
+ .setCreateIfMissing(true).setDbFile(dbDir).setDBType(dbType).build();
+ assertTrue(logCapturer.getOutput().contains("Using dbType " + dbType + "" +
+ " for metastore"));
+ dbStore.close();
+ dbStore.destroy();
+ FileUtils.deleteDirectory(dbDir);
+
+ }
+
+ @Test
+ public void testdbTypeNotSet() throws IOException {
+
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+ GenericTestUtils.setLogLevel(MetadataStoreBuilder.LOG, Level.DEBUG);
+ GenericTestUtils.LogCapturer logCapturer =
+ GenericTestUtils.LogCapturer.captureLogs(MetadataStoreBuilder.LOG);
+
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase() + "-test");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder().setConf(conf)
+ .setCreateIfMissing(true).setDbFile(dbDir).build();
+ assertTrue(logCapturer.getOutput().contains("dbType is null, using dbType" +
+ " " + storeImpl));
+ dbStore.close();
+ dbStore.destroy();
+ FileUtils.deleteDirectory(dbDir);
+
+ }
+
+ @After
+ public void cleanup() throws IOException {
- store.close();
- store.destroy();
- FileUtils.deleteDirectory(testDir);
++ if (store != null) {
++ store.close();
++ store.destroy();
++ }
++ if (testDir != null) {
++ FileUtils.deleteDirectory(testDir);
++ }
+ }
+
+ private byte[] getBytes(String str) {
+ return str == null ? null :
+ DFSUtilClient.string2Bytes(str);
+ }
+
+ private String getString(byte[] bytes) {
+ return bytes == null ? null :
+ DFSUtilClient.bytes2String(bytes);
+ }
+
+ @Test
+ public void testGetDelete() throws IOException {
+ for (int i=0; i<10; i++) {
+ byte[] va = store.get(getBytes("a" + i));
+ Assert.assertEquals("a-value" + i, getString(va));
+
+ byte[] vb = store.get(getBytes("b" + i));
+ Assert.assertEquals("b-value" + i, getString(vb));
+ }
+
+ String keyToDel = "del-" + UUID.randomUUID().toString();
+ store.put(getBytes(keyToDel), getBytes(keyToDel));
+ Assert.assertEquals(keyToDel, getString(store.get(getBytes(keyToDel))));
+ store.delete(getBytes(keyToDel));
+ Assert.assertEquals(null, store.get(getBytes(keyToDel)));
+ }
+
+ @Test
+ public void testPeekFrom() throws IOException {
+ // Test peek from an element that has prev as well as next
+ testPeek("a3", "a2", "a4");
+
+ // Test peek from an element that only has prev
+ testPeek("b9", "b8", null);
+
+ // Test peek from an element that only has next
+ testPeek("a0", null, "a1");
+ }
+
+ private String getExpectedValue(String key) {
+ if (key == null) {
+ return null;
+ }
+ char[] arr = key.toCharArray();
+ return new StringBuffer().append(arr[0]).append("-value")
+ .append(arr[arr.length - 1]).toString();
+ }
+
+ private void testPeek(String peekKey, String prevKey, String nextKey)
+ throws IOException {
+ // Look for current
+ String k = null;
+ String v = null;
+ ImmutablePair<byte[], byte[]> current =
+ store.peekAround(0, getBytes(peekKey));
+ if (current != null) {
+ k = getString(current.getKey());
+ v = getString(current.getValue());
+ }
+ Assert.assertEquals(peekKey, k);
+ Assert.assertEquals(v, getExpectedValue(peekKey));
+
+ // Look for prev
+ k = null;
+ v = null;
+ ImmutablePair<byte[], byte[]> prev =
+ store.peekAround(-1, getBytes(peekKey));
+ if (prev != null) {
+ k = getString(prev.getKey());
+ v = getString(prev.getValue());
+ }
+ Assert.assertEquals(prevKey, k);
+ Assert.assertEquals(v, getExpectedValue(prevKey));
+
+ // Look for next
+ k = null;
+ v = null;
+ ImmutablePair<byte[], byte[]> next =
+ store.peekAround(1, getBytes(peekKey));
+ if (next != null) {
+ k = getString(next.getKey());
+ v = getString(next.getValue());
+ }
+ Assert.assertEquals(nextKey, k);
+ Assert.assertEquals(v, getExpectedValue(nextKey));
+ }
+
+ @Test
+ public void testIterateKeys() throws IOException {
+ // iterate keys from b0
+ ArrayList<String> result = Lists.newArrayList();
+ store.iterate(getBytes("b0"), (k, v) -> {
+ // b-value{i}
+ String value = getString(v);
+ char num = value.charAt(value.length() - 1);
+ // each value adds 1
+ int i = Character.getNumericValue(num) + 1;
+ value = value.substring(0, value.length() - 1) + i;
+ result.add(value);
+ return true;
+ });
+
+ Assert.assertFalse(result.isEmpty());
+ for (int i=0; i<result.size(); i++) {
+ Assert.assertEquals("b-value" + (i+1), result.get(i));
+ }
+
+ // iterate from a non exist key
+ result.clear();
+ store.iterate(getBytes("xyz"), (k, v) -> {
+ result.add(getString(v));
+ return true;
+ });
+ Assert.assertTrue(result.isEmpty());
+
+ // iterate from the beginning
+ result.clear();
+ store.iterate(null, (k, v) -> {
+ result.add(getString(v));
+ return true;
+ });
+ Assert.assertEquals(20, result.size());
+ }
+
+ @Test
+ public void testGetRangeKVs() throws IOException {
+ List<Map.Entry<byte[], byte[]>> result = null;
+
+ // Set empty startKey will return values from beginning.
+ result = store.getRangeKVs(null, 5);
+ Assert.assertEquals(5, result.size());
+ Assert.assertEquals("a-value2", getString(result.get(2).getValue()));
+
+ // Empty list if startKey doesn't exist.
+ result = store.getRangeKVs(getBytes("a12"), 5);
+ Assert.assertEquals(0, result.size());
+
+ // Returns max available entries after a valid startKey.
+ result = store.getRangeKVs(getBytes("b0"), MAX_GETRANGE_LENGTH);
+ Assert.assertEquals(10, result.size());
+ Assert.assertEquals("b0", getString(result.get(0).getKey()));
+ Assert.assertEquals("b-value0", getString(result.get(0).getValue()));
+ result = store.getRangeKVs(getBytes("b0"), 5);
+ Assert.assertEquals(5, result.size());
+
+ // Both startKey and count are honored.
+ result = store.getRangeKVs(getBytes("a9"), 2);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a9", getString(result.get(0).getKey()));
+ Assert.assertEquals("a-value9", getString(result.get(0).getValue()));
+ Assert.assertEquals("b0", getString(result.get(1).getKey()));
+ Assert.assertEquals("b-value0", getString(result.get(1).getValue()));
+
+ // Filter keys by prefix.
+ // It should returns all "b*" entries.
- MetadataKeyFilter filter1 = new KeyPrefixFilter("b");
++ MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b");
+ result = store.getRangeKVs(null, 100, filter1);
+ Assert.assertEquals(10, result.size());
+ Assert.assertTrue(result.stream().allMatch(entry ->
+ new String(entry.getKey()).startsWith("b")
+ ));
+ Assert.assertEquals(20, filter1.getKeysScannedNum());
+ Assert.assertEquals(10, filter1.getKeysHintedNum());
+ result = store.getRangeKVs(null, 3, filter1);
+ Assert.assertEquals(3, result.size());
+ result = store.getRangeKVs(getBytes("b3"), 1, filter1);
+ Assert.assertEquals("b-value3", getString(result.get(0).getValue()));
+
+ // Define a customized filter that filters keys by suffix.
+ // Returns all "*2" entries.
+ MetadataKeyFilter filter2 = (preKey, currentKey, nextKey)
+ -> getString(currentKey).endsWith("2");
+ result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter2);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a2", getString(result.get(0).getKey()));
+ Assert.assertEquals("b2", getString(result.get(1).getKey()));
+ result = store.getRangeKVs(null, 1, filter2);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a2", getString(result.get(0).getKey()));
+
+ // Apply multiple filters.
+ result = store.getRangeKVs(null, MAX_GETRANGE_LENGTH, filter1, filter2);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("b2", getString(result.get(0).getKey()));
+ Assert.assertEquals("b-value2", getString(result.get(0).getValue()));
+
+ // If filter is null, no effect.
+ result = store.getRangeKVs(null, 1, null);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a0", getString(result.get(0).getKey()));
+ }
+
+ @Test
+ public void testGetSequentialRangeKVs() throws IOException {
+ MetadataKeyFilter suffixFilter = (preKey, currentKey, nextKey)
+ -> DFSUtil.bytes2String(currentKey).endsWith("2");
+ // Suppose to return a2 and b2
+ List<Map.Entry<byte[], byte[]>> result =
+ store.getRangeKVs(null, MAX_GETRANGE_LENGTH, suffixFilter);
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+ Assert.assertEquals("b2", DFSUtil.bytes2String(result.get(1).getKey()));
+
+ // Suppose to return just a2, because when it iterates to a3,
+ // the filter no long matches and it should stop from there.
+ result = store.getSequentialRangeKVs(null,
+ MAX_GETRANGE_LENGTH, suffixFilter);
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals("a2", DFSUtil.bytes2String(result.get(0).getKey()));
+ }
+
+ @Test
+ public void testGetRangeLength() throws IOException {
+ List<Map.Entry<byte[], byte[]>> result = null;
+
+ result = store.getRangeKVs(null, 0);
+ Assert.assertEquals(0, result.size());
+
+ result = store.getRangeKVs(null, 1);
+ Assert.assertEquals(1, result.size());
+
+ // Count less than zero is invalid.
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Invalid count given");
+ store.getRangeKVs(null, -1);
+ }
+
+ @Test
+ public void testInvalidStartKey() throws IOException {
+ // If startKey is invalid, the returned list should be empty.
+ List<Map.Entry<byte[], byte[]>> kvs =
+ store.getRangeKVs(getBytes("unknownKey"), MAX_GETRANGE_LENGTH);
+ Assert.assertEquals(kvs.size(), 0);
+ }
+
+ @Test
+ public void testDestroyDB() throws IOException {
+ // create a new DB to test db destroy
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase() + "-toDestroy");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(dbDir)
+ .build();
+
+ dbStore.put(getBytes("key1"), getBytes("value1"));
+ dbStore.put(getBytes("key2"), getBytes("value2"));
+
+ Assert.assertFalse(dbStore.isEmpty());
+ Assert.assertTrue(dbDir.exists());
+ Assert.assertTrue(dbDir.listFiles().length > 0);
+
+ dbStore.destroy();
+
+ Assert.assertFalse(dbDir.exists());
+ }
+
+ @Test
+ public void testBatchWrite() throws IOException {
+ Configuration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, storeImpl);
+
+ File dbDir = GenericTestUtils.getTestDir(getClass().getSimpleName()
+ + "-" + storeImpl.toLowerCase() + "-batchWrite");
+ MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setCreateIfMissing(true)
+ .setDbFile(dbDir)
+ .build();
+
+ List<String> expectedResult = Lists.newArrayList();
+ for (int i = 0; i<10; i++) {
+ dbStore.put(getBytes("batch-" + i), getBytes("batch-value-" + i));
+ expectedResult.add("batch-" + i);
+ }
+
+ BatchOperation batch = new BatchOperation();
+ batch.delete(getBytes("batch-2"));
+ batch.delete(getBytes("batch-3"));
+ batch.delete(getBytes("batch-4"));
+ batch.put(getBytes("batch-new-2"), getBytes("batch-new-value-2"));
+
+ expectedResult.remove("batch-2");
+ expectedResult.remove("batch-3");
+ expectedResult.remove("batch-4");
+ expectedResult.add("batch-new-2");
+
+ dbStore.writeBatch(batch);
+
+ Iterator<String> it = expectedResult.iterator();
+ AtomicInteger count = new AtomicInteger(0);
+ dbStore.iterate(null, (key, value) -> {
+ count.incrementAndGet();
+ return it.hasNext() && it.next().equals(getString(key));
+ });
+
+ Assert.assertEquals(8, count.get());
+ }
++
++ @Test
++ public void testKeyPrefixFilter() throws IOException {
++ List<Map.Entry<byte[], byte[]>> result = null;
++ RuntimeException exception = null;
++
++ try {
++ new KeyPrefixFilter().addFilter("b0", true).addFilter("b");
++ } catch (IllegalArgumentException e) {
++ exception = e;
++ }
++ Assert.assertTrue(
++ exception.getMessage().contains("KeyPrefix: b already rejected"));
++
++ try {
++ new KeyPrefixFilter().addFilter("b0").addFilter("b", true);
++ } catch (IllegalArgumentException e) {
++ exception = e;
++ }
++ Assert.assertTrue(
++ exception.getMessage().contains("KeyPrefix: b already accepted"));
++
++ try {
++ new KeyPrefixFilter().addFilter("b", true).addFilter("b0");
++ } catch (IllegalArgumentException e) {
++ exception = e;
++ }
++ Assert.assertTrue(
++ exception.getMessage().contains("KeyPrefix: b0 already rejected"));
++
++ try {
++ new KeyPrefixFilter().addFilter("b").addFilter("b0", true);
++ } catch (IllegalArgumentException e) {
++ exception = e;
++ }
++ Assert.assertTrue(
++ exception.getMessage().contains("KeyPrefix: b0 already accepted"));
++
++ MetadataKeyFilter filter1 = new KeyPrefixFilter(true)
++ .addFilter("a0")
++ .addFilter("a1")
++ .addFilter("b", true);
++ result = store.getRangeKVs(null, 100, filter1);
++ Assert.assertEquals(2, result.size());
++ Assert.assertTrue(result.stream()
++ .anyMatch(entry -> new String(entry.getKey()).startsWith("a0"))
++ && result.stream()
++ .anyMatch(entry -> new String(entry.getKey()).startsWith("a1")));
++
++ filter1 = new KeyPrefixFilter(true).addFilter("b", true);
++ result = store.getRangeKVs(null, 100, filter1);
++ Assert.assertEquals(0, result.size());
++
++ filter1 = new KeyPrefixFilter().addFilter("b", true);
++ result = store.getRangeKVs(null, 100, filter1);
++ Assert.assertEquals(10, result.size());
++ Assert.assertTrue(result.stream()
++ .allMatch(entry -> new String(entry.getKey()).startsWith("a")));
++ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/418cff48/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
index d9dd360,7f611b9..c63eb73
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
@@@ -121,14 -121,12 +121,15 @@@ public final class ContainerCache exten
* Returns a DB handle if available, create the handler otherwise.
*
* @param containerID - ID of the container.
+ * @param containerDBType - DB type of the container.
+ * @param containerDBPath - DB path of the container.
* @return MetadataStore.
*/
- public MetadataStore getDB(long containerID, String containerDBPath)
+ public MetadataStore getDB(long containerID, String containerDBType, String
+ containerDBPath)
throws IOException {
- Preconditions.checkState(containerID >= 0, "Container ID cannot be negative.");
+ Preconditions.checkState(containerID >= 0,
+ "Container ID cannot be negative.");
lock.lock();
try {
MetadataStore db = (MetadataStore) this.get(containerID);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org