You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sa...@apache.org on 2019/09/23 09:05:09 UTC
[hadoop] branch HDDS-1564 updated (326b5ac -> 7b5a5fe)
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
omit 326b5ac HDDS-2089: Add createPipeline CLI. (#1418)
omit 753fc67 Refactor to have PlacementPolicy and ScmCommonPolicy to be general basis. (#1395)
omit b640a5f HDDS-1577. Add default pipeline placement policy implementation. (#1366)
add 5d31a4e HDDS-1753. Datanode unable to find chunk while replication data using ratis. (#1318)
add 55cc115 HADOOP-16527. Add a whitelist of endpoints to skip Kerberos authentication (#1336) Contributed by Akira Ajisaka.
add c0499bd HDDS-1596. Create service endpoint to download configuration from SCM.
add dc72782 HDDS-738. Removing REST protocol support from OzoneClient. Contributed by Elek, Marton.
add 1407414 HDDS-1094. Performance test infrastructure : skip writing user data on Datanode. Contributed by Supratim Deka (#1323)
add c7d426d HDDS-1881. Design doc: decommissioning in Ozone (#1196)
add 48cb583 HDFS-14710. RBF: Improve some RPC performance by using previous block. Contributed by xuzq.
add 29bd6f3 HDFS-8631. WebHDFS : Support setQuota. Contributed by Chao Sun.
add addfb7f HDDS-1937. Acceptance tests fail if scm webui shows invalid json
add 2fcd0da HDDS-1942. Support copy during S3 multipart upload part creation
add aef6a4f HDDS-1950. S3 MPU part-list call fails if there are no parts
add 3e6a016 HDDS-2051. Rat check failure in decommissioning.md (#1372)
add 6f2226a YARN-9756: Create metric that sums total memory/vcores preempted per round. Contributed by Manikandan R (manirajv06).
add 872cdf4 HDDS-1941. Unused executor in SimpleContainerDownloader (#1367)
add 16576fd YARN-9754. Add support for arbitrary DAG AM Simulator. Contributed by Abhishek Modi.
add 371c9eb Revert "HDDS-1596. Create service endpoint to download configuration from SCM."
add c749f62 HDDS-2045. Partially started compose cluster left running
add 8c0759d YARN-9664. Improve response of scheduler/app activities for better understanding. Contributed by Tao Yang.
add 8e779a1 HDFS-14721. RBF: ProxyOpComplete is not accurate in FederationRPCPerformanceMonitor. Contributed by xuzq.
add f600fbb HDFS-11246. FSNameSystem#logAuditEvent should be called outside the read or write locks. Contributed by He Xiaoqiao, Kuhu Shukla.
add ffca734 HDFS-14104. Review getImageTxIdToRetain. Contributed by David Mollitor.
add 3b22fcd HDFS-14256. Review Logging of NameNode Class. Contributed by David Mollitor.
add fcb7884 HDFS-8178. QJM doesn't move aside stale inprogress edits files. Contributed by Istvan Fajth.
add 7b3fa4f HDDS-2050. Error while compiling ozone-recon-web (#1374)
add 4f5f46e HDDS-1935. Improve the visibility with Ozone Insight tool (#1255)
add 7bebad6 HDFS-14706. Checksums are not checked if block meta file is less than 7 bytes. Contributed by Stephen O'Donnell.
add dc0acce YARN-9783. Remove low-level zookeeper test to be able to build Hadoop against zookeeper 3.5.5. Contributed by Mate Szalay-Beko.
add 9152712 Make DataNodePeerMetrics#minOutlierDetectionSamples configurable (#1314). Contributed by Lisheng Sun.
add e220dac HDFS-12212. Options.Rename.To_TRASH is considered even when Options.Rename.NONE is specified. Contributed by Vinayakumar B.
add d59fc59 HADOOP-15998. Ensure jar validation works on Windows.
add 5b557de YARN-9714. ZooKeeper connection in ZKRMStateStore leaks after RM transitioned to standby. Contributed by Tao Yang.
add 22a5861 HDDS-2058. Remove hadoop dependencies in ozone build
add 96f7dc1 HDFS-14796. Define LOG instead of BlockManager.LOG in ErasureCodingWork/ReplicationWork. Contributed by Fei Hui.
add c929b38 YARN-9540. TestRMAppTransitions fails intermittently. Contributed by Tao Yang.
add ec34cee HDDS-2047. Datanodes fail to come up after 10 retries in a secure env… (#1379)
add 46696bd HDDS-2014. Create Symmetric Key for GDPR (#1362)
add 472a26d HDDS-2063. Integration tests create untracked file audit.log (#1384)
add 7085512 HDDS-2061. Add hdds.container.chunk.persistdata as exception to TestOzoneConfigurationFields (#1382)
add c4411f7 HDDS-2042. Avoid log on console with Ozone shell
add a2d083f HDDS-1413. Attempt to fix TestCloseContainerCommandHandler by adjusting timeouts
add eeccd2f YARN-9798. ApplicationMasterServiceTestBase#testRepeatedFinishApplicationMaster fails intermittently. Contributed by Tao Yang.
add 7d998cb YARN-9800. TestRMDelegationTokens can fail in testRemoveExpiredMasterKeyInRMStateStore. Contributed by Adam Antal.
add c187d2c HDDS-2060. Create Ozone specific LICENSE file for the Ozone source package
add d2d963f YARN-9790. Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero. Contributed by kyungwan nam.
add c7ef4fb YARN-8678. Queue Management API - rephrase error messages. Contributed by Prabhu Joseph.
add fef65b4 YARN-9793. Remove duplicate sentence from TimelineServiceV2.md. Contributed by Julia Kinga Marton.
add 18d74fe HDFS-14711. RBF: RBFMetrics throws NullPointerException if stateStore disabled. Contributed by Chen Zhang.
add 751b5a1 YARN-9791. Queue Mutation API does not allow to remove a config. Contributed by Prabhu Joseph.
add c3abfce HDFS-13843. RBF: Add optional parameter -d for detailed listing of mount points. Contributed by Ayush Saxena.
add bc27f7f YARN-7982. Do ACLs check while retrieving entity-types per application. Contribued by Prabhu Joseph.
add b939ae9 YARN-8174. Add containerId to ResourceLocalizationService fetch failure log statement. Contributed by Prabhu Joseph.
add f4d6e82 YARN-9400. Remove unnecessary if at EntityGroupFSTimelineStore#parseApplicationId. Contributed by Prabhu Joseph.
add 040f6e9 HDFS-14654. RBF: TestRouterRpc#testNamenodeMetrics is flaky. Contributed by Chen Zhang.
add d207aba Revert "HDFS-14706. Checksums are not checked if block meta file is less than 7 bytes. Contributed by Stephen O'Donnell."
add 915cbc9 HDFS-14706. Checksums are not checked if block meta file is less than 7 bytes. Contributed by Stephen O'Donnell.
add 0348912 YARN-9797. LeafQueue#activateApplications should use resourceCalculator#fitsIn. Contributed by Bilwa S T.
add bb26514 YARN-9785. Fix DominantResourceCalculator when one resource is zero. Contributed by Bibin A Chundatt, Sunil Govindan, Bilwa S T.
add 5ff76cb HDFS-14630. Configuration.getTimeDurationHelper() should not log time unit warning in info log. Contributed by hemanthboyina.
add b53d19a HDDS-1783 : Latency metric for applyTransaction in ContainerStateMachine (#1363).
add 0b9704f HDDS-1810. SCM command to Activate and Deactivate pipelines. (#1224)
add ac5a0ae HADOOP-16534. Exclude submarine from hadoop source build. (#1356)
add 0f549ec HDDS-2069. Default values of properties hdds.datanode.storage.utilization.{critical | warning}.threshold are not reasonable. (#1393)
add 91b01a1 HDFS-14807. SetTimes updates all negative values apart from -1. Contributed by Ayush Saxena.
add 3c11716 HDFS-14633. The StorageType quota and consume in QuotaFeature is not handled for rename. Contributed by Jinglun.
add f25fe92 HDDS-1909. Use new HA code for Non-HA in OM. (#1225)
add cfa41a49 HDDS-2018. Handle Set DtService of token for OM HA. (#1371)
add 0570475 HDFS-14777. RBF: Set ReadOnly is failing for mount Table but actually readonly succeed to set. Contributed by Ranith Sardar.
add 1ae7759 HDDS-2077. Add maven-gpg-plugin.version to pom.ozone.xml. (#1396)
add 337e9b7 HADOOP-16268. Allow StandbyException to be thrown as CallQueueOverflowException when RPC call queue is filled. Contributed by CR Hota.
add ae28747 HDDS-2079. Fix TestSecureOzoneManager. Contributed by Xiaoyu Yao. (#1400)
add 2f70b52 HDFS-14812. RBF: MountTableRefresherService should load cache when refresh. Contributed by xuzq.
add 3db7184 YARN-9804. Update ATSv2 document for latest feature supports.
add f347c34 Add support for checksum verification in data scrubber
add 172bcd8 YARN-8995. Log events info in AsyncDispatcher when event queue size cumulatively reaches a certain number every time. Contributed by zhuqi.
add 2b16d53 HDDS-1898. GrpcReplicationService#download cannot replicate the container. (#1326)
add 511df1e HADOOP-16430. S3AFilesystem.delete to incrementally update s3guard with deletions
add 72d8b92 HDFS-12904. Add DataTransferThrottler to the Datanode transfers. Contributed by Lisheng Sun.
add 2e2e540 YARN-9718. Fixed yarn.service.am.java.opts shell injection. Contributed by Eric Yang
add 0ccf4b0 YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
add acbea8d HDDS-1708. Add container scrubber metrics. Contributed by Hrishikesh Gadre.
add 494d75e HDFS-14784. Add more methods to WebHdfsTestUtil to support tests outside of package. Contributed by Chen Zhang.
add 6e4cdf8 HDDS-1561: Mark OPEN containers as QUASI_CLOSED as part of Ratis groupRemove (#1401)
add fa7f03f HDFS-14826. dfs.ha.zkfc.port property duplicated in hdfs-default.xml. Contributed by Renukaprasad C.
add d98c548 HDFS-13913. LazyPersistFileScrubber.run() should log meaningful warn message. Contributed by Daniel Green.
add ae42c8c HDFS-4819. [Dynamometer] Fix parsing of audit logs which contain = in path names. Contributed by Soya Miyoshi.
add e7d44e4 YARN-9761. Allow overriding application submissions based on server side configs. Contributed by Pralabh Kumar
add c92a3e9 HADOOP-15565. Add an inner FS cache to ViewFileSystem, separate from the global cache, to avoid file system leaks. Contributed by Jinglun.
add 9637097 HDFS-14817. [Dynamometer] Fix start script options parsing which incorrectly interpret options starting with h as a help argument. Contributed by Soya Miyoshi.
add a234175 HADOOP-16531. Log more timing information for slow RPCs. Contributed by Chen Zhang.
add b71a7f1 YARN-9763. Print application tags in application summary. Contributed by Manoj Kumar
add b15c116 HDDS-2015. Encrypt/decrypt key using symmetric key while writing/reading
add bb0b922 HADOOP-16255. Add ChecksumFs.rename(path, path, boolean)
add 5d497ab YARN-9817. Fix failing testcases due to not initialized AsyncDispatcher - ArithmeticException: / by zero. Contributed by Prabhu Joseph.
add 7357570 YARN-9795. ClusterMetrics to include AM allocation delay. Contributed by Fengnan Li.
add c46d43a HDDS-1553. Add metrics in rack aware container placement policy. (#1361)
add 34b82e6 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
add e8ca4fa YARN-9812. mvn javadoc:javadoc fails in hadoop-sls. Contributed by Abhishek Modi.
add 162af6f YARN-9784. org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue is flaky. Contributed by Julia Kinga Marton.
add ca32917 HDDS-2100. Ozone TokenRenewer provider is incorrectly configured. Contributed by Jitendra Nath Pandey.
add 43e389b YARN-9764. Print application submission context label in application summary. Contributed by Manoj Kumar
add 3b9584d HDDS-2087. Remove the hard coded config key in ChunkManager (#1409)
add 387c332 YARN-9820. RM logs InvalidStateTransitionException when app is submitted. Contributed by Prabhu Joseph
add 60af879 YARN-9821. NM hangs at serviceStop when ATSV2 Backend Hbase is Down. Contributed by Prabhu Joseph.
add 147f986 HADOOP-16438. ADLS Gen1 OpenSSL config control.
add 469165e HDDS-1843. Undetectable corruption after restart of a datanode. Contributed by Shashikant Banerjee(#1364).
add d69b811 HDDS-2102. HddsVolumeChecker should use java optional in place of Guava optional. Contributed by Mukul Kumar Singh. (#1416)
add 650c4ce HDDS-2053. Fix TestOzoneManagerRatisServer failure. Contributed by Xiaoyu Yao. (#1373)
add bc2d3a7 HADOOP-16549. Remove Unsupported SSL/TLS Versions from Docs/Properties. Contributed by Daisuke Kobayashi.
add c3beeb7 HDDS-2048: State check during container state transition in datanode should be lock protected (#1375)
add 38c1a10 HADOOP-16542. Update commons-beanutils version to 1.9.4. Contributed by kevin su.
add dc9abd2 HADOOP-16554. mvn javadoc:javadoc fails in hadoop-aws.
add 10144a5 YARN-9728. Bugfix for escaping illegal xml characters for Resource Manager REST API. Contributed by Prabhu Joseph
add f8f8598 YARN-9824. Fall back to configured queue ordering policy class name
add 524b553 HDFS-14835. RBF: Secured Router should not run when it can't initialize DelegationTokenSecretManager. (#1414)
add dacc448 HADOOP-15184. Add GitHub pull request template. (#1419)
add c255333 HDFS-14838. RBF: Display RPC (instead of HTTP) Port Number in RBF web UI. Contributed by Xieming Li
add 5a381f7 HDDS-2103. TestContainerReplication fails due to unhealthy container (#1421)
add 9221704 HADOOP-16490. Avoid/handle cached 404s during S3A file creation.
add 64ed6b1 HDDS-2075. Tracing in OzoneManager call is propagated with wrong parent (#1415)
add 56b7571 YARN-9815 ReservationACLsTestBase fails with NPE. Contributed by Ahmed Hussein
add f537410 HDDS-2106. Avoid usage of hadoop projects as parent of hdds/ozone
add 3b06f0b YARN-9819. Make TestOpportunisticContainerAllocatorAMService more resilient. Contribued by Abhishek Modi
add 68612a0 HDFS-14840. Use Java Conccurent Instead of Synchronization in BlockPoolTokenSecretManager. Contributed by David Mollitor.
add 44850f6 YARN-9816. EntityGroupFSTimelineStore#scanActiveLogs fails when undesired files are present under /ats/active. Contribued by Prabhu Joseph.
add 4e273a3 HADOOP-16423. S3Guard fsck: Check metadata consistency between S3 and metadatastore (log) (#1208). Contributed by Gabor Bota.
add f4f9f0f HADOOP-16562. [pb-upgrade] Update docker image to have 3.7.1 protoc executable (#1429). Contributed by Vinayakumar B.
add d1c303a HDFS-14699. Erasure Coding: Storage not considered in live replica when replication streams hard limit reached to threshold. Contributed by Zhao Yi Ming.
add 2ff2a7f HDFS-14798. Synchronize invalidateBlocks in DatanodeDescriptor. Contributed by hemanthboyina.
add fe8cdf0 HDDS-2076. Read fails because the block cannot be located in the container (#1410)
add 1505d3f HADOOP-16566. S3Guard fsck: Use org.apache.hadoop.util.StopWatch instead of com.google.common.base.Stopwatch (#1433). Contributed by Gabor Bota.
add 4852a90 HDFS-14754. Erasure Coding : The number of Under-Replicated Blocks never reduced. Contributed by hemanthboyina.
add 4a9a6a2 HDDS-2122. Broken logo image on category sub-pages (#1437)
add 39e82ac HADOOP-16562. [pb-upgrade] Update docker image to have 3.7.1 protoc executable (#1429).
add 95010a4 HDDS-2057. Incorrect Default OM Port in Ozone FS URI Error Message.
add 4239007 YARN-9787. Typo in analysesErrorMsg. Contributed by kevin su.
add 06ad654 HDFS-14821. Make HDFS-14617 (fsimage sub-sections) off by default. Contributed by Stephen O'Donnell.
add 8024818 HDDS-1786 : Datanodes takeSnapshot should delete previously created snapshots (#1163)
add 1843c46 HADOOP-16555. Update commons-compress to 1.19. (#1425) Contributed by YiSheng Lien.
add e22a324 HDDS-2007. Make ozone fs shell command work with OM HA service ids (#1360)
add 6a9f7ca Revert "HDDS-2057. Incorrect Default OM Port in Ozone FS URI Error Message."
add 9a931b8 HDDS-2125. maven-javadoc-plugin.version is missing in pom.ozone.xml
add e04b8a4 HDFS-14799. Do Not Call Map containsKey In Conjunction with get. Contributed by hemanthboyina.
add da2b4fe YARN-9794. RM crashes due to runtime errors in TimelineServiceV2Publisher. Contributed by Tarun Parimi.
add 85b1c72 HDDS-2129. Using dist profile fails with pom.ozone.xml as parent pom (#1449)
add 1e13fe6 HDDS-2044.Remove 'ozone' from the recon module names.
add b633438 HDDS-2096. Ozone ACL document missing AddAcl API
add e952ecf HDDS-2109. Refactor scm.container.client config
add 363373e HDDS-2124. Random next links
add 56f042c HDDS-2078. Get/Renew DelegationToken NPE after HDDS-1909
add c8d61ff HDDS-2030. Generate simplifed reports by the dev-support/checks/*.sh scripts
add 4222b62 HDDS-1879. Support multiple excluded scopes when choosing datanodes in NetworkTopology (#1194)
add 66bd168 HDDS-2107. Datanodes should retry forever to connect to SCM in an unsecure environment (#1424)
add 2358e53 HDDS-2111. XSS fragments can be injected to the S3g landing page
add 292bce7 HDDS-2114. Rename does not preserve non-explicitly created interim directories. Contributed by Lokesh Jain & Istvan Fajth.
add 4f67081 HDDS-2098 : Ozone shell command prints out ERROR when the log4j file … (#1411)
add e54977f HDDS-2132. TestKeyValueContainer is failing (#1457).
add 3a549ce HDDS-2120. Remove hadoop classes from ozonefs-current jar
add 55ce454 HADOOP-16371: Option to disable GCM for SSL connections when running on Java 8.
add f3de141 HDDS-2135. OM Metric mismatch (MultipartUpload failures)
add 7f90731 HDDS-2117. ContainerStateMachine#writeStateMachineData times out. (#1430)
add c474e24 YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during container launch. Contributed by Peter Bacsko.
add eefe9bc YARN-9837. Fixed reading YARN Service JSON spec file larger than 128k. Contributed by Tarun Parimi
add f580a87 HDFS-14795. Add Throttler for writing block. Contributed by Lisheng Sun.
add 3cf6e42 HDFS-14846: libhdfs tests are failing on trunk due to jni usage bugs
add a9ba2b6 HDDS-2142. OM metrics mismatch (abort multipart request) (#1461)
add b88b682 HDDS-2136. OM block allocation metric not paired with its failures (#1460)
add 0dbfc4d HDDS-2139. Update BeanUtils and Jackson Databind dependency versions (#1456)
add f8c1432 YARN-2255. YARN Audit logging not added to log4j.properties. Contributed by Aihua Xu.
add 01d7924 YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.
add e97f0f1 HADOOP-16565. Region must be provided when requesting session credentials or SdkClientException will be thrown (#1454). Contributed by Gabor Bota.
add 15fded2 HDDS-2022. Add additional freon tests
add 419dd0f HDDS-2134. OM Metrics graphs include empty request type
add 087ed86 HDDS-2138. OM bucket operations do not add up
add 285ed0a HDDS-2137. HddsClientUtils and OzoneUtils have duplicate verifyResourceName()
add 6d4b20c HDDS-2143. Rename classes under package org.apache.hadoop.utils
add 111b08a HDDS-2143. Rename classes under package org.apache.hadoop.utils
add c28e731 Wrong commit message: Revert second "HDDS-2143. Rename classes under package org.apache.hadoop.utils"
add 8d9e9ec HDDS-2065. Implement OMNodeDetails#toString
add 5dd859a HDFS-14836. FileIoProvider should not increase FileIoErrors metric in datanode volume metric. Contributed by Aiphago.
add 2c52d00 HDDS-2032. Ozone client should retry writes in case of any ratis/stateMachine exceptions. Contributed by Shashikant Banerjee (#1420).
add 5db32b8 HADOOP-16547. make sure that s3guard prune sets up the FS (#1402). Contributed by Steve Loughran.
add 53b57fd HDDS-2121. Create a shaded ozone filesystem (client) jar (#1452)
add f16eb09 HDDS-2144. MR job failing on secure Ozone cluster. (#1466)
add 4ed0aef HDFS-14822. [SBN read] Revisit GlobalStateIdContext locking when getting server state id. Contributed by Chen Liang.
add ef478fe HDDS-730. ozone fs cli prints hadoop fs in usage
add 1029060 HDDS-2147. Include dumpstream in test report
add 5c963a7 HDDS-2016. Add option to enforce GDPR in Bucket Create command
add e78848f HDDS-2119. Use checkstyle.xml and suppressions.xml in hdds/ozone projects for checkstyle validation
add 28913f7 HDDS-2148. Remove redundant code in CreateBucketHandler.java
add c9900a0 HDDS-2141. Missing total number of operations
add d4205dc HADOOP-16582. LocalFileSystem's mkdirs() does not work as expected under viewfs. Contributed by Kihwal Lee
add 56248f9 HADOOP-16556. Fix some alerts raised by LGTM.
add f6d884c HDDS-2110. Arbitrary file can be downloaded with the help of ProfilerServlet
add f260b5a HDDS-2127. Detailed Tools doc not reachable
add 1ada99b HDDS-2151. Ozone client logs the entire request payload at DEBUG level (#1477)
add a79f286 HDFS-14609. RBF: Security should use common AuthenticationFilter. Contributed by Chen Zhang.
add da1c67e HDDS-1054. List Multipart uploads in a bucket (#1277)
add 126ef77 HDDS-2154. Fix Checkstyle issues (#1475)
add b7ae8a9 HDDS-2101. Ozone filesystem provider doesn't exist (#1473)
add 298f07a HADOOP-16069. Support configure ZK_DTSM_ZK_KERBEROS_PRINCIPAL in ZKDelegationTokenSecretManager using principal with Schema /_HOST. Contributed by luhuachao.
add 9be448b HDDS-2156. Fix alignment issues in HDDS doc pages
add d072d33 HDDS-2020. Remove mTLS from Ozone GRPC. Contributed by Xiaoyu Yao.
add 1654497 HADOOP-16557. [pb-upgrade] Upgrade protobuf.version to 3.7.1 (#1432)
add dd08346 HADOOP-16581. Revise ValueQueue to correctly replenish queues that go below the watermark (#1463)
add 5553887 HDDS-1949. Missing or error-prone test cleanup. Contributed by Doroszlai, Attila.
add b3173e1 HDDS-2001. Update Ratis version to 0.4.0.
add 3f223be HDFS-14844. Make buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream configurable. Contributed by Lisheng Sun.
add 5363730 HDDS-2157. checkstyle: print filenames relative to project root (#1485)
add d7d6ec8 HDDS-2128. Make ozone sh command work with OM HA service ids (#1445)
add aa93866 HDFS-14833. RBF: Router Update Doesn't Sync Quota. Contributed by Ayush Saxena.
add efed445 HADOOP-16589. [pb-upgrade] Update docker image to make 3.7.1 protoc as default (#1482). Contributed by Vinayakumar B.
add dbdc612 HDDS-2163. Add 'Replication factor' to the output of list keys (#1493)
add e02b102 HADOOP-16445. Allow separate custom signing algorithms for S3 and DDB (#1332)
add a94aa1f HDDS-2150. Update dependency versions to avoid security vulnerabilities. (#1472)
add 659c888 HDFS-14818. Check native pmdk lib by 'hadoop checknative' command. Contributed by Feilong He.
add 4c0a7a9 Make upstream aware of 3.2.1 release.
add 07c81e9 HADOOP-16558. [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes (#1494). Contributed by Vinayakumar B.
new 7f76469 HDDS-1577. Add default pipeline placement policy implementation. (#1366)
new 0001e1d HDDS-1571. Create an interface for pipeline placement policy to support network topologies. (#1395)
new 7b5a5fe HDDS-2089: Add createPipeline CLI. (#1418)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (326b5ac)
\
N -- N -- N refs/heads/HDDS-1564 (7b5a5fe)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/pull_request_template.md | 6 +
BUILDING.txt | 25 +-
LICENSE-binary | 2 +-
dev-support/docker/Dockerfile | 6 +-
.../resources/assemblies/hadoop-src-with-hdds.xml | 56 -
.../src/main/resources/assemblies/hadoop-src.xml | 1 +
.../hadoop-client-check-invariants/pom.xml | 3 +-
.../resources/ensure-jars-have-correct-contents.sh | 31 +-
.../hadoop-client-check-test-invariants/pom.xml | 3 +-
.../resources/ensure-jars-have-correct-contents.sh | 33 +-
.../hadoop-client-runtime/pom.xml | 7 +
.../server/KerberosAuthenticationHandler.java | 37 +
.../security/authentication/util/KerberosName.java | 2 +-
.../server/TestKerberosAuthenticationHandler.java | 24 +
hadoop-common-project/hadoop-common/pom.xml | 76 +-
.../hadoop-common/src/CMakeLists.txt | 2 +-
.../hadoop-common/src/main/conf/log4j.properties | 30 +
.../java/org/apache/hadoop/conf/Configuration.java | 2 -
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 14 +-
.../main/java/org/apache/hadoop/fs/ChecksumFs.java | 26 +
.../apache/hadoop/fs/CommonConfigurationKeys.java | 5 +
.../main/java/org/apache/hadoop/fs/FileSystem.java | 48 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 4 +
.../org/apache/hadoop/fs/impl/FutureIOSupport.java | 8 +-
.../java/org/apache/hadoop/fs/shell/Command.java | 2 +-
.../hadoop/fs/shell/CommandWithDestination.java | 74 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 27 +-
.../org/apache/hadoop/fs/viewfs/Constants.java | 7 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 115 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 28 +-
.../org/apache/hadoop/ipc/CallQueueManager.java | 17 +-
.../java/org/apache/hadoop/ipc/FairCallQueue.java | 23 +-
.../org/apache/hadoop/ipc/RemoteException.java | 2 +-
.../java/org/apache/hadoop/ipc/RpcWritable.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 22 +-
.../security/ssl/DelegatingSSLSocketFactory.java | 287 +
.../org/apache/hadoop/security/ssl/SSLFactory.java | 2 +-
.../delegation/ZKDelegationTokenSecretManager.java | 2 +
.../java/org/apache/hadoop/util/DataChecksum.java | 28 +-
.../apache/hadoop/util/NativeLibraryChecker.java | 10 +
.../src/org/apache/hadoop/io/nativeio/NativeIO.c | 14 +-
.../src/org/apache/hadoop/io/nativeio/pmdk_load.c | 28 +-
.../src/org/apache/hadoop/io/nativeio/pmdk_load.h | 5 -
.../hadoop/security/JniBasedUnixGroupsMapping.c | 3 -
.../hadoop-common/src/main/proto/FSProtos.proto | 2 +-
.../src/main/proto/GenericRefreshProtocol.proto | 2 +-
.../src/main/proto/GetUserMappingsProtocol.proto | 2 +-
.../src/main/proto/HAServiceProtocol.proto | 2 +-
.../src/main/proto/IpcConnectionContext.proto | 2 +-
.../src/main/proto/ProtobufRpcEngine.proto | 2 +-
.../src/main/proto/ProtocolInfo.proto | 2 +-
.../proto/RefreshAuthorizationPolicyProtocol.proto | 2 +-
.../src/main/proto/RefreshCallQueueProtocol.proto | 2 +-
.../main/proto/RefreshUserMappingsProtocol.proto | 2 +-
.../hadoop-common/src/main/proto/RpcHeader.proto | 2 +-
.../hadoop-common/src/main/proto/Security.proto | 2 +-
.../hadoop-common/src/main/proto/TraceAdmin.proto | 2 +-
.../src/main/proto/ZKFCProtocol.proto | 2 +-
.../src/main/resources/core-default.xml | 70 +-
.../site/markdown/release/3.2.1/CHANGELOG.3.2.1.md | 553 +
.../markdown/release/3.2.1/RELEASENOTES.3.2.1.md | 80 +
.../apache/hadoop/crypto/key/TestValueQueue.java | 159 +-
.../org/apache/hadoop/fs/FileContextURIBase.java | 19 +-
.../java/org/apache/hadoop/fs/TestChecksumFs.java | 135 +
.../java/org/apache/hadoop/fs/TestFileUtil.java | 6 +
.../org/apache/hadoop/fs/TestFilterFileSystem.java | 3 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../AbstractContractGetFileStatusTest.java | 12 +-
.../hadoop/fs/contract/ContractTestUtils.java | 21 +-
.../java/org/apache/hadoop/fs/shell/TestCopy.java | 52 +-
.../hadoop/fs/viewfs/TestChRootedFileSystem.java | 28 +-
.../fs/viewfs/TestViewFileSystemDelegation.java | 40 +-
.../TestViewFileSystemDelegationTokenSupport.java | 12 +-
.../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 96 +
.../apache/hadoop/ipc/TestCallQueueManager.java | 15 +-
.../org/apache/hadoop/ipc/TestFairCallQueue.java | 80 +-
.../ssl/TestDelegatingSSLSocketFactory.java | 57 +
.../java/org/apache/hadoop/util/TestProtoUtil.java | 2 +-
.../hadoop-common/src/test/proto/test.proto | 2 +-
.../src/test/proto/test_rpc_service.proto | 1 +
.../registry/client/impl/zk/RegistrySecurity.java | 2 +-
.../hadoop/registry/secure/TestSecureRegistry.java | 34 -
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 32 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 115 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 34 +-
.../hdds/scm/client/ContainerOperationClient.java | 12 +
.../hadoop/hdds/scm/client/HddsClientUtils.java | 57 +-
hadoop-hdds/common/pom.xml | 5 +-
.../common}/src/main/bin/hadoop-config.cmd | 0
.../common}/src/main/bin/hadoop-config.sh | 0
.../common}/src/main/bin/hadoop-daemons.sh | 0
.../common}/src/main/bin/hadoop-functions.sh | 0
hadoop-hdds/common/src/main/bin/workers.sh | 59 +
.../common}/src/main/conf/core-site.xml | 0
hadoop-hdds/common/src/main/conf/hadoop-env.cmd | 90 +
.../common}/src/main/conf/hadoop-env.sh | 0
.../src/main/conf/hadoop-metrics2.properties | 99 +
hadoop-hdds/common/src/main/conf/hadoop-policy.xml | 275 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 43 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 15 +-
.../hadoop/hdds/cli/HddsVersionProvider.java | 2 +-
.../hadoop/hdds/client/ReplicationFactor.java | 18 +
.../apache/hadoop/hdds/client/ReplicationType.java | 22 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 9 +
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 66 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 15 -
.../apache/hadoop/hdds/scm/client/ScmClient.java | 16 +
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 5 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 56 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 41 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 35 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 136 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 16 +
...inerLocationProtocolClientSideTranslatorPB.java | 32 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 137 -
.../x509/certificate/client/CertificateClient.java | 6 +
.../client/DefaultCertificateClient.java | 31 +-
.../hadoop/hdds/utils/BackgroundService.java | 161 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 28 +
.../hadoop/hdds/utils/BackgroundTaskQueue.java | 64 +
.../hadoop/hdds/utils/BackgroundTaskResult.java | 44 +
.../apache/hadoop/hdds/utils/BatchOperation.java | 90 +
.../apache/hadoop/hdds/utils/EntryConsumer.java | 38 +
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 56 +
.../org/apache/hadoop/hdds/utils/LevelDBStore.java | 393 +
.../hadoop/hdds/utils/LevelDBStoreIterator.java | 63 +
.../hadoop/hdds/utils/MetaStoreIterator.java | 39 +
.../hadoop/hdds/utils/MetadataKeyFilters.java | 206 +
.../apache/hadoop/hdds/utils/MetadataStore.java | 233 +
.../hadoop/hdds/utils/MetadataStoreBuilder.java | 146 +
.../apache/hadoop/hdds/utils/RetriableTask.java | 78 +
.../org/apache/hadoop/hdds/utils/RocksDBStore.java | 405 +
.../hadoop/hdds/utils/RocksDBStoreIterator.java | 66 +
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 219 +
.../org/apache/hadoop/hdds/utils/Scheduler.java | 104 +
.../org/apache/hadoop/hdds/utils/UniqueId.java | 69 +
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 97 +
.../hadoop/hdds/utils/db/BatchOperation.java | 27 +
.../hadoop/hdds/utils/db/ByteArrayKeyValue.java | 67 +
.../org/apache/hadoop/hdds/utils/db/Codec.java | 43 +
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 113 +
.../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 74 +
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 146 +
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 120 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 196 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 223 +
.../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 52 +
.../apache/hadoop/hdds/utils/db/IntegerCodec.java | 38 +
.../org/apache/hadoop/hdds/utils/db/LongCodec.java | 46 +
.../hadoop/hdds/utils/db/RDBBatchOperation.java | 73 +
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 100 +
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 381 +
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 107 +
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 196 +
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 92 +
.../utils/db/SequenceNumberNotFoundException.java | 37 +
.../apache/hadoop/hdds/utils/db/StringCodec.java | 46 +
.../org/apache/hadoop/hdds/utils/db/Table.java | 168 +
.../apache/hadoop/hdds/utils/db/TableConfig.java | 93 +
.../apache/hadoop/hdds/utils/db/TableIterator.java | 63 +
.../apache/hadoop/hdds/utils/db/TypedTable.java | 361 +
.../hadoop/hdds/utils/db/cache/CacheKey.java | 56 +
.../hadoop/hdds/utils/db/cache/CacheResult.java | 76 +
.../hadoop/hdds/utils/db/cache/CacheValue.java | 47 +
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 74 +
.../hadoop/hdds/utils/db/cache/TableCache.java | 97 +
.../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 154 +
.../hadoop/hdds/utils/db/cache/package-info.java | 18 +
.../apache/hadoop/hdds/utils/db/package-info.java | 22 +
.../org/apache/hadoop/hdds/utils/package-info.java | 18 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 10 +
.../org/apache/hadoop/ozone/audit/SCMAction.java | 2 +
.../apache/hadoop/ozone/common/ChecksumData.java | 7 +-
.../ozone/protocolPB/ProtocolMessageMetrics.java | 105 +
...lockLocationProtocolServerSideTranslatorPB.java | 108 +-
...inerLocationProtocolServerSideTranslatorPB.java | 34 +
.../org/apache/hadoop/utils/BackgroundService.java | 161 -
.../org/apache/hadoop/utils/BackgroundTask.java | 28 -
.../apache/hadoop/utils/BackgroundTaskQueue.java | 64 -
.../apache/hadoop/utils/BackgroundTaskResult.java | 44 -
.../org/apache/hadoop/utils/BatchOperation.java | 90 -
.../org/apache/hadoop/utils/EntryConsumer.java | 38 -
.../org/apache/hadoop/utils/HddsVersionInfo.java | 56 -
.../java/org/apache/hadoop/utils/LevelDBStore.java | 393 -
.../apache/hadoop/utils/LevelDBStoreIterator.java | 65 -
.../org/apache/hadoop/utils/MetaStoreIterator.java | 39 -
.../apache/hadoop/utils/MetadataKeyFilters.java | 206 -
.../org/apache/hadoop/utils/MetadataStore.java | 233 -
.../apache/hadoop/utils/MetadataStoreBuilder.java | 146 -
.../org/apache/hadoop/utils/RetriableTask.java | 78 -
.../java/org/apache/hadoop/utils/RocksDBStore.java | 405 -
.../apache/hadoop/utils/RocksDBStoreIterator.java | 66 -
.../org/apache/hadoop/utils/RocksDBStoreMBean.java | 219 -
.../java/org/apache/hadoop/utils/Scheduler.java | 104 -
.../java/org/apache/hadoop/utils/UniqueId.java | 69 -
.../java/org/apache/hadoop/utils/VersionInfo.java | 97 -
.../org/apache/hadoop/utils/db/BatchOperation.java | 27 -
.../apache/hadoop/utils/db/ByteArrayKeyValue.java | 67 -
.../java/org/apache/hadoop/utils/db/Codec.java | 43 -
.../org/apache/hadoop/utils/db/CodecRegistry.java | 113 -
.../org/apache/hadoop/utils/db/DBCheckpoint.java | 74 -
.../apache/hadoop/utils/db/DBConfigFromFile.java | 146 -
.../java/org/apache/hadoop/utils/db/DBProfile.java | 120 -
.../java/org/apache/hadoop/utils/db/DBStore.java | 196 -
.../org/apache/hadoop/utils/db/DBStoreBuilder.java | 223 -
.../apache/hadoop/utils/db/DBUpdatesWrapper.java | 52 -
.../org/apache/hadoop/utils/db/IntegerCodec.java | 38 -
.../java/org/apache/hadoop/utils/db/LongCodec.java | 46 -
.../apache/hadoop/utils/db/RDBBatchOperation.java | 73 -
.../hadoop/utils/db/RDBCheckpointManager.java | 100 -
.../java/org/apache/hadoop/utils/db/RDBStore.java | 381 -
.../apache/hadoop/utils/db/RDBStoreIterator.java | 107 -
.../java/org/apache/hadoop/utils/db/RDBTable.java | 196 -
.../apache/hadoop/utils/db/RocksDBCheckpoint.java | 92 -
.../utils/db/SequenceNumberNotFoundException.java | 37 -
.../org/apache/hadoop/utils/db/StringCodec.java | 46 -
.../java/org/apache/hadoop/utils/db/Table.java | 168 -
.../org/apache/hadoop/utils/db/TableConfig.java | 93 -
.../org/apache/hadoop/utils/db/TableIterator.java | 63 -
.../org/apache/hadoop/utils/db/TypedTable.java | 361 -
.../org/apache/hadoop/utils/db/cache/CacheKey.java | 56 -
.../apache/hadoop/utils/db/cache/CacheResult.java | 76 -
.../apache/hadoop/utils/db/cache/CacheValue.java | 47 -
.../apache/hadoop/utils/db/cache/EpochEntry.java | 74 -
.../apache/hadoop/utils/db/cache/TableCache.java | 97 -
.../hadoop/utils/db/cache/TableCacheImpl.java | 166 -
.../apache/hadoop/utils/db/cache/package-info.java | 18 -
.../org/apache/hadoop/utils/db/package-info.java | 22 -
.../java/org/apache/hadoop/utils/package-info.java | 18 -
.../src/main/proto/DatanodeContainerProtocol.proto | 5 +-
.../common}/src/main/proto/FSProtos.proto | 0
hadoop-hdds/common/src/main/proto/Security.proto | 73 +
.../proto/StorageContainerLocationProtocol.proto | 22 +
hadoop-hdds/common/src/main/proto/hdds.proto | 3 +-
.../common/src/main/resources/ozone-default.xml | 79 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 74 +-
.../hadoop/hdds/utils/TestHddsIdFactory.java | 77 +
.../hadoop/hdds/utils/TestMetadataStore.java | 590 +
.../hadoop/hdds/utils/TestRetriableTask.java | 76 +
.../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 234 +
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 116 +
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 173 +
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 349 +
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 269 +
.../hdds/utils/db/TestTypedRDBTableStore.java | 373 +
.../hdds/utils/db/cache/TestTableCacheImpl.java | 173 +
.../hadoop/hdds/utils/db/cache/package-info.java | 22 +
.../apache/hadoop/hdds/utils/db/package-info.java | 22 +
.../org/apache/hadoop/hdds/utils/package-info.java | 22 +
.../org/apache/hadoop/utils/TestHddsIdFactory.java | 77 -
.../org/apache/hadoop/utils/TestMetadataStore.java | 589 -
.../org/apache/hadoop/utils/TestRetriableTask.java | 76 -
.../apache/hadoop/utils/TestRocksDBStoreMBean.java | 234 -
.../hadoop/utils/db/TestDBConfigFromFile.java | 116 -
.../apache/hadoop/utils/db/TestDBStoreBuilder.java | 173 -
.../org/apache/hadoop/utils/db/TestRDBStore.java | 349 -
.../apache/hadoop/utils/db/TestRDBTableStore.java | 269 -
.../hadoop/utils/db/TestTypedRDBTableStore.java | 373 -
.../hadoop/utils/db/cache/TestTableCacheImpl.java | 173 -
.../apache/hadoop/utils/db/cache/package-info.java | 22 -
.../org/apache/hadoop/utils/db/package-info.java | 22 -
.../java/org/apache/hadoop/utils/package-info.java | 22 -
.../hadoop/hdds/conf/ConfigFileGenerator.java | 16 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +-
.../container/common/helpers/ContainerMetrics.java | 9 +-
.../ozone/container/common/impl/ContainerData.java | 13 +
.../ozone/container/common/impl/ContainerSet.java | 65 +-
.../container/common/impl/HddsDispatcher.java | 48 +-
.../container/common/interfaces/Container.java | 29 +-
.../common/interfaces/ContainerDispatcher.java | 6 +-
.../ozone/container/common/interfaces/Handler.java | 16 +-
.../common/statemachine/SCMConnectionManager.java | 11 +-
.../CloseContainerCommandHandler.java | 55 +-
.../commandhandler/CommandDispatcher.java | 6 +
.../commandhandler/DeleteBlocksCommandHandler.java | 9 +-
.../common/transport/server/XceiverServerGrpc.java | 16 +-
.../common/transport/server/ratis/CSMMetrics.java | 19 +
.../server/ratis/ContainerStateMachine.java | 133 +-
.../transport/server/ratis/DispatcherContext.java | 24 +-
.../server/ratis/RatisServerConfiguration.java | 48 +
.../transport/server/ratis/XceiverServerRatis.java | 113 +-
.../container/common/utils/ContainerCache.java | 4 +-
.../container/common/utils/ReferenceCountedDB.java | 2 +-
.../container/common/volume/AsyncChecker.java | 65 +
.../container/common/volume/HddsVolumeChecker.java | 3 +-
.../common/volume/ThrottledAsyncChecker.java | 7 +-
.../container/keyvalue/KeyValueBlockIterator.java | 8 +-
.../container/keyvalue/KeyValueContainer.java | 77 +-
.../container/keyvalue/KeyValueContainerCheck.java | 99 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 180 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 14 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 39 +-
.../keyvalue/impl/ChunkManagerDummyImpl.java | 162 +
.../keyvalue/impl/ChunkManagerFactory.java | 91 +
.../container/keyvalue/impl/ChunkManagerImpl.java | 25 +-
.../background/BlockDeletingService.java | 110 +-
.../container/ozoneimpl/ContainerController.java | 12 +-
.../container/ozoneimpl/ContainerDataScanner.java | 166 +
.../ozoneimpl/ContainerDataScrubberMetrics.java | 115 +
.../ozoneimpl/ContainerMetadataScanner.java | 135 +
.../ContainerMetadataScrubberMetrics.java | 93 +
.../ozone/container/ozoneimpl/ContainerReader.java | 2 +-
.../container/ozoneimpl/ContainerScrubber.java | 158 -
.../ozoneimpl/ContainerScrubberConfiguration.java | 74 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 64 +-
.../OnDemandContainerReplicationSource.java | 18 +-
.../replication/SimpleContainerDownloader.java | 22 +-
.../ozone/container/common/TestContainerCache.java | 4 +-
.../TestCloseContainerCommandHandler.java | 414 +-
.../common/volume/TestHddsVolumeChecker.java | 3 +-
.../keyvalue/TestKeyValueBlockIterator.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 101 +-
.../ozoneimpl/TestContainerScrubberMetrics.java | 113 +
.../testutils/BlockDeletingServiceTestImpl.java | 9 +-
.../checkstyle/checkstyle-noframes-sorted.xsl | 189 +
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 196 +
.../dev-support}/checkstyle/suppressions.xml | 0
hadoop-hdds/docs/content/beyond/Tools.md | 49 -
hadoop-hdds/docs/content/design/decommissioning.md | 624 ++
hadoop-hdds/docs/content/gdpr/GDPR in Ozone.md | 42 +
hadoop-hdds/docs/content/gdpr/_index.md | 38 +
hadoop-hdds/docs/content/recipe/_index.md | 2 +-
hadoop-hdds/docs/content/security/SecurityAcls.md | 7 +-
hadoop-hdds/docs/content/security/_index.md | 2 +-
hadoop-hdds/docs/content/shell/BucketCommands.md | 4 +-
hadoop-hdds/docs/content/start/FromSource.md | 1 +
hadoop-hdds/docs/content/start/Kubernetes.md | 1 +
hadoop-hdds/docs/content/start/Minikube.md | 1 +
hadoop-hdds/docs/content/start/OnPrem.md | 1 +
hadoop-hdds/docs/content/start/RunningViaDocker.md | 1 +
.../docs/content/start/StartFromDockerHub.md | 1 +
hadoop-hdds/docs/content/tools/AuditParser.md | 2 +-
hadoop-hdds/docs/content/tools/Freon.md | 62 -
hadoop-hdds/docs/content/tools/Genconf.md | 2 +-
hadoop-hdds/docs/content/tools/SCMCLI.md | 2 +-
hadoop-hdds/docs/content/tools/TestTools.md | 2 +-
hadoop-hdds/docs/content/tools/Tools.md | 19 -
hadoop-hdds/docs/content/tools/_index.md | 65 +
.../themes/ozonedoc/layouts/_default/section.html | 69 +-
.../themes/ozonedoc/layouts/_default/single.html | 2 +
.../themes/ozonedoc/layouts/partials/navbar.html | 2 +-
.../docs/themes/ozonedoc/static/css/ozonedoc.css | 3 +
.../apache/hadoop/hdds/server/BaseHttpServer.java | 4 +-
.../hadoop/hdds/server/LogStreamServlet.java | 58 +
.../apache/hadoop/hdds/server/ProfileServlet.java | 59 +-
.../hadoop/hdds/server/PrometheusMetricsSink.java | 6 +-
.../hadoop/hdds/server/ServiceRuntimeInfoImpl.java | 2 +-
.../hadoop/hdds/server/events/EventQueue.java | 18 +-
.../src/main/resources/webapps/static/dfs-dust.js | 133 -
.../hadoop/hdds/server/TestProfileServlet.java | 54 +
hadoop-hdds/pom.xml | 10 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 2 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 6 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 8 +-
.../hdds/scm/container/ReplicationManager.java | 100 +-
.../hdds/scm/container/SCMContainerManager.java | 6 +-
.../ContainerPlacementPolicyFactory.java | 14 +-
.../algorithms/SCMContainerPlacementCapacity.java | 2 +-
.../algorithms/SCMContainerPlacementMetrics.java | 107 +
.../algorithms/SCMContainerPlacementRackAware.java | 31 +-
.../algorithms/SCMContainerPlacementRandom.java | 2 +-
.../replication/ReplicationActivityStatus.java | 2 +-
.../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 2 +-
.../metadata/DeletedBlocksTransactionCodec.java | 2 +-
.../apache/hadoop/hdds/scm/metadata/LongCodec.java | 2 +-
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 +-
.../hdds/scm/metadata/SCMMetadataStoreRDBImpl.java | 10 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 2 +-
.../hadoop/hdds/scm/node/NodeReportHandler.java | 1 -
.../hadoop/hdds/scm/node/SCMNodeManager.java | 78 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 5 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 19 +
.../hdds/scm/pipeline/PipelineReportHandler.java | 3 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 24 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 10 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 15 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 49 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 51 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 18 +
.../hdds/scm/server/StorageContainerManager.java | 21 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 6 +-
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestSCMContainerManager.java | 2 +-
.../algorithms/TestContainerPlacementFactory.java | 9 +-
.../TestSCMContainerPlacementCapacity.java | 3 +-
.../TestSCMContainerPlacementRackAware.java | 141 +-
.../TestSCMContainerPlacementRandom.java | 3 +-
.../hdds/scm/node/TestContainerPlacement.java | 10 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 2 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 +-
.../TestOneReplicaPipelineSafeModeRule.java | 2 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 10 +-
.../scm/server/TestSCMBlockProtocolServer.java | 8 +-
.../placement/TestContainerPlacement.java | 4 +-
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 4 +
.../cli/pipeline/ActivatePipelineSubcommand.java | 53 +
.../cli/pipeline/DeactivatePipelineSubcommand.java | 53 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 36 +-
.../apache/hadoop/hdfs/DistributedFileSystem.java | 2 +
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 +
.../hdfs/client/impl/BlockReaderFactory.java | 2 +-
.../hadoop/hdfs/client/impl/BlockReaderRemote.java | 11 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 6 +-
.../hdfs/protocol/datatransfer/PipelineAck.java | 2 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 17 +-
.../hdfs/server/datanode/BlockMetadataHeader.java | 32 +-
.../datanode/CorruptMetaHeaderException.java | 36 +
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 43 +
.../hdfs/web/resources/NameSpaceQuotaParam.java | 44 +
.../hadoop/hdfs/web/resources/PutOpParam.java | 3 +
.../hdfs/web/resources/StorageSpaceQuotaParam.java | 45 +
.../hdfs/web/resources/StorageTypeParam.java | 37 +
.../src/main/proto/ClientDatanodeProtocol.proto | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +-
.../src/main/proto/ReconfigurationProtocol.proto | 2 +-
.../hadoop-hdfs-client/src/main/proto/acl.proto | 2 +-
.../src/main/proto/datatransfer.proto | 2 +-
.../src/main/proto/encryption.proto | 2 +-
.../src/main/proto/erasurecoding.proto | 2 +-
.../hadoop-hdfs-client/src/main/proto/hdfs.proto | 2 +-
.../src/main/proto/inotify.proto | 2 +-
.../hadoop-hdfs-client/src/main/proto/xattr.proto | 2 +-
.../fs/http/server/HttpFSExceptionProvider.java | 2 +-
.../main/native/libhdfs-tests/native_mini_dfs.c | 2 +-
.../src/main/native/libhdfs/hdfs.c | 1 -
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 32 +-
.../hdfs/server/federation/metrics/RBFMetrics.java | 27 +-
.../router/MountTableRefresherService.java | 36 +-
.../hdfs/server/federation/router/Router.java | 2 +-
.../federation/router/RouterAdminServer.java | 91 +-
.../federation/router/RouterClientProtocol.java | 38 +-
.../server/federation/router/RouterRpcClient.java | 39 +-
.../federation/router/RouterWebHdfsMethods.java | 10 +-
.../router/security/RouterSecurityManager.java | 5 +-
.../impl/pb/FederationProtocolPBTranslator.java | 4 +-
.../hadoop/hdfs/tools/federation/RouterAdmin.java | 59 +-
.../src/main/proto/FederationProtocol.proto | 2 +-
.../src/main/proto/RouterProtocol.proto | 2 +-
.../fs/contract/router/SecurityConfUtil.java | 15 +-
.../hdfs/server/federation/MockResolver.java | 24 +-
.../federation/router/TestRouterAdminCLI.java | 98 +-
.../server/federation/router/TestRouterQuota.java | 9 +
.../server/federation/router/TestRouterRpc.java | 53 +-
.../router/TestRouterRpcMultiDestination.java | 98 +
.../router/TestRouterWithSecureStartup.java | 14 +-
.../security/TestRouterHttpDelegationToken.java | 105 +-
.../security/TestRouterSecurityManager.java | 16 +-
.../dev-support/jdiff/Apache_Hadoop_HDFS_3.2.1.xml | 674 ++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 48 +-
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +-
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 6 +-
...ientNamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 11 +-
.../token/block/BlockPoolTokenSecretManager.java | 17 +-
.../hdfs/server/blockmanagement/BlockManager.java | 27 +-
.../blockmanagement/BlockReconstructionWork.java | 6 +
.../server/blockmanagement/DatanodeDescriptor.java | 4 +-
.../server/blockmanagement/ErasureCodingWork.java | 8 +-
.../server/blockmanagement/InvalidateBlocks.java | 10 +-
.../server/blockmanagement/ReplicationWork.java | 3 +-
.../hadoop/hdfs/server/datanode/BlockSender.java | 13 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 47 +-
.../hadoop/hdfs/server/datanode/DataXceiver.java | 16 +-
.../hdfs/server/datanode/DataXceiverServer.java | 30 +
.../hdfs/server/datanode/FileIoProvider.java | 5 +-
.../datanode/erasurecode/StripedBlockReader.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetCache.java | 15 +-
.../datanode/metrics/DataNodePeerMetrics.java | 23 +-
.../hadoop/hdfs/server/namenode/FSDirAttrOp.java | 4 +-
.../hadoop/hdfs/server/namenode/FSDirRenameOp.java | 7 +-
.../hadoop/hdfs/server/namenode/FSDirectory.java | 5 +-
.../server/namenode/FSImageFormatProtobuf.java | 5 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 1064 +-
.../hdfs/server/namenode/FileJournalManager.java | 48 +-
.../hdfs/server/namenode/GlobalStateIdContext.java | 4 +-
.../server/namenode/NNStorageRetentionManager.java | 46 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 50 +-
.../web/resources/NamenodeWebHdfsMethods.java | 51 +-
.../tools/offlineImageViewer/PBImageXmlWriter.java | 2 +-
.../src/main/proto/AliasMapProtocol.proto | 2 +-
.../src/main/proto/DatanodeLifelineProtocol.proto | 2 +-
.../src/main/proto/DatanodeProtocol.proto | 2 +-
.../hadoop-hdfs/src/main/proto/HAZKInfo.proto | 2 +-
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 2 +-
.../src/main/proto/InterDatanodeProtocol.proto | 2 +-
.../src/main/proto/InterQJournalProtocol.proto | 2 +-
.../src/main/proto/JournalProtocol.proto | 2 +-
.../src/main/proto/NamenodeProtocol.proto | 2 +-
.../src/main/proto/QJournalProtocol.proto | 2 +-
.../hadoop-hdfs/src/main/proto/editlog.proto | 2 +-
.../hadoop-hdfs/src/main/proto/fsimage.proto | 2 +-
.../src/main/resources/hdfs-default.xml | 81 +-
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 64 +
.../hadoop/fs/viewfs/TestViewFsDefaultValue.java | 1 +
.../java/org/apache/hadoop/hdfs/TestQuota.java | 49 +
.../java/org/apache/hadoop/hdfs/TestSetTimes.java | 7 +
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 2 +-
.../server/blockmanagement/TestBlockManager.java | 74 +
.../server/datanode/TestCorruptMetadataFile.java | 165 +
.../server/datanode/TestDataNodePeerMetrics.java | 2 +-
.../hdfs/server/datanode/TestLargeBlockReport.java | 3 +
.../hdfs/server/datanode/TestTransferRbw.java | 13 +-
.../TestDataNodeOutlierDetectionViaMetrics.java | 13 +-
.../namenode/TestNNStorageRetentionManager.java | 169 +-
.../hdfs/server/namenode/TestRedudantBlocks.java | 137 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 58 +
.../hdfs/web/TestWebHdfsFileSystemContract.java | 5 +-
.../apache/hadoop/hdfs/web/TestWebHdfsTokens.java | 2 +-
.../apache/hadoop/hdfs/web/WebHdfsTestUtil.java | 25 +-
.../hadoop/hdfs/web/resources/TestParam.java | 29 +
.../apache/hadoop/mapreduce/v2/app/TestMRApp.java | 4 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 6 +-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 3 +
.../src/site/markdown/EncryptedShuffle.md | 2 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 12 +
.../hadoop/ozone/client/OzoneClientException.java | 41 +-
.../hadoop/ozone/client/OzoneClientFactory.java | 150 +-
.../hadoop/ozone/client/OzoneClientUtils.java | 91 -
.../org/apache/hadoop/ozone/client/OzoneKey.java | 17 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 4 +-
.../hadoop/ozone/client/OzoneMultipartUpload.java | 89 +
.../ozone/client/OzoneMultipartUploadList.java | 46 +
.../client/OzoneMultipartUploadPartListParts.java | 10 +
.../hadoop/ozone/client/io/KeyOutputStream.java | 29 +-
.../ozone/client/protocol/ClientProtocol.java | 5 +
.../client/rest/DefaultRestServerSelector.java | 36 -
.../ozone/client/rest/OzoneExceptionMapper.java | 45 -
.../hadoop/ozone/client/rest/RestClient.java | 1130 --
.../ozone/client/rest/RestServerSelector.java | 40 -
.../ozone/client/rest/exceptions/package-info.java | 22 -
.../ozone/client/rest/headers/package-info.java | 22 -
.../hadoop/ozone/client/rest/package-info.java | 23 -
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 157 +-
hadoop-ozone/common/pom.xml | 3 -
hadoop-ozone/common/src/main/bin/ozone | 18 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 25 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 1 +
.../hadoop/ozone/client/rest/OzoneException.java | 267 -
.../hadoop/ozone/client/rest/headers/Header.java | 74 -
.../ozone/client/rest/headers/package-info.java | 22 -
.../hadoop/ozone/client/rest/package-info.java | 22 -
.../ozone/client/rest/response/BucketInfo.java | 248 -
.../hadoop/ozone/client/rest/response/KeyInfo.java | 236 -
.../ozone/client/rest/response/KeyInfoDetails.java | 118 -
.../ozone/client/rest/response/KeyLocation.java | 89 -
.../ozone/client/rest/response/VolumeInfo.java | 215 -
.../ozone/client/rest/response/VolumeOwner.java | 61 -
.../ozone/client/rest/response/package-info.java | 24 -
.../apache/hadoop/ozone/om/OMMetadataManager.java | 11 +-
.../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 2 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 2 +-
.../ozone/om/codec/OmMultipartKeyInfoCodec.java | 2 +-
.../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 2 +-
.../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 2 +-
.../hadoop/ozone/om/codec/S3SecretValueCodec.java | 2 +-
.../ozone/om/codec/TokenIdentifierCodec.java | 2 +-
.../hadoop/ozone/om/codec/VolumeListCodec.java | 2 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 53 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 2 +
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +
.../ozone/om/helpers/OmMultipartKeyInfo.java | 2 -
.../hadoop/ozone/om/helpers/OmMultipartUpload.java | 149 +
.../om/helpers/OmMultipartUploadCompleteList.java | 63 +
.../ozone/om/helpers/OmMultipartUploadList.java | 41 +-
.../om/helpers/OmMultipartUploadListParts.java | 13 +
.../hadoop/ozone/om/helpers/ServiceInfo.java | 23 -
.../hadoop/ozone/om/helpers/ServiceInfoEx.java | 47 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 21 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 80 +-
.../hadoop/ozone/security/GDPRSymmetricKey.java | 85 +
.../security/OzoneDelegationTokenSelector.java | 18 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 4 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +-
.../hadoop/ozone/web/handlers/BucketArgs.java | 134 -
.../apache/hadoop/ozone/web/handlers/KeyArgs.java | 117 -
.../apache/hadoop/ozone/web/handlers/ListArgs.java | 142 -
.../apache/hadoop/ozone/web/handlers/UserArgs.java | 172 -
.../hadoop/ozone/web/handlers/VolumeArgs.java | 143 -
.../hadoop/ozone/web/handlers/package-info.java | 22 -
.../hadoop/ozone/web/request/OzoneQuota.java | 214 -
.../hadoop/ozone/web/request/package-info.java | 23 -
.../hadoop/ozone/web/response/BucketInfo.java | 335 -
.../apache/hadoop/ozone/web/response/KeyInfo.java | 311 -
.../hadoop/ozone/web/response/KeyInfoDetails.java | 80 -
.../hadoop/ozone/web/response/KeyLocation.java | 82 -
.../hadoop/ozone/web/response/ListBuckets.java | 154 -
.../apache/hadoop/ozone/web/response/ListKeys.java | 209 -
.../hadoop/ozone/web/response/ListVolumes.java | 152 -
.../hadoop/ozone/web/response/VolumeInfo.java | 308 -
.../hadoop/ozone/web/response/VolumeOwner.java | 60 -
.../hadoop/ozone/web/response/package-info.java | 23 -
.../apache/hadoop/ozone/web/utils/OzoneUtils.java | 70 +-
.../src/main/proto/OzoneManagerProtocol.proto | 41 +-
.../ozone/om/helpers/TestOmMultipartUpload.java | 39 +
.../ozone/security/TestGDPRSymmetricKey.java | 67 +
.../security/TestOzoneDelegationTokenSelector.java | 87 +
.../apache/hadoop/ozone/web/TestBucketInfo.java | 82 -
.../org/apache/hadoop/ozone/web/TestQuota.java | 115 -
.../org/apache/hadoop/ozone/web/TestUtils.java | 101 -
.../apache/hadoop/ozone/web/TestVolumeStructs.java | 73 -
.../org/apache/hadoop/ozone/web/package-info.java | 21 -
hadoop-ozone/datanode/pom.xml | 4 -
hadoop-ozone/dev-support/checks/README.md | 27 +
.../dev-support/checks/_mvn_unit_report.sh | 69 +
hadoop-ozone/dev-support/checks/acceptance.sh | 17 +-
hadoop-ozone/dev-support/checks/author.sh | 16 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 26 +-
hadoop-ozone/dev-support/checks/findbugs.sh | 20 +-
hadoop-ozone/dev-support/checks/integration.sh | 12 +-
hadoop-ozone/dev-support/checks/rat.sh | 11 +-
hadoop-ozone/dev-support/checks/shellcheck.sh | 12 +-
hadoop-ozone/dev-support/checks/unit.sh | 17 +-
hadoop-ozone/dev-support/intellij/ozone-site.xml | 4 +
.../dist/dev-support/bin/dist-layout-stitching | 25 +-
.../dist/dev-support/bin/dist-tar-stitching | 6 +-
hadoop-ozone/dist/pom.xml | 12 +
.../dist/src/main/assemblies/ozone-src.xml | 101 +
.../main/compose/ozone-mr/hadoop27/docker-config | 1 -
.../main/compose/ozone-mr/hadoop31/docker-config | 1 -
.../main/compose/ozone-mr/hadoop32/docker-config | 1 -
.../src/main/compose/ozone-om-ha/docker-config | 11 +-
.../dist/src/main/compose/ozone-topology/test.sh | 3 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 3 +-
.../src/main/compose/ozonesecure-mr/docker-config | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 13 +-
.../src/main/conf/ozone-shell-log4j.properties | 33 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 443 +
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 520 +
.../license/bin/licenses}/LICENSE-angular-nvd3.txt | 0
.../main/license/bin/licenses}/LICENSE-angular.txt | 0
.../LICENSE-com.google.code.findbugs-jsr305.txt | 8 +
.../bin/licenses/LICENSE-com.google.re2j-re2j.txt | 32 +
.../bin/licenses/LICENSE-com.jcraft-jsch.txt | 30 +
.../bin/licenses/LICENSE-com.sun.jersey.txt | 274 +
.../bin/licenses/LICENSE-com.sun.xml.bind.txt | 759 ++
...ICENSE-com.thoughtworks.paranamer-paranamer.txt | 29 +
.../src/main/license/bin/licenses}/LICENSE-d3.txt | 0
.../bin/licenses/LICENSE-dnsjava-dnsjava.txt | 24 +
...E-jakarta.annotation-jakarta.annotation-api.txt | 277 +
.../LICENSE-javax.activation-activation.txt | 134 +
...CENSE-javax.annotation-javax.annotation-api.txt | 263 +
.../bin/licenses/LICENSE-javax.el-javax.el-api.txt | 263 +
...NSE-javax.interceptor-javax.interceptor-api.txt | 263 +
.../LICENSE-javax.servlet-javax.servlet-api.txt | 263 +
.../licenses/LICENSE-javax.servlet.jsp-jsp-api.txt | 759 ++
.../LICENSE-javax.ws.rs-javax.ws.rs-api.txt | 759 ++
.../licenses/LICENSE-javax.ws.rs-jsr311-api.txt | 759 ++
.../licenses/LICENSE-javax.xml.bind-jaxb-api.txt | 274 +
.../main/license/bin/licenses/LICENSE-jersey.txt | 759 ++
.../main/license/bin/licenses/LICENSE-jetty.txt | 415 +
.../main/license/bin/licenses}/LICENSE-jquery.txt | 0
.../LICENSE-net.sf.jopt-simple-jopt-simple.txt | 24 +
.../main/license/bin/licenses}/LICENSE-nvd3.txt | 0
...rg.codehaus.mojo-animal-sniffer-annotations.txt | 21 +
.../LICENSE-org.codehaus.woodstox-stax2-api.txt | 13 +
...SE-org.fusesource.leveldbjni-leveldbjni-all.txt | 27 +
.../bin/licenses/LICENSE-org.glassfish.hk2.txt | 759 ++
.../licenses/LICENSE-org.openjdk.jmh-jmh-core.txt | 347 +
...SE-org.openjdk.jmh-jmh-generator-annprocess.txt | 347 +
.../bin/licenses/LICENSE-org.ow2.asm-asm.txt | 28 +
.../license/bin/licenses/LICENSE-org.slf4j.txt | 21 +
.../main/license/bin/licenses/LICENSE-protobuf.txt | 32 +
.../bin/licenses/LICENSE-ratis-thirdparty-misc.txt | 353 +
.../bin/licenses/NOTICE-ratis-thirtparty-misc.txt | 340 +
hadoop-ozone/dist/src/main/license/src/LICENSE.txt | 239 +
hadoop-ozone/dist/src/main/license/src/NOTICE.txt | 33 +
.../src/main/license/src/licenses/IMPORTANT.md | 21 +
.../license/src/licenses}/LICENSE-angular-nvd3.txt | 0
.../main/license/src/licenses}/LICENSE-angular.txt | 0
.../src/main/license/src/licenses}/LICENSE-d3.txt | 0
.../main/license/src/licenses}/LICENSE-jquery.txt | 0
.../main/license/src/licenses}/LICENSE-nvd3.txt | 0
.../src/main/smoketest/basic/ozone-shell.robot | 44 +-
.../dist/src/main/smoketest/createbucketenv.robot | 1 -
.../dist/src/main/smoketest/createmrenv.robot | 1 -
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 72 +
.../insight/dev-support/findbugsExcludeFile.xml | 19 +
hadoop-ozone/insight/pom.xml | 132 +
.../hadoop/ozone/insight/BaseInsightPoint.java | 188 +
.../ozone/insight/BaseInsightSubCommand.java | 101 +
.../org/apache/hadoop/ozone/insight/Component.java | 116 +
.../ozone/insight/ConfigurationSubCommand.java | 89 +
.../org/apache/hadoop/ozone/insight/Insight.java | 41 +
.../apache/hadoop/ozone/insight/InsightPoint.java | 49 +
.../hadoop/ozone/insight/ListSubCommand.java | 59 +
.../apache/hadoop/ozone/insight/LogSubcommand.java | 167 +
.../apache/hadoop/ozone/insight/LoggerSource.java | 72 +
.../apache/hadoop/ozone/insight/MetricDisplay.java | 69 +
.../hadoop/ozone/insight/MetricGroupDisplay.java | 69 +
.../hadoop/ozone/insight/MetricsSubCommand.java | 132 +
.../ozone/insight/datanode/RatisInsight.java | 75 +
.../ozone/insight/datanode/package-info.java | 23 +
.../hadoop/ozone/insight/om/KeyManagerInsight.java | 78 +
.../hadoop/ozone/insight/om/OmProtocolInsight.java | 67 +
.../hadoop/ozone/insight/om/package-info.java | 23 +
.../apache/hadoop/ozone/insight/package-info.java | 24 +
.../ozone/insight/scm/EventQueueInsight.java | 47 +
.../ozone/insight/scm/NodeManagerInsight.java | 74 +
.../ozone/insight/scm/ReplicaManagerInsight.java | 60 +
.../scm/ScmProtocolBlockLocationInsight.java | 71 +
.../hadoop/ozone/insight/scm/package-info.java | 23 +
.../hadoop/ozone/insight/LogSubcommandTest.java | 41 +
hadoop-ozone/integration-test/pom.xml | 4 -
.../scm/pipeline/TestPipelineStateManager.java | 7 +
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 71 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 24 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 122 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 19 +-
.../org/apache/hadoop/ozone/RatisTestHelper.java | 28 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 95 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 21 +-
.../org/apache/hadoop/ozone/TestOzoneHelper.java | 413 -
.../hadoop/ozone/TestStorageContainerManager.java | 257 +-
.../ozone/TestStorageContainerManagerHelper.java | 76 +-
.../ozone/client/CertificateClientTestImpl.java | 7 +-
.../rpc/TestContainerReplicationEndToEnd.java | 219 +
.../client/rpc/TestContainerStateMachine.java | 62 +-
.../rpc/TestContainerStateMachineFailures.java | 88 +
.../client/rpc/TestDeleteWithSlowFollower.java | 291 +
.../client/rpc/TestOzoneRpcClientAbstract.java | 130 +-
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 5 +-
.../ozone/container/ContainerTestHelper.java | 33 +-
.../ozone/container/TestContainerReplication.java | 51 +-
.../container/common/TestBlockDeletingService.java | 38 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 36 +-
.../commandhandler/TestBlockDeletion.java | 23 +-
.../TestCloseContainerByPipeline.java | 86 +-
.../commandhandler/TestCloseContainerHandler.java | 47 +-
.../transport/server/ratis/TestCSMMetrics.java | 25 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 104 +-
.../container/server/TestContainerServer.java | 9 +-
.../server/TestSecureContainerServer.java | 12 +-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 6 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 10 +-
.../ozone/om/TestMultipleContainerReadWrite.java | 208 -
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 7 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 95 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 93 +-
.../org/apache/hadoop/ozone/om/TestOmInit.java | 18 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +
.../apache/hadoop/ozone/om/TestOzoneManager.java | 1449 ---
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 13 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 28 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 53 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 54 +-
.../snapshot/TestOzoneManagerSnapshotProvider.java | 9 +-
.../ozone/ozShell/TestOzoneDatanodeShell.java | 28 +-
.../hadoop/ozone/ozShell/TestOzoneShell.java | 1266 ---
.../hadoop/ozone/ozShell/TestOzoneShellHA.java | 343 +
.../apache/hadoop/ozone/ozShell/TestS3Shell.java | 2 +-
.../TestSCMContainerPlacementPolicyMetrics.java | 156 +
.../hadoop/ozone/scm/TestSCMNodeManagerMXBean.java | 8 +
.../hadoop/ozone/scm/TestXceiverClientManager.java | 21 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 5 +-
.../ozone/web/TestOzoneRestWithMiniCluster.java | 221 -
.../apache/hadoop/ozone/web/TestOzoneVolumes.java | 187 -
.../hadoop/ozone/web/TestOzoneWebAccess.java | 118 -
.../hadoop/ozone/web/client/TestBuckets.java | 349 -
.../hadoop/ozone/web/client/TestBucketsRatis.java | 108 -
.../apache/hadoop/ozone/web/client/TestKeys.java | 734 --
.../hadoop/ozone/web/client/TestKeysRatis.java | 126 -
.../hadoop/ozone/web/client/TestOzoneClient.java | 304 -
.../apache/hadoop/ozone/web/client/TestVolume.java | 381 -
.../hadoop/ozone/web/client/TestVolumeRatis.java | 155 -
.../hadoop/ozone/web/client/package-info.java | 22 -
.../org/apache/hadoop/ozone/web/package-info.java | 22 -
.../{log4j2.properties => auditlog.properties} | 0
hadoop-ozone/objectstore-service/pom.xml | 126 -
.../hdfs/server/datanode/ObjectStoreHandler.java | 160 -
.../hadoop/hdfs/server/datanode/package-info.java | 22 -
.../org/apache/hadoop/ozone/OzoneRestUtils.java | 222 -
.../java/org/apache/hadoop/ozone/package-info.java | 22 -
.../hadoop/ozone/web/ObjectStoreApplication.java | 59 -
.../hadoop/ozone/web/OzoneHddsDatanodeService.java | 89 -
.../hadoop/ozone/web/exceptions/ErrorTable.java | 225 -
.../hadoop/ozone/web/exceptions/package-info.java | 22 -
.../hadoop/ozone/web/handlers/BucketHandler.java | 190 -
.../ozone/web/handlers/BucketProcessTemplate.java | 294 -
.../hadoop/ozone/web/handlers/KeyHandler.java | 302 -
.../ozone/web/handlers/KeyProcessTemplate.java | 235 -
.../hadoop/ozone/web/handlers/ServiceFilter.java | 61 -
.../ozone/web/handlers/StorageHandlerBuilder.java | 80 -
.../ozone/web/handlers/UserHandlerBuilder.java | 75 -
.../hadoop/ozone/web/handlers/VolumeHandler.java | 274 -
.../ozone/web/handlers/VolumeProcessTemplate.java | 276 -
.../hadoop/ozone/web/handlers/package-info.java | 22 -
.../hadoop/ozone/web/interfaces/Accounting.java | 57 -
.../apache/hadoop/ozone/web/interfaces/Bucket.java | 184 -
.../apache/hadoop/ozone/web/interfaces/Keys.java | 177 -
.../ozone/web/interfaces/StorageHandler.java | 308 -
.../hadoop/ozone/web/interfaces/UserAuth.java | 101 -
.../apache/hadoop/ozone/web/interfaces/Volume.java | 185 -
.../hadoop/ozone/web/interfaces/package-info.java | 22 -
.../ozone/web/localstorage/package-info.java | 18 -
.../LengthInputStreamMessageBodyWriter.java | 59 -
.../web/messages/StringMessageBodyWriter.java | 62 -
.../hadoop/ozone/web/messages/package-info.java | 18 -
.../ozone/web/netty/CloseableCleanupListener.java | 46 -
.../ozone/web/netty/ObjectStoreChannelHandler.java | 78 -
.../web/netty/ObjectStoreJerseyContainer.java | 348 -
.../netty/ObjectStoreJerseyContainerProvider.java | 40 -
.../ozone/web/netty/ObjectStoreRestHttpServer.java | 215 -
.../ozone/web/netty/ObjectStoreURLDispatcher.java | 61 -
.../RequestContentObjectStoreChannelHandler.java | 117 -
.../RequestDispatchObjectStoreChannelHandler.java | 103 -
.../hadoop/ozone/web/netty/package-info.java | 26 -
.../org/apache/hadoop/ozone/web/package-info.java | 22 -
.../web/storage/DistributedStorageHandler.java | 620 --
.../hadoop/ozone/web/storage/package-info.java | 27 -
.../apache/hadoop/ozone/web/userauth/Simple.java | 169 -
.../hadoop/ozone/web/userauth/package-info.java | 23 -
.../com.sun.jersey.spi.container.ContainerProvider | 16 -
.../org/apache/hadoop/ozone/web/TestErrorCode.java | 53 -
.../org/apache/hadoop/ozone/web/package-info.java | 22 -
.../apache/hadoop/ozone/om/KeyDeletingService.java | 16 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 147 +-
.../hadoop/ozone/om/OMDBCheckpointServlet.java | 4 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 57 +-
.../org/apache/hadoop/ozone/om/OMNodeDetails.java | 13 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 60 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 8 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 115 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 4 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 2 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 2 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 4 +-
.../om/request/bucket/OMBucketCreateRequest.java | 6 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 4 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 4 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 4 +-
.../om/request/file/OMDirectoryCreateRequest.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 10 +-
.../om/request/key/OMAllocateBlockRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 4 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 7 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 13 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 4 +-
.../request/key/acl/prefix/OMPrefixAclRequest.java | 4 +-
.../request/s3/bucket/S3BucketCreateRequest.java | 4 +-
.../request/s3/bucket/S3BucketDeleteRequest.java | 4 +-
.../S3InitiateMultipartUploadRequest.java | 6 +-
.../multipart/S3MultipartUploadAbortRequest.java | 6 +-
.../S3MultipartUploadCommitPartRequest.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 37 +-
.../om/request/s3/security/S3GetSecretRequest.java | 4 +-
.../security/OMCancelDelegationTokenRequest.java | 4 +-
.../security/OMGetDelegationTokenRequest.java | 75 +-
.../security/OMRenewDelegationTokenRequest.java | 8 +-
.../om/request/volume/OMVolumeCreateRequest.java | 17 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 4 +-
.../ozone/om/request/volume/OMVolumeRequest.java | 4 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 4 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 4 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 2 +-
.../om/response/bucket/OMBucketCreateResponse.java | 2 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 2 +-
.../bucket/OMBucketSetPropertyResponse.java | 2 +-
.../response/bucket/acl/OMBucketAclResponse.java | 2 +-
.../response/file/OMDirectoryCreateResponse.java | 2 +-
.../om/response/key/OMAllocateBlockResponse.java | 2 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 2 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 2 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 2 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 2 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 2 +-
.../om/response/key/acl/OMKeyAclResponse.java | 2 +-
.../key/acl/prefix/OMPrefixAclResponse.java | 2 +-
.../response/s3/bucket/S3BucketCreateResponse.java | 2 +-
.../response/s3/bucket/S3BucketDeleteResponse.java | 2 +-
.../S3InitiateMultipartUploadResponse.java | 2 +-
.../multipart/S3MultipartUploadAbortResponse.java | 2 +-
.../S3MultipartUploadCommitPartResponse.java | 2 +-
.../S3MultipartUploadCompleteResponse.java | 2 +-
.../response/s3/security/S3GetSecretResponse.java | 2 +-
.../security/OMCancelDelegationTokenResponse.java | 4 +-
.../security/OMGetDelegationTokenResponse.java | 7 +-
.../security/OMRenewDelegationTokenResponse.java | 4 +-
.../om/response/volume/OMVolumeAclOpResponse.java | 2 +-
.../om/response/volume/OMVolumeCreateResponse.java | 2 +-
.../om/response/volume/OMVolumeDeleteResponse.java | 2 +-
.../response/volume/OMVolumeSetOwnerResponse.java | 2 +-
.../response/volume/OMVolumeSetQuotaResponse.java | 2 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 163 +-
.../protocolPB/OzoneManagerRequestHandler.java | 57 +-
.../hadoop/ozone/web/ozShell/ObjectPrinter.java | 39 +
.../hadoop/ozone/web/ozShell/OzoneAddress.java | 38 +-
.../web/ozShell/bucket/CreateBucketHandler.java | 25 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 7 +-
.../web/ozShell/bucket/ListBucketHandler.java | 20 +-
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 12 +-
.../ozone/web/ozShell/keys/ListKeyHandler.java | 23 +-
.../ozone/web/ozShell/keys/PutKeyHandler.java | 8 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 6 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 6 +-
.../web/ozShell/volume/ListVolumeHandler.java | 21 +-
.../web/ozShell/volume/UpdateVolumeHandler.java | 14 +-
.../resources/webapps/ozoneManager/om-metrics.html | 2 +-
.../resources/webapps/ozoneManager/ozoneManager.js | 6 +-
.../hadoop/ozone/om/TestBucketManagerImpl.java | 12 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 11 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 180 +
.../hadoop/ozone/om/TestS3BucketManager.java | 70 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 171 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +
.../ozone/om/request/TestOMRequestUtils.java | 82 +-
.../request/bucket/TestOMBucketCreateRequest.java | 4 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 2 +-
.../request/volume/TestOMVolumeDeleteRequest.java | 6 +-
.../volume/acl/TestOMVolumeAddAclRequest.java | 1 -
.../volume/acl/TestOMVolumeSetAclRequest.java | 1 -
.../bucket/TestOMBucketCreateResponse.java | 17 +-
.../bucket/TestOMBucketDeleteResponse.java | 2 +-
.../bucket/TestOMBucketSetPropertyResponse.java | 15 +-
.../file/TestOMDirectoryCreateResponse.java | 2 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 2 +-
.../s3/bucket/TestS3BucketCreateResponse.java | 17 +-
.../s3/bucket/TestS3BucketDeleteResponse.java | 2 +-
.../s3/multipart/TestS3MultipartResponse.java | 2 +-
.../volume/TestOMVolumeCreateResponse.java | 8 +-
.../volume/TestOMVolumeDeleteResponse.java | 2 +-
.../volume/TestOMVolumeSetOwnerResponse.java | 15 +-
.../volume/TestOMVolumeSetQuotaResponse.java | 15 +-
.../ozone/web/ozShell/TestObjectPrinter.java | 50 +
.../hadoop/ozone/web/ozShell/TestOzoneAddress.java | 7 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 185 -
.../recon/recovery/ReconOmMetadataManagerImpl.java | 96 -
.../recon/spi/ContainerDBServiceProvider.java | 159 -
.../spi/impl/ContainerDBServiceProviderImpl.java | 394 -
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 87 -
.../spi/impl/OzoneManagerServiceProviderImpl.java | 347 -
.../recon/spi/impl/ReconContainerDBProvider.java | 89 -
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 235 -
.../ozone/recon/tasks/FileSizeCountTask.java | 251 -
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 225 -
.../webapps/recon/ozone-recon-web/package.json | 45 -
.../src/components/Breadcrumbs/Breadcrumbs.tsx | 58 -
.../src/components/NavBar/NavBar.tsx | 67 -
.../webapps/recon/ozone-recon-web/yarn.lock | 9638 ----------------
.../ozone/recon/AbstractOMMetadataManagerTest.java | 231 -
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 58 -
.../ozone/recon/api/TestContainerKeyService.java | 373 -
.../recovery/TestReconOmMetadataManagerImpl.java | 131 -
.../impl/TestOzoneManagerServiceProviderImpl.java | 317 -
.../spi/impl/TestReconContainerDBProvider.java | 87 -
.../recon/tasks/TestContainerKeyMapperTask.java | 314 -
.../ozone/recon/tasks/TestFileSizeCountTask.java | 130 -
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 207 -
.../recon/types/GuiceInjectorUtilsForTests.java | 117 -
hadoop-ozone/ozonefs-lib-current/pom.xml | 155 +-
.../services/org.apache.hadoop.fs.FileSystem | 0
.../services/org.apache.hadoop.fs.FileSystem | 16 +
hadoop-ozone/ozonefs/pom.xml | 5 -
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 34 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 35 +-
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 90 +
.../org.apache.hadoop.security.token.TokenRenewer | 2 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 44 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 77 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 99 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 348 +
.../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 35 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 42 +-
hadoop-ozone/pom.xml | 30 +-
.../{ozone-recon-codegen => recon-codegen}/pom.xml | 0
.../ozone/recon/codegen/JooqCodeGenerator.java | 0
.../recon/codegen/ReconSchemaGenerationModule.java | 0
.../ozone/recon/codegen/TableNamingStrategy.java | 0
.../hadoop/ozone/recon/codegen/package-info.java | 0
.../schema/ReconInternalSchemaDefinition.java | 0
.../ozone/recon/schema/ReconSchemaDefinition.java | 0
.../ozone/recon/schema/StatsSchemaDefinition.java | 0
.../recon/schema/UtilizationSchemaDefinition.java | 0
.../hadoop/ozone/recon/schema/package-info.java | 0
.../dev-support/findbugsExcludeFile.xml | 0
hadoop-ozone/{ozone-recon => recon}/pom.xml | 0
.../hadoop/ozone/recon/ConfigurationProvider.java | 0
.../apache/hadoop/ozone/recon/ReconConstants.java | 0
.../hadoop/ozone/recon/ReconControllerModule.java | 185 +
.../recon/ReconGuiceServletContextListener.java | 0
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 0
.../hadoop/ozone/recon/ReconRestServletModule.java | 0
.../org/apache/hadoop/ozone/recon/ReconServer.java | 0
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 0
.../hadoop/ozone/recon/ReconTaskBindingModule.java | 0
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 0
.../ozone/recon/api/ContainerKeyService.java | 0
.../hadoop/ozone/recon/api/UtilizationService.java | 0
.../hadoop/ozone/recon/api/package-info.java | 0
.../ozone/recon/api/types/ContainerKeyPrefix.java | 0
.../ozone/recon/api/types/ContainerMetadata.java | 0
.../ozone/recon/api/types/ContainersResponse.java | 0
.../ozone/recon/api/types/IsoDateAdapter.java | 0
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 0
.../hadoop/ozone/recon/api/types/KeysResponse.java | 0
.../hadoop/ozone/recon/api/types/package-info.java | 0
.../apache/hadoop/ozone/recon/package-info.java | 0
.../recon/persistence/DataSourceConfiguration.java | 0
.../persistence/DefaultDataSourceProvider.java | 0
.../recon/persistence/JooqPersistenceModule.java | 0
.../TransactionalMethodInterceptor.java | 0
.../ozone/recon/persistence/package-info.java | 0
.../recon/recovery/ReconOMMetadataManager.java | 0
.../recon/recovery/ReconOmMetadataManagerImpl.java | 96 +
.../hadoop/ozone/recon/recovery/package-info.java | 0
.../recon/spi/ContainerDBServiceProvider.java | 159 +
.../recon/spi/HddsDatanodeServiceProvider.java | 0
.../recon/spi/OzoneManagerServiceProvider.java | 0
.../recon/spi/StorageContainerServiceProvider.java | 0
.../spi/impl/ContainerDBServiceProviderImpl.java | 394 +
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 87 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 347 +
.../recon/spi/impl/ReconContainerDBProvider.java | 89 +
.../hadoop/ozone/recon/spi/impl/package-info.java | 0
.../hadoop/ozone/recon/spi/package-info.java | 0
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 235 +
.../ozone/recon/tasks/FileSizeCountTask.java | 251 +
.../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 0
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 225 +
.../ozone/recon/tasks/OMUpdateEventBatch.java | 0
.../ozone/recon/tasks/ReconDBUpdateTask.java | 0
.../ozone/recon/tasks/ReconTaskController.java | 0
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 0
.../hadoop/ozone/recon/tasks/package-info.java | 0
.../main/resources/webapps/recon/WEB-INF/web.xml | 0
.../webapps/recon/ozone-recon-web/.gitignore | 0
.../webapps/recon/ozone-recon-web/LICENSE | 0
.../resources/webapps/recon/ozone-recon-web/NOTICE | 0
.../webapps/recon/ozone-recon-web/README.md | 0
.../recon/ozone-recon-web/config-overrides.js | 0
.../webapps/recon/ozone-recon-web/package.json | 47 +
.../recon/ozone-recon-web/public/favicon.ico | Bin
.../recon/ozone-recon-web/public/index.html | 0
.../recon/ozone-recon-web/public/manifest.json | 0
.../webapps/recon/ozone-recon-web/src/App.less | 0
.../webapps/recon/ozone-recon-web/src/App.test.tsx | 0
.../webapps/recon/ozone-recon-web/src/App.tsx | 0
.../src/components/Breadcrumbs/Breadcrumbs.tsx | 58 +
.../src/components/NavBar/NavBar.less | 0
.../src/components/NavBar/NavBar.tsx | 67 +
.../src/constants/breadcrumbs.constants.tsx | 0
.../webapps/recon/ozone-recon-web/src/index.less | 0
.../webapps/recon/ozone-recon-web/src/index.tsx | 0
.../webapps/recon/ozone-recon-web/src/logo.png | Bin
.../ozone-recon-web/src/makeRouteWithSubRoutes.tsx | 0
.../recon/ozone-recon-web/src/react-app-env.d.ts | 0
.../webapps/recon/ozone-recon-web/src/routes.tsx | 0
.../recon/ozone-recon-web/src/routes.types.tsx | 0
.../recon/ozone-recon-web/src/serviceWorker.ts | 0
.../views/ContainerBrowser/ContainerBrowser.tsx | 0
.../src/views/Dashboard/Dashboard.tsx | 0
.../src/views/NotFound/NotFound.tsx | 0
.../webapps/recon/ozone-recon-web/tsconfig.json | 0
.../webapps/recon/ozone-recon-web/yarn.lock | 11114 +++++++++++++++++++
.../ozone/recon/AbstractOMMetadataManagerTest.java | 232 +
.../recon/GuiceInjectorUtilsForTestsImpl.java | 0
.../apache/hadoop/ozone/recon/TestReconCodecs.java | 58 +
.../apache/hadoop/ozone/recon/TestReconUtils.java | 0
.../ozone/recon/api/TestContainerKeyService.java | 373 +
.../ozone/recon/api/TestUtilizationService.java | 0
.../hadoop/ozone/recon/api/package-info.java | 0
.../apache/hadoop/ozone/recon/package-info.java | 0
.../recon/persistence/AbstractSqlDatabaseTest.java | 0
.../TestReconInternalSchemaDefinition.java | 0
.../persistence/TestStatsSchemaDefinition.java | 0
.../TestUtilizationSchemaDefinition.java | 0
.../ozone/recon/persistence/package-info.java | 0
.../recovery/TestReconOmMetadataManagerImpl.java | 140 +
.../hadoop/ozone/recon/recovery/package-info.java | 0
.../impl/TestContainerDBServiceProviderImpl.java | 0
.../impl/TestOzoneManagerServiceProviderImpl.java | 317 +
.../spi/impl/TestReconContainerDBProvider.java | 87 +
.../hadoop/ozone/recon/spi/impl/package-info.java | 0
.../hadoop/ozone/recon/tasks/DummyReconDBTask.java | 0
.../recon/tasks/TestContainerKeyMapperTask.java | 314 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 130 +
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 207 +
.../recon/tasks/TestReconTaskControllerImpl.java | 0
.../hadoop/ozone/recon/tasks/package-info.java | 0
.../recon/types/GuiceInjectorUtilsForTests.java | 117 +
.../org.mockito.plugins.MockMaker | 0
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 30 +-
.../hadoop/ozone/s3/endpoint/CopyPartResult.java | 69 +
.../s3/endpoint/ListMultipartUploadsResult.java | 268 +
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 89 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 11 +-
.../src/main/resources/webapps/static/index.html | 8 +-
.../src/main/resources/webapps/static/s3g.js | 23 +
.../hadoop/ozone/client/OzoneBucketStub.java | 25 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 25 +-
.../hadoop/ozone/s3/endpoint/TestBucketGet.java | 39 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 233 +
.../ozone/s3/endpoint/TestObjectEndpoint.java | 53 +
hadoop-ozone/tools/pom.xml | 9 +
.../hadoop/ozone/freon/BaseFreonGenerator.java | 333 +
.../hadoop/ozone/freon/ContentGenerator.java | 62 +
.../java/org/apache/hadoop/ozone/freon/Freon.java | 11 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 99 +
.../hadoop/ozone/freon/HadoopFsValidator.java | 100 +
.../hadoop/ozone/freon/OmBucketGenerator.java | 85 +
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 100 +
.../ozone/freon/OzoneClientKeyGenerator.java | 114 +
.../ozone/freon/OzoneClientKeyValidator.java | 99 +
.../org/apache/hadoop/ozone/freon/PathSchema.java | 38 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 11 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 109 +
.../apache/hadoop/ozone/freon/SameKeyReader.java | 104 +
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 4 +-
.../ozone/genesis/BenchMarkMetadataStoreReads.java | 2 +-
.../genesis/BenchMarkMetadataStoreWrites.java | 2 +-
.../ozone/genesis/BenchMarkRocksDbStore.java | 4 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +-
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 +-
.../freon/TestDataValidateWithDummyContainers.java | 74 +
.../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 92 +-
hadoop-project-dist/pom.xml | 2 +-
hadoop-project/pom.xml | 55 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 74 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 3 +-
.../hadoop/fs/s3a/InconsistentAmazonS3Client.java | 32 +-
.../apache/hadoop/fs/s3a/InternalConstants.java | 53 -
.../java/org/apache/hadoop/fs/s3a/Listing.java | 8 +-
.../hadoop/fs/s3a/RemoteFileChangedException.java | 16 +-
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 56 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 539 +-
.../apache/hadoop/fs/s3a/S3ALocatedFileStatus.java | 28 +-
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 52 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 76 +-
.../hadoop/fs/s3a/S3GuardExistsRetryPolicy.java | 32 +-
.../org/apache/hadoop/fs/s3a/SignerManager.java | 99 +
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 6 +-
.../fs/s3a/auth/MarshalledCredentialBinding.java | 32 +-
.../hadoop/fs/s3a/auth/STSClientFactory.java | 8 +-
.../s3a/auth/delegation/SessionTokenBinding.java | 4 +-
.../hadoop/fs/s3a/commit/CommitOperations.java | 10 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 23 +-
.../apache/hadoop/fs/s3a/impl/ChangeTracker.java | 5 +-
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 577 +
.../fs/s3a/impl/ExecutingStoreOperation.java | 69 +
.../hadoop/fs/s3a/impl/InternalConstants.java | 28 +
.../fs/s3a/impl/MultiObjectDeleteSupport.java | 10 +-
.../apache/hadoop/fs/s3a/impl/NetworkBinding.java | 113 +
.../hadoop/fs/s3a/impl/OperationCallbacks.java | 198 +
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 164 +-
.../apache/hadoop/fs/s3a/impl/StatusProbeEnum.java | 44 +
.../apache/hadoop/fs/s3a/impl/StoreContext.java | 19 +
.../fs/s3a/s3guard/DelayedUpdateRenameTracker.java | 4 +-
.../fs/s3a/s3guard/DynamoDBClientFactory.java | 4 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 77 +-
.../hadoop/fs/s3a/s3guard/InternalIterators.java | 91 +
.../hadoop/fs/s3a/s3guard/LocalMetadataStore.java | 16 +-
.../hadoop/fs/s3a/s3guard/MetadataStore.java | 27 +-
.../hadoop/fs/s3a/s3guard/NullMetadataStore.java | 12 +-
.../fs/s3a/s3guard/ProgressiveRenameTracker.java | 3 +-
.../org/apache/hadoop/fs/s3a/s3guard/S3Guard.java | 9 +
.../apache/hadoop/fs/s3a/s3guard/S3GuardFsck.java | 484 +
.../s3a/s3guard/S3GuardFsckViolationHandler.java | 346 +
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 130 +-
.../fs/s3a/select/InternalSelectConstants.java | 2 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 6 +-
.../site/markdown/tools/hadoop-aws/performance.md | 51 +
.../tools/hadoop-aws/troubleshooting_s3a.md | 109 +
.../fs/contract/s3a/ITestS3AContractRootDir.java | 3 +-
.../fs/contract/s3a/ITestS3AContractSeek.java | 22 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 68 +
.../hadoop/fs/s3a/ITestS3AEmptyDirectory.java | 5 +-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 8 +-
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 20 +-
.../s3a/ITestS3AMetadataPersistenceException.java | 42 +-
.../hadoop/fs/s3a/ITestS3ARemoteFileChanged.java | 149 +-
.../fs/s3a/ITestS3ATemporaryCredentials.java | 2 +-
.../hadoop/fs/s3a/ITestS3GuardEmptyDirs.java | 10 +-
.../hadoop/fs/s3a/ITestS3GuardListConsistency.java | 126 +-
.../fs/s3a/ITestS3GuardOutOfBandOperations.java | 81 +-
.../org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java | 16 +-
.../hadoop/fs/s3a/ITestS3GuardWriteBack.java | 1 +
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 5 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 11 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 40 +-
.../apache/hadoop/fs/s3a/TestSignerManager.java | 130 +
.../fs/s3a/commit/AbstractITCommitMRJob.java | 25 +-
.../fs/s3a/commit/ITestCommitOperations.java | 4 +-
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 4 +-
.../fs/s3a/impl/TestPartialDeleteFailures.java | 16 +-
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 45 +-
.../s3guard/ITestDynamoDBMetadataStoreScale.java | 4 +-
.../hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java | 504 +
.../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java | 30 +
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 18 +-
.../scale/AbstractITestS3AMetadataStoreScale.java | 2 +-
.../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 94 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 4 +
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../java/org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 4 +
.../src/site/markdown/troubleshooting_adl.md | 10 +
.../fs/adl/live/TestAdlSdkConfiguration.java | 32 +-
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 +-
.../constants/FileSystemConfigurations.java | 6 +-
.../hadoop/fs/azurebfs/services/AbfsClient.java | 8 +-
.../fs/azurebfs/services/AbfsHttpOperation.java | 4 +-
.../fs/azurebfs/utils/SSLSocketFactoryEx.java | 246 -
.../TestAbfsConfigurationFieldsValidation.java | 16 +-
.../fs/azurebfs/services/TestAbfsClient.java | 6 +-
.../apache/hadoop/tools/dynamometer/Client.java | 59 +-
.../audit/AuditLogDirectParser.java | 19 +-
.../audit/TestAuditLogDirectParser.java | 9 +
.../hadoop/hdfs/server/namenode/ImageWriter.java | 4 +-
hadoop-tools/hadoop-sls/pom.xml | 1 +
.../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 10 +-
.../hadoop/yarn/sls/appmaster/DAGAMSimulator.java | 288 +
.../hadoop/yarn/sls/conf/SLSConfiguration.java | 3 +
.../yarn/sls/scheduler/ContainerSimulator.java | 13 +-
.../apache/hadoop/yarn/sls/TestDagAMSimulator.java | 80 +
.../hadoop/yarn/sls/TestSLSDagAMSimulator.java | 76 +
.../hadoop/yarn/sls/appmaster/TestAMSimulator.java | 6 +-
.../hadoop-sls/src/test/resources/sls-runner.xml | 4 +
.../hadoop-sls/src/test/resources/sls_dag.json | 62 +
hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 6 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 +
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 42 +
.../hadoop/yarn/service/client/ServiceClient.java | 3 +
.../service/exceptions/RestApiErrorMessages.java | 1 +
.../hadoop/yarn/service/utils/JsonSerDeser.java | 14 +-
.../hadoop/yarn/service/utils/ServiceApiUtil.java | 12 +
.../yarn/service/utils/TestServiceApiUtil.java | 6 +
.../pb/PlacementConstraintToProtoConverter.java | 22 +-
.../impl/pb/NodePublishVolumeRequestPBImpl.java | 4 +-
.../ValidateVolumeCapabilitiesRequestPBImpl.java | 4 +-
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 39 +
.../LogAggregationFileController.java | 25 +-
.../util/resource/DominantResourceCalculator.java | 26 +-
.../yarn/webapp/GenericExceptionHandler.java | 2 -
.../src/main/resources/yarn-default.xml | 51 +
.../hadoop/yarn/event/TestAsyncDispatcher.java | 57 +
.../TestLogAggregationFileController.java | 91 +
.../yarn/util/resource/TestResourceCalculator.java | 23 +
.../server/nodemanager/DirectoryCollection.java | 7 +-
.../containermanager/launcher/ContainerLaunch.java | 3 +-
.../localizer/ResourceLocalizationService.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 23 +
.../server/resourcemanager/ClusterMetrics.java | 9 +
.../yarn/server/resourcemanager/RMAppManager.java | 13 +-
.../server/resourcemanager/ResourceManager.java | 3 +-
.../metrics/TimelineServiceV2Publisher.java | 2 +
.../preprocessor/ContextProcessor.java | 44 +
.../preprocessor/NodeLabelProcessor.java | 33 +
.../preprocessor/QueueProcessor.java | 34 +
.../SubmissionContextPreProcessor.java | 223 +
.../preprocessor/TagAddProcessor.java | 44 +
.../resourcemanager/preprocessor/package-info.java | 28 +
.../resourcemanager/recovery/RMStateStore.java | 7 +
.../recovery/RMStateUpdateAppEvent.java | 7 +-
.../resourcemanager/recovery/ZKRMStateStore.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 8 +
.../resourcemanager/scheduler/QueueMetrics.java | 31 +
.../scheduler/QueueMetricsForCustomResources.java | 6 +
.../scheduler/activities/ActivitiesLogger.java | 184 +-
.../scheduler/activities/ActivitiesManager.java | 39 +-
.../scheduler/activities/ActivitiesUtils.java | 12 +-
.../activities/ActivityDiagnosticConstant.java | 112 +-
.../scheduler/activities/ActivityLevel.java | 29 +
.../scheduler/activities/ActivityNode.java | 36 +-
.../scheduler/activities/AllocationActivity.java | 46 +-
.../scheduler/activities/AppAllocation.java | 42 +-
.../scheduler/activities/NodeAllocation.java | 25 +-
.../scheduler/capacity/AbstractCSQueue.java | 4 +-
.../scheduler/capacity/CSQueueMetrics.java | 39 +
.../scheduler/capacity/CSQueueUtils.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 45 +-
.../capacity/CapacitySchedulerConfiguration.java | 17 +-
.../capacity/CapacitySchedulerQueueManager.java | 6 +-
.../scheduler/capacity/LeafQueue.java | 30 +-
.../scheduler/capacity/ParentQueue.java | 48 +-
.../allocator/RegularContainerAllocator.java | 94 +-
.../conf/MutableCSConfigurationProvider.java | 8 +-
.../security/ReservationsACLsManager.java | 2 +-
.../webapp/JAXBContextResolver.java | 3 +-
.../resourcemanager/webapp/RMWebServices.java | 98 +-
.../resourcemanager/webapp/dao/ActivitiesInfo.java | 40 +-
.../webapp/dao/ActivityNodeInfo.java | 48 +-
.../webapp/dao/AppActivitiesInfo.java | 44 +-
.../webapp/dao/AppAllocationInfo.java | 24 +-
.../server/resourcemanager/webapp/dao/AppInfo.java | 4 +
.../webapp/dao/AppRequestAllocationInfo.java | 25 +-
.../webapp/dao/NodeAllocationInfo.java | 26 +-
.../ApplicationMasterServiceTestBase.java | 1 +
.../hadoop/yarn/server/resourcemanager/MockNM.java | 60 +-
.../server/resourcemanager/TestAppManager.java | 8 +-
.../resourcemanager/TestClientRMService.java | 178 +
.../server/resourcemanager/TestClusterMetrics.java | 8 +-
...stOpportunisticContainerAllocatorAMService.java | 208 +-
.../preprocessor/TestContextProcessor.java | 63 +
.../preprocessor/TestNodeLabelProcessor.java | 45 +
.../preprocessor/TestQueueProcessor.java | 43 +
.../preprocessor/TestTagAddProcessor.java | 47 +
.../recovery/TestZKRMStateStore.java | 2 +
.../rmapp/TestRMAppTransitions.java | 8 +-
.../activities/TestActivitiesManager.java | 70 +-
.../capacity/TestApplicationLimitsByPartition.java | 85 +
.../scheduler/capacity/TestCapacityScheduler.java | 37 +-
.../TestCapacitySchedulerSurgicalPreemption.java | 7 +
.../scheduler/capacity/TestLeafQueue.java | 2 +-
.../scheduler/capacity/TestQueueState.java | 4 +-
.../conf/TestMutableCSConfigurationProvider.java | 37 +
.../security/TestRMDelegationTokens.java | 33 +-
.../webapp/ActivitiesTestUtils.java | 139 +-
.../resourcemanager/webapp/TestRMWebServices.java | 82 +
.../TestRMWebServicesForCSWithPartitions.java | 97 +
.../TestRMWebServicesSchedulerActivities.java | 461 +-
...esSchedulerActivitiesWithMultiNodesEnabled.java | 190 +-
.../timeline/EntityGroupFSTimelineStore.java | 19 +-
.../timeline/TestEntityGroupFSTimelineStore.java | 17 +
.../storage/HBaseTimelineWriterImpl.java | 74 +-
.../collector/TimelineCollectorWebService.java | 4 +
.../reader/TimelineReaderManager.java | 2 +-
.../reader/TimelineReaderWebServices.java | 9 +-
.../storage/FileSystemTimelineReaderImpl.java | 4 +
.../reader/TestTimelineReaderWebServicesACL.java | 179 +
.../src/site/markdown/TimelineServiceV2.md | 35 +-
pom.ozone.xml | 1823 ++-
pom.xml | 62 +-
1339 files changed, 65633 insertions(+), 47820 deletions(-)
create mode 100644 .github/pull_request_template.md
delete mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml
mode change 100644 => 100755 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.1/CHANGELOG.3.2.1.md
create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.1/RELEASENOTES.3.2.1.md
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFs.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestDelegatingSSLSocketFactory.java
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/bin/hadoop-config.cmd (100%)
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/bin/hadoop-config.sh (100%)
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/bin/hadoop-daemons.sh (100%)
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/bin/hadoop-functions.sh (100%)
create mode 100755 hadoop-hdds/common/src/main/bin/workers.sh
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/conf/core-site.xml (100%)
create mode 100644 hadoop-hdds/common/src/main/conf/hadoop-env.cmd
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/conf/hadoop-env.sh (100%)
create mode 100644 hadoop-hdds/common/src/main/conf/hadoop-metrics2.properties
create mode 100644 hadoop-hdds/common/src/main/conf/hadoop-policy.xml
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTaskQueue.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTaskResult.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BatchOperation.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/EntryConsumer.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/HddsVersionInfo.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/LevelDBStoreIterator.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/MetaStoreIterator.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/MetadataStore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/MetadataStoreBuilder.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/RetriableTask.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreIterator.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/RocksDBStoreMBean.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/Scheduler.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/UniqueId.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/VersionInfo.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/BatchOperation.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/ByteArrayKeyValue.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Codec.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBCheckpoint.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBConfigFromFile.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBProfile.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DBUpdatesWrapper.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/IntegerCodec.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/LongCodec.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStoreIterator.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RDBTable.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/RocksDBCheckpoint.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/SequenceNumberNotFoundException.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/StringCodec.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/TableConfig.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/TableIterator.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/TypedTable.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheKey.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheResult.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheValue.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/EpochEntry.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCache.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCacheImpl.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/cache/package-info.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/package-info.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/package-info.java
create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/ProtocolMessageMetrics.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BatchOperation.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/HddsVersionInfo.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/LevelDBStoreIterator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetaStoreIterator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStore.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RetriableTask.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreIterator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/Scheduler.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/UniqueId.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/VersionInfo.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/BatchOperation.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/ByteArrayKeyValue.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBCheckpoint.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBConfigFromFile.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBProfile.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBUpdatesWrapper.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/IntegerCodec.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/LongCodec.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBBatchOperation.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBCheckpointManager.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStoreIterator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBTable.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RocksDBCheckpoint.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/SequenceNumberNotFoundException.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/StringCodec.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Table.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableConfig.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TableIterator.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/CacheKey.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/CacheResult.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/CacheValue.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/EpochEntry.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/TableCache.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/TableCacheImpl.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/package-info.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/package-info.java
delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/package-info.java
copy {hadoop-common-project/hadoop-common => hadoop-hdds/common}/src/main/proto/FSProtos.proto (100%)
create mode 100644 hadoop-hdds/common/src/main/proto/Security.proto
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestHddsIdFactory.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestMetadataStore.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestRetriableTask.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/TestRocksDBStoreMBean.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBConfigFromFile.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBStore.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/TestTypedRDBTableStore.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/cache/TestTableCacheImpl.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/cache/package-info.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/db/package-info.java
create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/utils/package-info.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestHddsIdFactory.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRetriableTask.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBConfigFromFile.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestDBStoreBuilder.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBTableStore.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestTypedRDBTableStore.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/cache/TestTableCacheImpl.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/cache/package-info.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/package-info.java
delete mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/package-info.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/RatisServerConfiguration.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/AsyncChecker.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScanner.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerDataScrubberMetrics.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScanner.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScrubberMetrics.java
delete mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScrubber.java
create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScrubberConfiguration.java
create mode 100644 hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberMetrics.java
create mode 100644 hadoop-hdds/dev-support/checkstyle/checkstyle-noframes-sorted.xsl
create mode 100644 hadoop-hdds/dev-support/checkstyle/checkstyle.xml
copy {hadoop-build-tools/src/main/resources => hadoop-hdds/dev-support}/checkstyle/suppressions.xml (100%)
delete mode 100644 hadoop-hdds/docs/content/beyond/Tools.md
create mode 100644 hadoop-hdds/docs/content/design/decommissioning.md
create mode 100644 hadoop-hdds/docs/content/gdpr/GDPR in Ozone.md
create mode 100644 hadoop-hdds/docs/content/gdpr/_index.md
delete mode 100644 hadoop-hdds/docs/content/tools/Freon.md
delete mode 100644 hadoop-hdds/docs/content/tools/Tools.md
create mode 100644 hadoop-hdds/docs/content/tools/_index.md
create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/LogStreamServlet.java
delete mode 100644 hadoop-hdds/framework/src/main/resources/webapps/static/dfs-dust.js
create mode 100644 hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestProfileServlet.java
create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementMetrics.java
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/ActivatePipelineSubcommand.java
create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/DeactivatePipelineSubcommand.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/CorruptMetaHeaderException.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/NameSpaceQuotaParam.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StorageSpaceQuotaParam.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StorageTypeParam.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.1.xml
mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCorruptMetadataFile.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRedudantBlocks.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUpload.java
create mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
delete mode 100644 hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfoDetails.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyLocation.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUpload.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadCompleteList.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java
create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/GDPRSymmetricKey.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfoDetails.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyLocation.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmMultipartUpload.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestGDPRSymmetricKey.java
create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSelector.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestQuota.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/TestVolumeStructs.java
delete mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/web/package-info.java
create mode 100755 hadoop-ozone/dev-support/checks/README.md
create mode 100755 hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
create mode 100644 hadoop-ozone/dist/src/main/assemblies/ozone-src.xml
create mode 100644 hadoop-ozone/dist/src/main/conf/ozone-shell-log4j.properties
create mode 100644 hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/NOTICE.txt
copy {licenses => hadoop-ozone/dist/src/main/license/bin/licenses}/LICENSE-angular-nvd3.txt (100%)
copy {licenses => hadoop-ozone/dist/src/main/license/bin/licenses}/LICENSE-angular.txt (100%)
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.google.code.findbugs-jsr305.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.google.re2j-re2j.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.jcraft-jsch.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.sun.jersey.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.sun.xml.bind.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-com.thoughtworks.paranamer-paranamer.txt
copy {licenses => hadoop-ozone/dist/src/main/license/bin/licenses}/LICENSE-d3.txt (100%)
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-dnsjava-dnsjava.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-jakarta.annotation-jakarta.annotation-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.activation-activation.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.annotation-javax.annotation-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.el-javax.el-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.interceptor-javax.interceptor-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.servlet-javax.servlet-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.servlet.jsp-jsp-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.ws.rs-javax.ws.rs-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.ws.rs-jsr311-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-javax.xml.bind-jaxb-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-jersey.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-jetty.txt
copy {licenses => hadoop-ozone/dist/src/main/license/bin/licenses}/LICENSE-jquery.txt (100%)
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-net.sf.jopt-simple-jopt-simple.txt
copy {licenses => hadoop-ozone/dist/src/main/license/bin/licenses}/LICENSE-nvd3.txt (100%)
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.codehaus.mojo-animal-sniffer-annotations.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.codehaus.woodstox-stax2-api.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.fusesource.leveldbjni-leveldbjni-all.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.glassfish.hk2.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-core.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.openjdk.jmh-jmh-generator-annprocess.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.ow2.asm-asm.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.slf4j.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-protobuf.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-ratis-thirdparty-misc.txt
create mode 100644 hadoop-ozone/dist/src/main/license/bin/licenses/NOTICE-ratis-thirtparty-misc.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/LICENSE.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/NOTICE.txt
create mode 100644 hadoop-ozone/dist/src/main/license/src/licenses/IMPORTANT.md
copy {licenses => hadoop-ozone/dist/src/main/license/src/licenses}/LICENSE-angular-nvd3.txt (100%)
copy {licenses => hadoop-ozone/dist/src/main/license/src/licenses}/LICENSE-angular.txt (100%)
copy {licenses => hadoop-ozone/dist/src/main/license/src/licenses}/LICENSE-d3.txt (100%)
copy {licenses => hadoop-ozone/dist/src/main/license/src/licenses}/LICENSE-jquery.txt (100%)
copy {licenses => hadoop-ozone/dist/src/main/license/src/licenses}/LICENSE-nvd3.txt (100%)
create mode 100644 hadoop-ozone/insight/dev-support/findbugsExcludeFile.xml
create mode 100644 hadoop-ozone/insight/pom.xml
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/BaseInsightPoint.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/BaseInsightSubCommand.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/Component.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/ConfigurationSubCommand.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/Insight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/InsightPoint.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/ListSubCommand.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LogSubcommand.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/LoggerSource.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/MetricDisplay.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/MetricGroupDisplay.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/MetricsSubCommand.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/RatisInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/package-info.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/om/KeyManagerInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/om/OmProtocolInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/om/package-info.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/package-info.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/scm/EventQueueInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/scm/NodeManagerInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/scm/ReplicaManagerInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/scm/ScmProtocolBlockLocationInsight.java
create mode 100644 hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/scm/package-info.java
create mode 100644 hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/LogSubcommandTest.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDataUtil.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneHelper.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithSlowFollower.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShellHA.java
create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMContainerPlacementPolicyMetrics.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneWebAccess.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestOzoneClient.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/package-info.java
delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/package-info.java
rename hadoop-ozone/integration-test/src/test/resources/{log4j2.properties => auditlog.properties} (100%)
delete mode 100644 hadoop-ozone/objectstore-service/pom.xml
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/OzoneRestUtils.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/OzoneHddsDatanodeService.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreRestHttpServer.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreURLDispatcher.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
delete mode 100644 hadoop-ozone/objectstore-service/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
delete mode 100644 hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
delete mode 100644 hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/package-info.java
create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/ObjectPrinter.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java
create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/web/ozShell/TestObjectPrinter.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerKeyPrefixCodec.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
delete mode 100644 hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/ozone-recon-web/package.json
delete mode 100644 hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/Breadcrumbs/Breadcrumbs.tsx
delete mode 100644 hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx
delete mode 100644 hadoop-ozone/ozone-recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractOMMetadataManagerTest.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconCodecs.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerDBProvider.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
delete mode 100644 hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
copy hadoop-ozone/{ozonefs/src/test => ozonefs-lib-current/src/main}/resources/META-INF/services/org.apache.hadoop.fs.FileSystem (100%)
create mode 100644 hadoop-ozone/ozonefs-lib-legacy/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
create mode 100644 hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsShell.java
create mode 100644 hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/pom.xml (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/codegen/JooqCodeGenerator.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/codegen/TableNamingStrategy.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/codegen/package-info.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/schema/ReconInternalSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/schema/ReconSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/schema/StatsSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/schema/UtilizationSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon-codegen => recon-codegen}/src/main/java/org/hadoop/ozone/recon/schema/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/dev-support/findbugsExcludeFile.xml (100%)
rename hadoop-ozone/{ozone-recon => recon}/pom.xml (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconGuiceServletContextListener.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconHttpServer.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconTaskBindingModule.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/UtilizationService.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerKeyPrefix.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainersResponse.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/IsoDateAdapter.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeysResponse.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/api/types/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/persistence/DataSourceConfiguration.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/persistence/DefaultDataSourceProvider.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/persistence/JooqPersistenceModule.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/persistence/TransactionalMethodInterceptor.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/persistence/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOMMetadataManager.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/recovery/package-info.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/spi/HddsDatanodeServiceProvider.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/spi/StorageContainerServiceProvider.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerKeyPrefixCodec.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/spi/package-info.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdateEvent.java (100%)
create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMUpdateEventBatch.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconDBUpdateTask.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskController.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconTaskControllerImpl.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/java/org/apache/hadoop/ozone/recon/tasks/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/WEB-INF/web.xml (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/.gitignore (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/LICENSE (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/NOTICE (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/README.md (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/config-overrides.js (100%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/package.json
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/public/favicon.ico (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/public/index.html (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/public/manifest.json (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/App.less (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/App.test.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/App.tsx (100%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/Breadcrumbs/Breadcrumbs.tsx
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.less (100%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/NavBar/NavBar.tsx
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/index.less (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/index.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/logo.png (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/makeRouteWithSubRoutes.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/react-app-env.d.ts (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/routes.types.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/serviceWorker.ts (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/views/ContainerBrowser/ContainerBrowser.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/views/Dashboard/Dashboard.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/src/views/NotFound/NotFound.tsx (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/main/resources/webapps/recon/ozone-recon-web/tsconfig.json (100%)
create mode 100644 hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/yarn.lock
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/AbstractOMMetadataManagerTest.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/GuiceInjectorUtilsForTestsImpl.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconCodecs.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/TestReconUtils.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/api/TestUtilizationService.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/api/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractSqlDatabaseTest.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestReconInternalSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestStatsSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/persistence/TestUtilizationSchemaDefinition.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/persistence/package-info.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/recovery/TestReconOmMetadataManagerImpl.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/recovery/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestOzoneManagerServiceProviderImpl.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerDBProvider.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/package-info.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/tasks/DummyReconDBTask.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestFileSizeCountTask.java
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestReconTaskControllerImpl.java (100%)
rename hadoop-ozone/{ozone-recon => recon}/src/test/java/org/apache/hadoop/ozone/recon/tasks/package-info.java (100%)
create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/types/GuiceInjectorUtilsForTests.java
rename hadoop-ozone/{ozone-recon => recon}/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker (100%)
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyPartResult.java
create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java
create mode 100644 hadoop-ozone/s3gateway/src/main/resources/webapps/static/s3g.js
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectEndpoint.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyValidator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/PathSchema.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SameKeyReader.java
create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InternalConstants.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ExecutingStoreOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/NetworkBinding.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/OperationCallbacks.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/InternalIterators.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsck.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardFsckViolationHandler.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/SSLSocketFactoryEx.java
create mode 100644 hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/DAGAMSimulator.java
create mode 100644 hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestDagAMSimulator.java
create mode 100644 hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSDagAMSimulator.java
create mode 100644 hadoop-tools/hadoop-sls/src/test/resources/sls_dag.json
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/ContextProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/NodeLabelProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/QueueProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/SubmissionContextPreProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/TagAddProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/package-info.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivityLevel.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/TestContextProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/TestNodeLabelProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/TestQueueProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/preprocessor/TestTagAddProcessor.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesACL.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDDS-2089: Add createPipeline CLI. (#1418)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7b5a5fe74d45b536dec0997cdb8034159bbcbf79
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Fri Sep 13 07:01:16 2019 +0800
HDDS-2089: Add createPipeline CLI. (#1418)
(cherry picked from commit 326b5acd4a63fe46821919322867f5daff30750c)
---
.../org/apache/hadoop/ozone/audit/SCMAction.java | 1 +
...inerLocationProtocolServerSideTranslatorPB.java | 10 ++-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 8 +--
.../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 2 +
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 71 ++++++++++++++++++++++
6 files changed, 87 insertions(+), 7 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index d03ad15..f8b1bbf 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -31,6 +31,7 @@ public enum SCMAction implements AuditAction {
GET_CONTAINER,
GET_CONTAINER_WITH_PIPELINE,
LIST_CONTAINER,
+ CREATE_PIPELINE,
LIST_PIPELINE,
CLOSE_PIPELINE,
ACTIVATE_PIPELINE,
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
index 99c9e8d..092aba3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -242,8 +242,14 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB
public PipelineResponseProto allocatePipeline(
RpcController controller, PipelineRequestProto request)
throws ServiceException {
- // TODO : Wiring this up requires one more patch.
- return null;
+ try (Scope scope = TracingUtil
+ .importAndCreateScope("createPipeline", request.getTraceID())) {
+ impl.createReplicationPipeline(request.getReplicationType(),
+ request.getReplicationFactor(), request.getNodePool());
+ return PipelineResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
}
@Override
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
index ab98dfa..54e2141 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java
@@ -48,7 +48,7 @@ public class SimplePipelineProvider implements PipelineProvider {
String e = String
.format("Cannot create pipeline of factor %d using %d nodes.",
factor.getNumber(), dns.size());
- throw new IOException(e);
+ throw new InsufficientDatanodesException(e);
}
Collections.shuffle(dns);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 7d9cb3e..7708bed 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -390,10 +390,10 @@ public class SCMClientProtocolServer implements
public Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
throws IOException {
- // TODO: will be addressed in future patch.
- // This is needed only for debugging purposes to make sure cluster is
- // working correctly.
- return null;
+ Pipeline result = scm.getPipelineManager().createPipeline(type, factor);
+ AUDIT.logWriteSuccess(
+ buildAuditMessageForSuccess(SCMAction.CREATE_PIPELINE, null));
+ return result;
}
@Override
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
index 1b95418..1246fae 100644
--- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.cli.container.ListSubcommand;
import org.apache.hadoop.hdds.scm.cli.pipeline.ActivatePipelineSubcommand;
import org.apache.hadoop.hdds.scm.cli.pipeline.ClosePipelineSubcommand;
import org.apache.hadoop.hdds.scm.cli.pipeline.DeactivatePipelineSubcommand;
+import org.apache.hadoop.hdds.scm.cli.pipeline.CreatePipelineSubcommand;
import org.apache.hadoop.hdds.scm.cli.pipeline.ListPipelinesSubcommand;
import org.apache.hadoop.hdds.scm.client.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
@@ -85,6 +86,7 @@ import picocli.CommandLine.Option;
DeleteSubcommand.class,
CreateSubcommand.class,
CloseSubcommand.class,
+ CreatePipelineSubcommand.class,
ListPipelinesSubcommand.class,
ActivatePipelineSubcommand.class,
DeactivatePipelineSubcommand.class,
diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
new file mode 100644
index 0000000..edeb786
--- /dev/null
+++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/pipeline/CreatePipelineSubcommand.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.pipeline;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.cli.SCMCLI;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Handler of createPipeline command.
+ */
+@CommandLine.Command(
+ name = "createPipeline",
+ description = "create pipeline",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class CreatePipelineSubcommand implements Callable<Void> {
+ @CommandLine.ParentCommand
+ private SCMCLI parent;
+
+ @CommandLine.Option(
+ names = {"-t", "--replicationType"},
+ description = "Replication type (STAND_ALONE, RATIS)",
+ defaultValue = "STAND_ALONE"
+ )
+ private HddsProtos.ReplicationType type
+ = HddsProtos.ReplicationType.STAND_ALONE;
+
+ @CommandLine.Option(
+ names = {"-f", "--replicationFactor"},
+ description = "Replication factor (ONE, THREE)",
+ defaultValue = "ONE"
+ )
+ private HddsProtos.ReplicationFactor factor
+ = HddsProtos.ReplicationFactor.ONE;
+
+ @Override
+ public Void call() throws Exception {
+ if (type == HddsProtos.ReplicationType.CHAINED) {
+ throw new IllegalArgumentException(type.name()
+ + " is not supported yet.");
+ }
+ try (ScmClient scmClient = parent.createScmClient()) {
+ scmClient.createReplicationPipeline(
+ type,
+ factor,
+ HddsProtos.NodePool.getDefaultInstance());
+ return null;
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: HDDS-1571. Create an interface for pipeline
placement policy to support network topologies. (#1395)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0001e1df5f8f55d1b41160dbc1a61ccc48421b06
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Tue Sep 10 20:15:51 2019 +0800
HDDS-1571. Create an interface for pipeline placement policy to support network topologies. (#1395)
(cherry picked from commit 753fc6703a39154ed6013e44dbae572391748906)
---
...erPlacementPolicy.java => PlacementPolicy.java} | 12 +++----
.../placement/algorithms/package-info.java | 21 -----------
.../common/src/main/resources/ozone-default.xml | 6 ++--
...onPolicy.java => SCMCommonPlacementPolicy.java} | 23 ++++++------
.../hdds/scm/container/ReplicationManager.java | 13 +++----
.../ContainerPlacementPolicyFactory.java | 18 +++++-----
.../algorithms/SCMContainerPlacementCapacity.java | 4 ++-
.../algorithms/SCMContainerPlacementRackAware.java | 8 +++--
.../algorithms/SCMContainerPlacementRandom.java | 6 ++--
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 42 +++++++++++++---------
.../hdds/scm/pipeline/RatisPipelineProvider.java | 14 ++++----
.../hdds/scm/server/StorageContainerManager.java | 4 +--
.../hdds/scm/container/TestReplicationManager.java | 7 ++--
.../algorithms/TestContainerPlacementFactory.java | 7 ++--
.../hdds/scm/node/TestContainerPlacement.java | 5 ++-
.../hdds/scm/safemode/TestSafeModeHandler.java | 5 ++-
.../hadoop/ozone/TestContainerOperations.java | 4 +--
.../TestContainerStateMachineIdempotency.java | 5 ++-
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +--
.../hadoop/ozone/scm/TestContainerSmallFile.java | 4 +--
.../scm/TestGetCommittedBlockLengthAndPutKey.java | 5 ++-
21 files changed, 105 insertions(+), 112 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
similarity index 80%
rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
index 52ce796..f6a0e8b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/PlacementPolicy.java
@@ -15,7 +15,7 @@
* the License.
*/
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+package org.apache.hadoop.hdds.scm;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -23,14 +23,14 @@ import java.io.IOException;
import java.util.List;
/**
- * A ContainerPlacementPolicy support choosing datanodes to build replication
- * pipeline with specified constraints.
+ * A PlacementPolicy support choosing datanodes to build
+ * pipelines or containers with specified constraints.
*/
-public interface ContainerPlacementPolicy {
+public interface PlacementPolicy {
/**
- * Given the replication factor and size required, return set of datanodes
- * that satisfy the nodes and size requirement.
+ * Given an initial set of datanodes and the size required,
+ * return set of datanodes that satisfy the nodes and size requirement.
*
* @param excludedNodes - list of nodes to be excluded.
* @param favoredNodes - list of nodes preferred.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java
deleted file mode 100644
index dac4752..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-/**
- Contains container placement policy interface definition.
- **/
\ No newline at end of file
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c1e97a9..a038047 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -823,9 +823,11 @@
</value>
<tag>OZONE, MANAGEMENT</tag>
<description>
- The full name of class which implements org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy.
+ The full name of class which implements
+ org.apache.hadoop.hdds.scm.PlacementPolicy.
The class decides which datanode will be used to host the container replica. If not set,
- org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default value.
+ org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom will be used as default
+ value.
</description>
</property>
<property>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
similarity index 90%
rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index 77cdd83..25457f7 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMCommonPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -15,7 +15,7 @@
* the License.
*/
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
+package org.apache.hadoop.hdds.scm;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
@@ -33,25 +33,25 @@ import java.util.Random;
import java.util.stream.Collectors;
/**
- * SCM CommonPolicy implements a set of invariants which are common
- * for all container placement policies, acts as the repository of helper
+ * This policy implements a set of invariants which are common
+ * for all basic placement policies, acts as the repository of helper
* functions which are common to placement policies.
*/
-public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
+public abstract class SCMCommonPlacementPolicy implements PlacementPolicy {
@VisibleForTesting
static final Logger LOG =
- LoggerFactory.getLogger(SCMCommonPolicy.class);
+ LoggerFactory.getLogger(SCMCommonPlacementPolicy.class);
private final NodeManager nodeManager;
private final Random rand;
private final Configuration conf;
/**
- * Constructs SCM Common Policy Class.
+ * Constructor.
*
* @param nodeManager NodeManager
* @param conf Configuration class.
*/
- public SCMCommonPolicy(NodeManager nodeManager, Configuration conf) {
+ public SCMCommonPlacementPolicy(NodeManager nodeManager, Configuration conf) {
this.nodeManager = nodeManager;
this.rand = new Random();
this.conf = conf;
@@ -85,7 +85,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
}
/**
- * Given the replication factor and size required, return set of datanodes
+ * Given size required, return set of datanodes
* that satisfy the nodes and size requirement.
* <p>
* Here are some invariants of container placement.
@@ -149,7 +149,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
* @param datanodeDetails DatanodeDetails
* @return true if we have enough space.
*/
- boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
+ public boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
long sizeRequired) {
SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeDetails);
return (nodeMetric != null) && (nodeMetric.get() != null)
@@ -164,7 +164,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
* @param nodesRequired - Nodes Required
* @param healthyNodes - List of Nodes in the result set.
* @return List of Datanodes that can be used for placement.
- * @throws SCMException
+ * @throws SCMException SCMException
*/
public List<DatanodeDetails> getResultSet(
int nodesRequired, List<DatanodeDetails> healthyNodes)
@@ -190,8 +190,7 @@ public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
/**
* Choose a datanode according to the policy, this function is implemented
- * by the actual policy class. For example, PlacementCapacity or
- * PlacementRandom.
+ * by the actual policy class.
*
* @param healthyNodes - Set of healthy nodes we can choose from.
* @return DatanodeDetails
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 5540d73..2ba8856 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -38,8 +38,9 @@ import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -85,7 +86,7 @@ public class ReplicationManager implements MetricsSource {
* PlacementPolicy which is used to identify where a container
* should be replicated.
*/
- private final ContainerPlacementPolicy containerPlacement;
+ private final PlacementPolicy containerPlacement;
/**
* EventPublisher to fire Replicate and Delete container events.
@@ -131,12 +132,12 @@ public class ReplicationManager implements MetricsSource {
*
* @param conf OzoneConfiguration
* @param containerManager ContainerManager
- * @param containerPlacement ContainerPlacementPolicy
+ * @param containerPlacement PlacementPolicy
* @param eventPublisher EventPublisher
*/
public ReplicationManager(final ReplicationManagerConfiguration conf,
final ContainerManager containerManager,
- final ContainerPlacementPolicy containerPlacement,
+ final PlacementPolicy containerPlacement,
final EventPublisher eventPublisher,
final LockManager<ContainerID> lockManager) {
this.containerManager = containerManager;
@@ -474,7 +475,7 @@ public class ReplicationManager implements MetricsSource {
/**
* If the given container is under replicated, identify a new set of
- * datanode(s) to replicate the container using ContainerPlacementPolicy
+ * datanode(s) to replicate the container using PlacementPolicy
* and send replicate container command to the identified datanode(s).
*
* @param container ContainerInfo
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
index 18ec2c3..adaeb87 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicyFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -34,22 +35,23 @@ public final class ContainerPlacementPolicyFactory {
private static final Logger LOG =
LoggerFactory.getLogger(ContainerPlacementPolicyFactory.class);
- private static final Class<? extends ContainerPlacementPolicy>
+ private static final Class<? extends PlacementPolicy>
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT =
SCMContainerPlacementRandom.class;
private ContainerPlacementPolicyFactory() {
}
- public static ContainerPlacementPolicy getPolicy(Configuration conf,
- final NodeManager nodeManager, NetworkTopology clusterMap,
- final boolean fallback, SCMContainerPlacementMetrics metrics)
- throws SCMException{
- final Class<? extends ContainerPlacementPolicy> placementClass = conf
+
+ public static PlacementPolicy getPolicy(Configuration conf,
+ final NodeManager nodeManager, NetworkTopology clusterMap,
+ final boolean fallback, SCMContainerPlacementMetrics metrics)
+ throws SCMException{
+ final Class<? extends PlacementPolicy> placementClass = conf
.getClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
OZONE_SCM_CONTAINER_PLACEMENT_IMPL_DEFAULT,
- ContainerPlacementPolicy.class);
- Constructor<? extends ContainerPlacementPolicy> constructor;
+ PlacementPolicy.class);
+ Constructor<? extends PlacementPolicy> constructor;
try {
constructor = placementClass.getDeclaredConstructor(NodeManager.class,
Configuration.class, NetworkTopology.class, boolean.class,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
index 85d281c..1909344 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
@@ -21,6 +21,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -65,7 +66,8 @@ import org.slf4j.LoggerFactory;
* little or no work and the cluster will achieve a balanced distribution
* over time.
*/
-public final class SCMContainerPlacementCapacity extends SCMCommonPolicy {
+public final class SCMContainerPlacementCapacity
+ extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementCapacity.class);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
index 8eccf45..8933fe9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackAware.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetConstants;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -45,7 +46,8 @@ import java.util.List;
* recommend to use this if the network topology has more layers.
* <p>
*/
-public final class SCMContainerPlacementRackAware extends SCMCommonPolicy {
+public final class SCMContainerPlacementRackAware
+ extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementRackAware.class);
@@ -271,8 +273,8 @@ public final class SCMContainerPlacementRackAware extends SCMCommonPolicy {
throw new SCMException("No satisfied datanode to meet the" +
" excludedNodes and affinityNode constrains.", null);
}
- if (hasEnoughSpace((DatanodeDetails)node, sizeRequired)) {
- LOG.debug("Datanode {} is chosen for container. Required size is {}",
+ if (super.hasEnoughSpace((DatanodeDetails)node, sizeRequired)) {
+ LOG.debug("Datanode {} is chosen. Required size is {}",
node.toString(), sizeRequired);
metrics.incrDatanodeChooseSuccessCount();
if (isFallbacked) {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
index 6b1a5c8..ce5d10d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdds.scm.container.placement.algorithms;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@@ -37,8 +39,8 @@ import java.util.List;
* Balancer will need to support containers as a feature before this class
* can be practically used.
*/
-public final class SCMContainerPlacementRandom extends SCMCommonPolicy
- implements ContainerPlacementPolicy {
+public final class SCMContainerPlacementRandom extends SCMCommonPlacementPolicy
+ implements PlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(SCMContainerPlacementRandom.class);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
index cb9954d..1983ed6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMCommonPolicy;
+import org.apache.hadoop.hdds.scm.SCMCommonPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
@@ -47,7 +47,7 @@ import java.util.stream.Collectors;
* 3. Choose an anchor node among the viable nodes.
* 4. Choose other nodes around the anchor node based on network topology
*/
-public final class PipelinePlacementPolicy extends SCMCommonPolicy {
+public final class PipelinePlacementPolicy extends SCMCommonPlacementPolicy {
@VisibleForTesting
static final Logger LOG =
LoggerFactory.getLogger(PipelinePlacementPolicy.class);
@@ -150,33 +150,41 @@ public final class PipelinePlacementPolicy extends SCMCommonPolicy {
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
int nodesRequired, final long sizeRequired) throws SCMException {
- // get a list of viable nodes based on criteria
+ // Get a list of viable nodes based on criteria
+ // and make sure excludedNodes are excluded from list.
List<DatanodeDetails> healthyNodes =
filterViableNodes(excludedNodes, nodesRequired);
-
- List<DatanodeDetails> results = new ArrayList<>();
-
+
// Randomly picks nodes when all nodes are equal.
// This happens when network topology is absent or
// all nodes are on the same rack.
if (checkAllNodesAreEqual(nodeManager.getClusterNetworkTopologyMap())) {
LOG.info("All nodes are considered equal. Now randomly pick nodes. " +
"Required nodes: {}", nodesRequired);
- results = super.getResultSet(nodesRequired, healthyNodes);
- if (results.size() < nodesRequired) {
- LOG.error("Unable to find the required number of healthy nodes that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
- throw new SCMException("Unable to find required number of nodes.",
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- return results;
+ return super.getResultSet(nodesRequired, healthyNodes);
+ } else {
+ // Since topology and rack awareness are available, picks nodes
+ // based on them.
+ return this.getResultSet(nodesRequired, healthyNodes);
}
+ }
+ /**
+ * Get result set based on the pipeline placement algorithm which considers
+ * network topology and rack awareness.
+ * @param nodesRequired - Nodes Required
+ * @param healthyNodes - List of Nodes in the result set.
+ * @return a list of datanodes
+ * @throws SCMException SCMException
+ */
+ @Override
+ public List<DatanodeDetails> getResultSet(
+ int nodesRequired, List<DatanodeDetails> healthyNodes)
+ throws SCMException {
+ List <DatanodeDetails> results = new ArrayList<>(nodesRequired);
// Since nodes are widely distributed, the results should be selected
// base on distance in topology, rack awareness and load balancing.
List<DatanodeDetails> exclude = new ArrayList<>();
- exclude.addAll(excludedNodes);
// First choose an anchor nodes randomly
DatanodeDetails anchor = chooseNode(healthyNodes);
if (anchor == null) {
@@ -193,7 +201,7 @@ public final class PipelinePlacementPolicy extends SCMCommonPolicy {
// Choose the second node on different racks from anchor.
DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
- healthyNodes, excludedNodes,
+ healthyNodes, exclude,
nodeManager.getClusterNetworkTopologyMap(), anchor);
if (nodeOnDifferentRack == null) {
LOG.error("Unable to find nodes on different racks that " +
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
index a5e3d37..9409728d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
@@ -103,16 +103,16 @@ public class RatisPipelineProvider implements PipelineProvider {
* @return SCM container placement policy implementation instance.
*/
@SuppressWarnings("unchecked")
- // TODO: should we rename ContainerPlacementPolicy to PipelinePlacementPolicy?
- private static ContainerPlacementPolicy createContainerPlacementPolicy(
+ // TODO: should we rename PlacementPolicy to PipelinePlacementPolicy?
+ private static PlacementPolicy createContainerPlacementPolicy(
final NodeManager nodeManager, final Configuration conf) {
- Class<? extends ContainerPlacementPolicy> implClass =
- (Class<? extends ContainerPlacementPolicy>) conf.getClass(
+ Class<? extends PlacementPolicy> implClass =
+ (Class<? extends PlacementPolicy>) conf.getClass(
ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
SCMContainerPlacementRandom.class);
try {
- Constructor<? extends ContainerPlacementPolicy> ctor =
+ Constructor<? extends PlacementPolicy> ctor =
implClass.getDeclaredConstructor(NodeManager.class,
Configuration.class);
return ctor.newInstance(nodeManager, conf);
@@ -125,7 +125,7 @@ public class RatisPipelineProvider implements PipelineProvider {
// LOG.error("Unhandled exception occurred, Placement policy will not " +
// "be functional.");
throw new IllegalArgumentException("Unable to load " +
- "ContainerPlacementPolicy", e);
+ "PlacementPolicy", e);
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 4ecab37..c25b3a0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.IncrementalContainerReportHandler;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
@@ -394,7 +394,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
SCMContainerPlacementMetrics placementMetrics =
SCMContainerPlacementMetrics.create();
- ContainerPlacementPolicy containerPlacementPolicy =
+ PlacementPolicy containerPlacementPolicy =
ContainerPlacementPolicyFactory.getPolicy(conf, scmNodeManager,
clusterMap, true, placementMetrics);
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
index 1631447..63735f7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestReplicationManager.java
@@ -27,8 +27,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -66,7 +65,7 @@ public class TestReplicationManager {
private ReplicationManager replicationManager;
private ContainerStateManager containerStateManager;
- private ContainerPlacementPolicy containerPlacementPolicy;
+ private PlacementPolicy containerPlacementPolicy;
private EventQueue eventQueue;
private DatanodeCommandHandler datanodeCommandHandler;
@@ -93,7 +92,7 @@ public class TestReplicationManager {
.thenAnswer(invocation -> containerStateManager
.getContainerReplicas((ContainerID)invocation.getArguments()[0]));
- containerPlacementPolicy = Mockito.mock(ContainerPlacementPolicy.class);
+ containerPlacementPolicy = Mockito.mock(PlacementPolicy.class);
Mockito.when(containerPlacementPolicy.chooseDatanodes(
Mockito.anyListOf(DatanodeDetails.class),
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
index 18c4a64..81f8c10 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestContainerPlacementFactory.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -99,7 +100,7 @@ public class TestContainerPlacementFactory {
when(nodeManager.getNodeStat(datanodes.get(4)))
.thenReturn(new SCMNodeMetric(storageCapacity, 70L, 30L));
- ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory
+ PlacementPolicy policy = ContainerPlacementPolicyFactory
.getPolicy(conf, nodeManager, cluster, true,
SCMContainerPlacementMetrics.create());
@@ -117,7 +118,7 @@ public class TestContainerPlacementFactory {
@Test
public void testDefaultPolicy() throws IOException {
- ContainerPlacementPolicy policy = ContainerPlacementPolicyFactory
+ PlacementPolicy policy = ContainerPlacementPolicyFactory
.getPolicy(conf, null, null, true, null);
Assert.assertSame(SCMContainerPlacementRandom.class, policy.getClass());
}
@@ -125,7 +126,7 @@ public class TestContainerPlacementFactory {
/**
* A dummy container placement implementation for test.
*/
- public static class DummyImpl implements ContainerPlacementPolicy {
+ public static class DummyImpl implements PlacementPolicy {
@Override
public List<DatanodeDetails> chooseDatanodes(
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
index 26ffd8d..2206e4d 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
@@ -27,8 +27,7 @@ import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -135,7 +134,7 @@ public class TestContainerPlacement {
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
testDir.getAbsolutePath());
conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
SCMNodeManager nodeManager = createNodeManager(conf);
SCMContainerManager containerManager =
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
index 5572e9a..4ad3456 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeHandler.java
@@ -25,8 +25,7 @@ import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ReplicationManager;
import org.apache.hadoop.hdds.scm.container.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
@@ -70,7 +69,7 @@ public class TestSafeModeHandler {
.thenReturn(new HashSet<>());
replicationManager = new ReplicationManager(
new ReplicationManagerConfiguration(),
- containerManager, Mockito.mock(ContainerPlacementPolicy.class),
+ containerManager, Mockito.mock(PlacementPolicy.class),
eventQueue, new LockManager(configuration));
scmPipelineManager = Mockito.mock(SCMPipelineManager.class);
blockManager = Mockito.mock(BlockManagerImpl.class);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
index 129cf04..6e307a0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
@@ -53,7 +53,7 @@ public class TestContainerOperations {
containerSizeGB * OzoneConsts.GB);
ozoneConf = new OzoneConfiguration();
ozoneConf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster = MiniOzoneCluster.newBuilder(ozoneConf).setNumDatanodes(1).build();
StorageContainerLocationProtocolClientSideTranslatorPB client =
cluster.getStorageContainerLocationClient();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
index 2d2d028..7f2d629 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java
@@ -29,8 +29,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.
- ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.
SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.
@@ -60,7 +59,7 @@ public class TestContainerStateMachineIdempotency {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster =
MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
index 863a2b3..3019222 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.HddsDatanodeService;
@@ -83,7 +83,7 @@ public class TestDataScrubber {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.set(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, "1s");
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster = MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(1)
.build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
index 4c62c70..ca7a615 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocolPB
@@ -59,7 +59,7 @@ public class TestContainerSmallFile {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster = MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(1)
.build();
cluster.waitForClusterToBeReady();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
index 8e4645f..feb74c2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java
@@ -31,8 +31,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.
- ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.
SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.protocolPB.
@@ -62,7 +61,7 @@ public class TestGetCommittedBlockLengthAndPutKey {
public static void init() throws Exception {
ozoneConfig = new OzoneConfiguration();
ozoneConfig.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ SCMContainerPlacementCapacity.class, PlacementPolicy.class);
cluster =
MiniOzoneCluster.newBuilder(ozoneConfig).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HDDS-1577. Add default pipeline placement policy
implementation. (#1366)
Posted by sa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-1564
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7f76469d50ebd7555a353514e3826198d14414a2
Author: Li Cheng <bl...@gmail.com>
AuthorDate: Thu Sep 5 11:51:40 2019 +0800
HDDS-1577. Add default pipeline placement policy implementation. (#1366)
(cherry picked from commit b640a5f6d53830aee4b9c2a7d17bf57c987962cd)
---
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
.../common/src/main/resources/ozone-default.xml | 7 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 14 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 9 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 19 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 4 +-
.../hdds/scm/node/states/Node2PipelineMap.java | 12 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 338 +++++++++++++++++++++
.../hadoop/hdds/scm/container/MockNodeManager.java | 36 ++-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 197 ++++++++++++
.../testutils/ReplicationNodeManagerMock.java | 16 +
11 files changed, 653 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index f00ecb2..ad7073e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -313,6 +313,11 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
"ozone.scm.pipeline.owner.container.count";
public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
+ // Pipeline placement policy:
+ // the max number of pipelines can a single datanode be engaged in.
+ public static final String OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT =
+ "ozone.scm.datanode.max.pipeline.engagement";
+ public static final int OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT = 5;
public static final String
OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 9e4c5ea..c1e97a9 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -836,6 +836,13 @@
</description>
</property>
<property>
+ <name>ozone.scm.datanode.max.pipeline.engagement</name>
+ <value>5</value>
+ <tag>OZONE, SCM, PIPELINE</tag>
+ <description>Max number of pipelines per datanode can be engaged in.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.container.size</name>
<value>5GB</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index d8890fb..d638ee9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -118,6 +119,13 @@ public interface NodeManager extends StorageContainerNodeProtocol,
Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails);
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ int getPipelinesCount(DatanodeDetails datanodeDetails);
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -199,4 +207,10 @@ public interface NodeManager extends StorageContainerNodeProtocol,
* @return the given datanode, or null if not found
*/
DatanodeDetails getNodeByAddress(String address);
+
+ /**
+ * Get cluster map as in network topology for this node manager.
+ * @return cluster map
+ */
+ NetworkTopology getClusterNetworkTopologyMap();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
index 954cb0e..9d2a9f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java
@@ -284,6 +284,15 @@ public class NodeStateManager implements Runnable, Closeable {
}
/**
+ * Get the count of pipelines associated to single datanode.
+ * @param datanodeDetails single datanode
+ * @return number of pipelines associated with it
+ */
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return node2PipelineMap.getPipelinesCount(datanodeDetails.getUuid());
+ }
+
+ /**
* Get information about the node.
*
* @param datanodeDetails DatanodeDetails
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index d3df858..d7c6da9 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -472,7 +472,6 @@ public class SCMNodeManager implements NodeManager {
return nodeInfo;
}
-
/**
* Get set of pipelines a datanode is part of.
* @param datanodeDetails - datanodeID
@@ -483,6 +482,15 @@ public class SCMNodeManager implements NodeManager {
return nodeStateManager.getPipelineByDnID(datanodeDetails.getUuid());
}
+ /**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return nodeStateManager.getPipelinesCount(datanodeDetails);
+ }
/**
* Add pipeline information in the NodeManager.
@@ -609,6 +617,15 @@ public class SCMNodeManager implements NodeManager {
return null;
}
+ /**
+ * Get cluster map as in network topology for this node manager.
+ * @return cluster map
+ */
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return clusterMap;
+ }
+
private String nodeResolve(String hostname) {
List<String> hosts = new ArrayList<>(1);
hosts.add(hostname);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
index 37525b0..57a377d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
@@ -67,6 +67,7 @@ public class Node2ObjectsMap<T> {
* @param datanodeID -- Datanode UUID
* @param containerIDs - List of ContainerIDs.
*/
+ @VisibleForTesting
public void insertNewDatanode(UUID datanodeID, Set<T> containerIDs)
throws SCMException {
Preconditions.checkNotNull(containerIDs);
@@ -83,7 +84,8 @@ public class Node2ObjectsMap<T> {
*
* @param datanodeID - Datanode ID.
*/
- void removeDatanode(UUID datanodeID) {
+ @VisibleForTesting
+ public void removeDatanode(UUID datanodeID) {
Preconditions.checkNotNull(datanodeID);
dn2ObjectMap.computeIfPresent(datanodeID, (k, v) -> null);
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
index f8633f9..714188d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java
@@ -42,7 +42,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
}
/**
- * Returns null if there no pipelines associated with this datanode ID.
+ * Returns null if there are no pipelines associated with this datanode ID.
*
* @param datanode - UUID
* @return Set of pipelines or Null.
@@ -52,6 +52,16 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
}
/**
+ * Return 0 if there are no pipelines associated with this datanode ID.
+ * @param datanode - UUID
+ * @return Number of pipelines or 0.
+ */
+ public int getPipelinesCount(UUID datanode) {
+ Set<PipelineID> pipelines = getObjects(datanode);
+ return pipelines == null ? 0 : pipelines.size();
+ }
+
+ /**
* Adds a pipeline entry to a given dataNode in the map.
*
* @param pipeline Pipeline to be added
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
new file mode 100644
index 0000000..cb9954d
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicy.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMCommonPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.Node;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Pipeline placement policy that choose datanodes based on load balancing
+ * and network topology to supply pipeline creation.
+ * <p>
+ * 1. get a list of healthy nodes
+ * 2. filter out nodes that are not too heavily engaged in other pipelines
+ * 3. Choose an anchor node among the viable nodes.
+ * 4. Choose other nodes around the anchor node based on network topology
+ */
+public final class PipelinePlacementPolicy extends SCMCommonPolicy {
+ @VisibleForTesting
+ static final Logger LOG =
+ LoggerFactory.getLogger(PipelinePlacementPolicy.class);
+ private final NodeManager nodeManager;
+ private final Configuration conf;
+ private final int heavyNodeCriteria;
+
+ /**
+ * Constructs a pipeline placement with considering network topology,
+ * load balancing and rack awareness.
+ *
+ * @param nodeManager Node Manager
+ * @param conf Configuration
+ */
+ public PipelinePlacementPolicy(
+ final NodeManager nodeManager, final Configuration conf) {
+ super(nodeManager, conf);
+ this.nodeManager = nodeManager;
+ this.conf = conf;
+ heavyNodeCriteria = conf.getInt(
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT,
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT);
+ }
+
+ /**
+ * Returns true if this node meets the criteria.
+ *
+ * @param datanodeDetails DatanodeDetails
+ * @return true if we have enough space.
+ */
+ @VisibleForTesting
+ boolean meetCriteria(DatanodeDetails datanodeDetails, long heavyNodeLimit) {
+ return (nodeManager.getPipelinesCount(datanodeDetails) <= heavyNodeLimit);
+ }
+
+ /**
+ * Filter out viable nodes based on
+ * 1. nodes that are healthy
+ * 2. nodes that are not too heavily engaged in other pipelines
+ *
+ * @param excludedNodes - excluded nodes
+ * @param nodesRequired - number of datanodes required.
+ * @return a list of viable nodes
+ * @throws SCMException when viable nodes are not enough in numbers
+ */
+ List<DatanodeDetails> filterViableNodes(
+ List<DatanodeDetails> excludedNodes, int nodesRequired)
+ throws SCMException {
+ // get nodes in HEALTHY state
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ if (excludedNodes != null) {
+ healthyNodes.removeAll(excludedNodes);
+ }
+ String msg;
+ if (healthyNodes.size() == 0) {
+ msg = "No healthy node found to allocate pipeline.";
+ LOG.error(msg);
+ throw new SCMException(msg, SCMException.ResultCodes
+ .FAILED_TO_FIND_HEALTHY_NODES);
+ }
+
+ if (healthyNodes.size() < nodesRequired) {
+ msg = String.format("Not enough healthy nodes to allocate pipeline. %d "
+ + " datanodes required. Found %d",
+ nodesRequired, healthyNodes.size());
+ LOG.error(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ // filter nodes that meet the size and pipeline engagement criteria.
+ // Pipeline placement doesn't take node space left into account.
+ List<DatanodeDetails> healthyList = healthyNodes.stream().filter(d ->
+ meetCriteria(d, heavyNodeCriteria)).collect(Collectors.toList());
+
+ if (healthyList.size() < nodesRequired) {
+ msg = String.format("Unable to find enough nodes that meet " +
+ "the criteria that cannot engage in more than %d pipelines." +
+ " Nodes required: %d Found: %d",
+ heavyNodeCriteria, nodesRequired, healthyList.size());
+ LOG.error(msg);
+ throw new SCMException(msg,
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return healthyList;
+ }
+
+ /**
+ * Pipeline placement choose datanodes to join the pipeline.
+ *
+ * @param excludedNodes - excluded nodes
+ * @param favoredNodes - list of nodes preferred.
+ * @param nodesRequired - number of datanodes required.
+ * @param sizeRequired - size required for the container or block.
+ * @return a list of chosen datanodeDetails
+ * @throws SCMException when chosen nodes are not enough in numbers
+ */
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes,
+ int nodesRequired, final long sizeRequired) throws SCMException {
+ // get a list of viable nodes based on criteria
+ List<DatanodeDetails> healthyNodes =
+ filterViableNodes(excludedNodes, nodesRequired);
+
+ List<DatanodeDetails> results = new ArrayList<>();
+
+ // Randomly picks nodes when all nodes are equal.
+ // This happens when network topology is absent or
+ // all nodes are on the same rack.
+ if (checkAllNodesAreEqual(nodeManager.getClusterNetworkTopologyMap())) {
+ LOG.info("All nodes are considered equal. Now randomly pick nodes. " +
+ "Required nodes: {}", nodesRequired);
+ results = super.getResultSet(nodesRequired, healthyNodes);
+ if (results.size() < nodesRequired) {
+ LOG.error("Unable to find the required number of healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return results;
+ }
+
+ // Since nodes are widely distributed, the results should be selected
+ // base on distance in topology, rack awareness and load balancing.
+ List<DatanodeDetails> exclude = new ArrayList<>();
+ exclude.addAll(excludedNodes);
+ // First choose an anchor nodes randomly
+ DatanodeDetails anchor = chooseNode(healthyNodes);
+ if (anchor == null) {
+ LOG.error("Unable to find the first healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ results.add(anchor);
+ exclude.add(anchor);
+ nodesRequired--;
+
+ // Choose the second node on different racks from anchor.
+ DatanodeDetails nodeOnDifferentRack = chooseNodeBasedOnRackAwareness(
+ healthyNodes, excludedNodes,
+ nodeManager.getClusterNetworkTopologyMap(), anchor);
+ if (nodeOnDifferentRack == null) {
+ LOG.error("Unable to find nodes on different racks that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ results.add(nodeOnDifferentRack);
+ exclude.add(nodeOnDifferentRack);
+ nodesRequired--;
+
+ // Then choose nodes close to anchor based on network topology
+ for (int x = 0; x < nodesRequired; x++) {
+ // invoke the choose function defined in the derived classes.
+ DatanodeDetails pick = chooseNodeFromNetworkTopology(
+ nodeManager.getClusterNetworkTopologyMap(), anchor, exclude);
+ if (pick != null) {
+ results.add(pick);
+ // exclude the picked node for next time
+ exclude.add(pick);
+ }
+ }
+
+ if (results.size() < nodesRequired) {
+ LOG.error("Unable to find the required number of healthy nodes that " +
+ "meet the criteria. Required nodes: {}, Found nodes: {}",
+ nodesRequired, results.size());
+ throw new SCMException("Unable to find required number of nodes.",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+ return results;
+ }
+
+ /**
+ * Find a node from the healthy list and return it after removing it from the
+ * list that we are operating on.
+ *
+ * @param healthyNodes - Set of healthy nodes we can choose from.
+ * @return chosen datanodDetails
+ */
+ @Override
+ public DatanodeDetails chooseNode(
+ List<DatanodeDetails> healthyNodes) {
+ int firstNodeNdx = getRand().nextInt(healthyNodes.size());
+ int secondNodeNdx = getRand().nextInt(healthyNodes.size());
+
+ DatanodeDetails datanodeDetails;
+ // There is a possibility that both numbers will be same.
+ // if that is so, we just return the node.
+ if (firstNodeNdx == secondNodeNdx) {
+ datanodeDetails = healthyNodes.get(firstNodeNdx);
+ } else {
+ DatanodeDetails firstNodeDetails = healthyNodes.get(firstNodeNdx);
+ DatanodeDetails secondNodeDetails = healthyNodes.get(secondNodeNdx);
+ SCMNodeMetric firstNodeMetric =
+ nodeManager.getNodeStat(firstNodeDetails);
+ SCMNodeMetric secondNodeMetric =
+ nodeManager.getNodeStat(secondNodeDetails);
+ datanodeDetails = firstNodeMetric.isGreater(secondNodeMetric.get())
+ ? firstNodeDetails : secondNodeDetails;
+ }
+ // the pick is decided and it should be removed from candidates.
+ healthyNodes.remove(datanodeDetails);
+ return datanodeDetails;
+ }
+
+ /**
+ * Choose node on different racks as anchor is on based on rack awareness.
+ * If a node on different racks cannot be found, then return a random node.
+ * @param healthyNodes healthy nodes
+ * @param excludedNodes excluded nodes
+ * @param networkTopology network topology
+ * @param anchor anchor node
+ * @return a node on different rack
+ */
+ @VisibleForTesting
+ protected DatanodeDetails chooseNodeBasedOnRackAwareness(
+ List<DatanodeDetails> healthyNodes, List<DatanodeDetails> excludedNodes,
+ NetworkTopology networkTopology, DatanodeDetails anchor) {
+ Preconditions.checkArgument(networkTopology != null);
+ if (checkAllNodesAreEqual(networkTopology)) {
+ return null;
+ }
+
+ for (DatanodeDetails node : healthyNodes) {
+ if (excludedNodes.contains(node)
+ || networkTopology.isSameParent(anchor, node)) {
+ continue;
+ } else {
+ // the pick is decided and it should be removed from candidates.
+ healthyNodes.remove(node);
+ return node;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Check if all nodes are equal in topology.
+ * They are equal when network topology is absent or there are on
+ * the same rack.
+ * @param topology network topology
+ * @return true when all nodes are equal
+ */
+ private boolean checkAllNodesAreEqual(NetworkTopology topology) {
+ if (topology == null) {
+ return true;
+ }
+ return (topology.getNumOfNodes(topology.getMaxLevel() - 1) == 1);
+ }
+
+ /**
+ * Choose node based on network topology.
+ * @param networkTopology network topology
+ * @param anchor anchor datanode to start with
+ * @param excludedNodes excluded datanodes
+ * @return chosen datanode
+ */
+ @VisibleForTesting
+ protected DatanodeDetails chooseNodeFromNetworkTopology(
+ NetworkTopology networkTopology, DatanodeDetails anchor,
+ List<DatanodeDetails> excludedNodes) {
+ Preconditions.checkArgument(networkTopology != null);
+
+ Collection<Node> excluded = new ArrayList<>();
+ if (excludedNodes != null && excludedNodes.size() != 0) {
+ excluded.addAll(excludedNodes);
+ }
+ excluded.add(anchor);
+
+ Node pick = networkTopology.chooseRandom(
+ anchor.getNetworkLocation(), excluded);
+ DatanodeDetails pickedNode = (DatanodeDetails) pick;
+ // exclude the picked node for next time
+ if (excludedNodes != null) {
+ excludedNodes.add(pickedNode);
+ }
+ return pickedNode;
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index b7a9813..29da9da 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -16,11 +16,13 @@
*/
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.net.NetConstants;
import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
import org.apache.hadoop.hdds.scm.net.Node;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
@@ -85,7 +87,7 @@ public class MockNodeManager implements NodeManager {
private final SCMNodeStat aggregateStat;
private boolean safemode;
private final Map<UUID, List<SCMCommand>> commandMap;
- private final Node2PipelineMap node2PipelineMap;
+ private Node2PipelineMap node2PipelineMap;
private final Node2ContainerMap node2ContainerMap;
private NetworkTopology clusterMap;
private ConcurrentHashMap<String, String> dnsToUuidMap;
@@ -99,6 +101,7 @@ public class MockNodeManager implements NodeManager {
this.node2ContainerMap = new Node2ContainerMap();
this.dnsToUuidMap = new ConcurrentHashMap();
aggregateStat = new SCMNodeStat();
+ clusterMap = new NetworkTopologyImpl(new Configuration());
if (initializeFakeNodes) {
for (int x = 0; x < nodeCount; x++) {
DatanodeDetails dd = TestUtils.randomDatanodeDetails();
@@ -250,6 +253,16 @@ public class MockNodeManager implements NodeManager {
}
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param datanodeDetails DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails datanodeDetails) {
+ return node2PipelineMap.getPipelinesCount(datanodeDetails.getUuid());
+ }
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -259,6 +272,22 @@ public class MockNodeManager implements NodeManager {
}
/**
+ * Get the entire Node2PipelineMap.
+ * @return Node2PipelineMap
+ */
+ public Node2PipelineMap getNode2PipelineMap() {
+ return node2PipelineMap;
+ }
+
+ /**
+ * Set the Node2PipelineMap.
+ * @param node2PipelineMap Node2PipelineMap
+ */
+ public void setNode2PipelineMap(Node2PipelineMap node2PipelineMap) {
+ this.node2PipelineMap = node2PipelineMap;
+ }
+
+ /**
* Remove a pipeline information from the NodeManager.
* @param pipeline - Pipeline to be removed
*/
@@ -488,6 +517,11 @@ public class MockNodeManager implements NodeManager {
return getNodeByUuid(dnsToUuidMap.get(address));
}
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return clusterMap;
+ }
+
public void setNetworkTopology(NetworkTopology topology) {
this.clusterMap = topology;
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
new file mode 100644
index 0000000..2e0d0b1
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.*;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Test for PipelinePlacementPolicy.
+ */
+public class TestPipelinePlacementPolicy {
+ private MockNodeManager nodeManager;
+ private PipelinePlacementPolicy placementPolicy;
+ private static final int PIPELINE_PLACEMENT_MAX_NODES_COUNT = 10;
+
+ @Before
+ public void init() throws Exception {
+ nodeManager = new MockNodeManager(true,
+ PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ placementPolicy =
+ new PipelinePlacementPolicy(nodeManager, new OzoneConfiguration());
+ }
+
+ @Test
+ public void testChooseNodeBasedOnNetworkTopology() {
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ DatanodeDetails anchor = placementPolicy.chooseNode(healthyNodes);
+ // anchor should be removed from healthyNodes after being chosen.
+ Assert.assertFalse(healthyNodes.contains(anchor));
+
+ List<DatanodeDetails> excludedNodes =
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT);
+ DatanodeDetails nextNode = placementPolicy.chooseNodeFromNetworkTopology(
+ nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes);
+ // excludedNodes should contain nextNode after being chosen.
+ Assert.assertTrue(excludedNodes.contains(nextNode));
+ // nextNode should not be the same as anchor.
+ Assert.assertTrue(anchor.getUuid() != nextNode.getUuid());
+ }
+
+ @Test
+ public void testChooseNodeBasedOnRackAwareness() {
+ List<DatanodeDetails> healthyNodes = overWriteLocationInNodes(
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY));
+ DatanodeDetails anchor = placementPolicy.chooseNode(healthyNodes);
+ NetworkTopology topologyWithDifRacks =
+ createNetworkTopologyOnDifRacks();
+ DatanodeDetails nextNode = placementPolicy.chooseNodeBasedOnRackAwareness(
+ healthyNodes, new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ topologyWithDifRacks, anchor);
+ Assert.assertFalse(topologyWithDifRacks.isSameParent(anchor, nextNode));
+ }
+
+ private final static Node[] NODES = new NodeImpl[] {
+ new NodeImpl("h1", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h2", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h3", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h4", "/r1", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h5", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h6", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h7", "/r2", NetConstants.NODE_COST_DEFAULT),
+ new NodeImpl("h8", "/r2", NetConstants.NODE_COST_DEFAULT),
+ };
+
+
+ private NetworkTopology createNetworkTopologyOnDifRacks() {
+ NetworkTopology topology = new NetworkTopologyImpl(new Configuration());
+ for (Node n : NODES) {
+ topology.add(n);
+ }
+ return topology;
+ }
+
+ private List<DatanodeDetails> overWriteLocationInNodes(
+ List<DatanodeDetails> datanodes) {
+ List<DatanodeDetails> results = new ArrayList<>(datanodes.size());
+ for (int i = 0; i < datanodes.size(); i++) {
+ DatanodeDetails datanode = datanodes.get(i);
+ DatanodeDetails result = DatanodeDetails.newBuilder()
+ .setUuid(datanode.getUuidString())
+ .setHostName(datanode.getHostName())
+ .setIpAddress(datanode.getIpAddress())
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.STANDALONE))
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.RATIS))
+ .addPort(datanode.getPort(DatanodeDetails.Port.Name.REST))
+ .setNetworkLocation(NODES[i].getNetworkLocation()).build();
+ results.add(result);
+ }
+ return results;
+ }
+
+ @Test
+ public void testHeavyNodeShouldBeExcluded() throws SCMException{
+ List<DatanodeDetails> healthyNodes =
+ nodeManager.getNodes(HddsProtos.NodeState.HEALTHY);
+ int nodesRequired = healthyNodes.size()/2;
+ // only minority of healthy NODES are heavily engaged in pipelines.
+ int minorityHeavy = healthyNodes.size()/2 - 1;
+ List<DatanodeDetails> pickedNodes1 = placementPolicy.chooseDatanodes(
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ nodesRequired, 0);
+ // modify node to pipeline mapping.
+ insertHeavyNodesIntoNodeManager(healthyNodes, minorityHeavy);
+ // NODES should be sufficient.
+ Assert.assertEquals(nodesRequired, pickedNodes1.size());
+ // make sure pipeline placement policy won't select duplicated NODES.
+ Assert.assertTrue(checkDuplicateNodesUUID(pickedNodes1));
+
+ // majority of healthy NODES are heavily engaged in pipelines.
+ int majorityHeavy = healthyNodes.size()/2 + 2;
+ insertHeavyNodesIntoNodeManager(healthyNodes, majorityHeavy);
+ boolean thrown = false;
+ List<DatanodeDetails> pickedNodes2 = null;
+ try {
+ pickedNodes2 = placementPolicy.chooseDatanodes(
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ new ArrayList<>(PIPELINE_PLACEMENT_MAX_NODES_COUNT),
+ nodesRequired, 0);
+ } catch (SCMException e) {
+ Assert.assertFalse(thrown);
+ thrown = true;
+ }
+ // NODES should NOT be sufficient and exception should be thrown.
+ Assert.assertNull(pickedNodes2);
+ Assert.assertTrue(thrown);
+ }
+
+ private boolean checkDuplicateNodesUUID(List<DatanodeDetails> nodes) {
+ HashSet<UUID> uuids = nodes.stream().
+ map(DatanodeDetails::getUuid).
+ collect(Collectors.toCollection(HashSet::new));
+ return uuids.size() == nodes.size();
+ }
+
+ private Set<PipelineID> mockPipelineIDs(int count) {
+ Set<PipelineID> pipelineIDs = new HashSet<>(count);
+ for (int i = 0; i < count; i++) {
+ pipelineIDs.add(PipelineID.randomId());
+ }
+ return pipelineIDs;
+ }
+
+ private void insertHeavyNodesIntoNodeManager(
+ List<DatanodeDetails> nodes, int heavyNodeCount) throws SCMException{
+ if (nodes == null) {
+ throw new SCMException("",
+ SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
+ }
+
+ int considerHeavyCount =
+ ScmConfigKeys.OZONE_DATANODE_MAX_PIPELINE_ENGAGEMENT_DEFAULT + 1;
+
+ Node2PipelineMap mockMap = new Node2PipelineMap();
+ for (DatanodeDetails node : nodes) {
+ // mock heavy node
+ if (heavyNodeCount > 0) {
+ mockMap.insertNewDatanode(
+ node.getUuid(), mockPipelineIDs(considerHeavyCount));
+ heavyNodeCount--;
+ } else {
+ mockMap.insertNewDatanode(node.getUuid(), mockPipelineIDs(1));
+ }
+ }
+ nodeManager.setNode2PipelineMap(mockMap);
+ }
+}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 30a75ef..f36d97f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -20,6 +20,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
@@ -165,6 +166,16 @@ public class ReplicationNodeManagerMock implements NodeManager {
}
/**
+ * Get the count of pipelines a datanodes is associated with.
+ * @param dnId DatanodeDetails
+ * @return The number of pipelines
+ */
+ @Override
+ public int getPipelinesCount(DatanodeDetails dnId) {
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+
+ /**
* Add pipeline information in the NodeManager.
* @param pipeline - Pipeline to be added
*/
@@ -326,4 +337,9 @@ public class ReplicationNodeManagerMock implements NodeManager {
public DatanodeDetails getNodeByAddress(String address) {
return null;
}
+
+ @Override
+ public NetworkTopology getClusterNetworkTopologyMap() {
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org