You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2021/07/16 21:30:50 UTC
[hadoop] branch fgl updated (9a60e53 -> b1e2c07)
This is an automated email from the ASF dual-hosted git repository.
shv pushed a change to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
omit 9a60e53 Add namespace key for INode. (shv)
omit 1f1a0c4 INodeMap with PartitionedGSet and per-partition locking.
add 1b69942 HDFS-15923. RBF: Authentication failed when rename accross sub clusters (#2819). Contributed by zhuobin zheng.
add 7f93349 HADOOP-17644. Add back the exceptions removed by HADOOP-17432 for compatibility. Contributed by Quan Li.
add 6d6766b HADOOP-17690. Improve the log for The DecayRpcScheduler. Contributed by Bhavik Patel.
add 359c0c1 HDFS-16003. ProcessReport print invalidatedBlocks should judge debug level at first. Contributed by lei w.
add 8f850b4 HADOOP-17678. Add Dockerfile for Centos 7 (#2967)
add 8d5cc98 HDFS-15997. Implement dfsadmin -provisionSnapshotTrash -all (#2958)
add 9143088 HADOOP-17665 Ignore missing keystore configuration in reloading mechanism
add b944084 HDFS-16007. Deserialization of ReplicaState should avoid throwing ArrayIndexOutOfBoundsException (#2982)
add c80f074 HADOOP-17686. Avoid potential NPE by using Path#getParentPath API in hadoop-huaweicloud (#2990)
add 29105ff HADOOP-17683. Update commons-io to 2.8.0 (#2974)
add 626be24 YARN-10571. Refactor dynamic queue handling logic. Contributed by Andras Gyori.
add fdd20a3 HADOOP-17689. Avoid Potential NPE in org.apache.hadoop.fs (#3008)
add 35ca1dc HADOOP-17685. Fix junit deprecation warnings in hadoop-common module. (#2983)
add 2c4ab72 HADOOP-16822. Provide source artifacts for hadoop-client-api. Contributed by Karel Kolman.
add 2f67a26 Bump acorn (#3003)
add 881ab4e Bump underscore (#2997)
add 2e58fb6 HADOOP-17693. Dockerfile for building on Centos 8 (#3006)
add 5404ab4 MAPREDUCE-7343. Increase the job name max length in mapred job -list. (#2995). Contributed by Ayush Saxena.
add d2b0675 YARN-10737: Fix typos in CapacityScheduler#schedule. (#2911)
add e7f0e80 YARN-10761: Add more event type to RM Dispatcher event metrics. Contributed by Qi Zhu.
add 344cacc YARN-10545. Improve the readability of diagnostics log in yarn-ui2 web page. #2540
add fcd4140 HDFS-15988. Stabilise HDFS Pre-Commit. (#2860). Contributed by Ayush Saxena.
add d92a25b YARN-10555. Missing access check before getAppAttempts (#2608)
add 8891e5c YARN-10763. Add the number of containers assigned per second metrics to ClusterMetrics. Contributed by chaosju.
add eb72628 YARN-10258. Add metrics for 'ApplicationsRunning' in NodeManager. Contributed by ANANDA G B.
add 110cda3 HADOOP-17703. checkcompatibility.py errors out when specifying annotations. (#3017)
add 86729e1 HADOOP-17699. Remove hardcoded SunX509 usage from SSLFactory. (#3016)
add acd712c HDFS-15877. BlockReconstructionWork should resetTargets() before BlockManager#validateReconstructionWork return false (#2747)
add f8b0063 Bump node-sass (#3004)
add 2960d83 HADOOP-17426. Upgrade to hadoop-thirdparty-1.1.0. (#3024)
add 43bf009 HDFS-15757 RBF: Improving Router Connection Management (#2651)
add 3f5a66c HADOOP-17663. Remove useless property hadoop.assemblies.version in pom file. (#3025)
add f724792 HDFS-16018. Optimize the display of hdfs "count -e" or "count -t" com… (#2994)
add e4062ad HADOOP-17115. Replace Guava Sets usage by Hadoop's own Sets in hadoop-common and hadoop-tools (#2985)
add 0d59500 HADOOP-14922. Build of Mapreduce Native Task module fails with unknown opcode "bswap". Contributed by Anup Halarnkar.
add c807381 YARN-9279. Remove the old hamlet package. (#2986)
add 43e77ca YARN-10779. Add option to disable lowercase conversion in GetApplicationsRequestPBImpl and ApplicationSubmissionContextPBImpl. Contributed by Peter Bacsko
add d146ab3 YARN-10766. [UI2] Bump moment-timezone to 0.5.33. Contributed by Andras Gyori
add ad923ad HADOOP-17718. Explicitly set locale in the Dockerfile. (#3034)
add c70ee2d HADOOP-17700. ExitUtil#halt info log should log HaltException (#3015)
add 1e44bdb YARN-7769. FS QueueManager should not create default queue at init. Contributed by Benjamin Teke
add 6bb0892 YARN-10753. Document the removal of FS default queue creation. Contributed by Benjamin Teke
add 2bbeae3 HDFS-15790. Make ProtobufRpcEngineProtos and ProtobufRpcEngineProtos2 Co-Exist (#2767)
add 1576f81 HADOOP-17723. [build] fix the Dockerfile for ARM (#3037)
add c665ab0 HADOOP-17670. S3AFS and ABFS to log IOStats at DEBUG mode or optionally at INFO level in close() (#2963)
add 5f40003 HADOOP-17705. S3A to add Config to set AWS region (#3020)
add 59172ad YARN-10771. Add cluster metric for size of SchedulerEventQueue and RMEventQueue. Contributed by chaosju.
add 2a206c2 HADOOP-17722. Replace Guava Sets usage by Hadoop's own Sets in hadoop-mapreduce-project (#3032)
add 028ec47 HADOOP-17720. Replace Guava Sets usage by Hadoop's own Sets in HDFS (#3031)
add 996d31f HADOOP-17721. Replace Guava Sets usage by Hadoop's own Sets in hadoop-yarn-project (#3033)
add 832a3c6 HADOOP-17511. Add audit/telemetry logging to S3A connector (#2807)
add 1cbcde0 YARN-10770. container-executor permission is wrong in SecureContainer.md. Contributed by Siddharth Ahuja.
add 2541efa YARN-10783. Allow definition of auto queue template properties in root. Contributed by Andras Gyori
add b86a6eb YARN-10782. Extend /scheduler endpoint with template properties. Contributed by Andras Gyori
add c458182 YARN-10691. DominantResourceCalculator isInvalidDivisor should consider only countable resource types. Contributed by Bilwa S T.
add e18d806 HDFS-16024: RBF: Rename data to the Trash should be based on src loca… (#3009)
add 3fdeb74 HDFS-16031. Possible Resource Leak in org.apache.hadoop.hdfs.server.aliasmap#InMemoryAliasMap (#3027)
add 8a489ce MAPREDUCE-7348. TestFrameworkUploader#testNativeIO fails. (#3053)
add 986d0a4 HADOOP-17732. Keep restrict-imports-enforcer-rule for Guava Sets in hadoop-main pom (#3049)
add 98a74e2 HDFS-16041. TestErasureCodingCLI fails (#3052)
add bcaeb1a Revert "Revert "HDFS-15971. Make mkstemp cross platform (#2898)"" (#3044)
add 1abd03d HDFS-15915. Race condition with async edits logging due to updating txId outside of the namesystem log. Contributed by Konstantin V Shvachko.
add 53ff2df Revert "HADOOP-17563. Update Bouncy Castle to 1.68. (#2740)" (#3055)
add 8ce30f5 HDFS-16040. RpcQueueTime metric counts requeued calls as unique events. Contributed by Simbarashe Dzinamarira.
add d5ad181 MAPREDUCE-7287. Distcp will delete exists file , If we use "-delete and -update" options and distcp file. (#2852)
add 9983ab8 HDFS-16046. TestBalancerProcedureScheduler and TestDistCpProcedure timeouts. (#3060)
add b7f6946 HDFS-15973. RBF: Add permission check before doing router federation rename. Contributed by Jinglun.
add 2b11e16 Revert "HDFS-15973." because the user.name and user.email message is not correct.
add e90c41a HDFS-15973. RBF: Add permission check before doing router federation rename. Contributed by Jinglun.
add bd7b97b HADOOP-17735. Upgrade AWS SDK to 1.11.1026 (#3057)
add bce14e7 HDFS-16051. Misspelt words in DataXceiver.java line 881 and line 885. Contributed by Ning Sheng.
add b8ab193 Revert "HDFS-15982. Deleted data using HTTP API should be saved to the trash (#2927)"
add 9a0a808 HADOOP-17739. Use hadoop-thirdparty 1.1.1. (#3064)
add b38b00e HDFS-15998. Fix NullPointException In listOpenFiles (#3036)
add e9339aa YARN-10797. Logging parameter issues in scheduler package. Contributed by Szilard Nemeth
add 2707f69 YARN-10787. Queue submit ACL check is wrong when CS queue is ambiguous. Contributed by Gergely Pollak
add 76d92eb HADOOP-17596. ABFS: Change default Readahead Queue Depth from num(processors) to const (#2795)
add 59fc406 HADOOP-17152. Provide Hadoop's own Lists utility to reduce dependency on Guava (#3061)
add 200eec8 YARN-10796. Capacity Scheduler: dynamic queue cannot scale out properly if its capacity is 0%. Contributed by Peter Bacsko
add 4a26a61 HDFS-16033 Fix issue of the StatisticsDataReferenceCleaner cleanUp (#3042)
add 9c7b8cf Fix container-executor
add 207c927 MAPREDUCE-7350. Replace Guava Lists usage by Hadoop's own Lists in hadoop-mapreduce-project (#3074)
add f4b24c6 HADOOP-17743. Replace Guava Lists usage by Hadoop's own Lists in hadoop-common, hadoop-tools and cloud-storage projects (#3072)
add 57a3613 HDFS-16050. Some dynamometer tests fail. (#3079)
add 1c0b2ed HADOOP-17727. Modularize docker images (#3043)
add c748fce HDFS-16048. RBF: Print network topology on the router web (#3062)
add ec16b1d YARN-10807. Parents node labels are incorrectly added to child queues in weight mode. Contributed by Benjamin Teke.
add 9445abb YARN-10792. Set Completed AppAttempt LogsLink to Log Server URL. Contributed by Abhinaba Sarkar
add a2a0283 HDFS-16042. DatanodeAdminMonitor scan should be delay based (#3058)
add 762a83e HADOOP-17631. Configuration ${env.VAR:-FALLBACK} to eval FALLBACK when restrictSystemProps=true (#2977)
add 00d372b HADOOP-17725. Improve error message for token providers in ABFS (#3041)
add a1a3184 HDFS-15916. Addendum. DistCp: Backward compatibility: Distcp fails from Hadoop 3 to Hadoop 2 for snapshotdiff. (#3056)
add ca27744 YARN-10809. Missing dependency causing NoClassDefFoundError in TestHBaseTimelineStorageUtils (#3081)
add 4c039fa HADOOP-17715 ABFS: Append blob tests with non HNS accounts fail (#3028)
add eefa664 HDFS-16054. Replace Guava Lists usage by Hadoop's own Lists in hadoop-hdfs-project (#3073)
add 81d7069 YARN-10805. Replace Guava Lists usage by Hadoop's own Lists in hadoop-yarn-project (#3075)
add b3c7254 HADOOP-17750. Fix asf license errors in newly added files by HADOOP-17727. (#3083)
add 9a17f97 YARN-10803. [JDK 11] TestRMFailoverProxyProvider and TestNoHaRMFailoverProxyProvider fails by ClassCastException. (#3068)
add 4ef27a5 HADOOP-17753. Keep restrict-imports-enforcer-rule for Guava Lists in top level hadoop-main pom (#3087)
add 6e5692e Revert "HDFS-16033 Fix issue of the StatisticsDataReferenceCleaner cleanUp (#3042)"
add 51ebf7b HDFS-16057. Make sure the order for location in ENTERING_MAINTENANCE state (#3084)
add 352949d HDFS-15671. testBalancerRPCDelayQpsDefault fails intermittently (#3097)
add 2cf952b HADOOP-17643 WASB : Make metadata checks case insensitive (#2972)
add 7003997 YARN-10789. RM HA startup can fail due to race conditions in ZKConfigurationStore. Contributed by Tarun Parimi
add f0bdc42 YARN-10816. Avoid doing delegation token ops when yarn.timeline-service.http-authentication.type=simple. Contributed by Tarun Parimi
add de9ca9f HADOOP-17547 Magic committer to downgrade abort in cleanup if list uploads fails with access denied (#3051)
add 5970c63 HADOOP-17645 Fix test failures in org.apache.hadoop.fs.azure.ITestOutputStreamSemantics. (#2926)
add 35e4c31 HADOOP-17714 ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs (#3035)
add ae9630f HADOOP-17756. Increase precommit job timeout from 20 hours to 24 hours. (#3090)
add ebee2ae HDFS-16055. Quota is not preserved in snapshot INode (#3078)
add e31d060 YARN-10802. Change Capacity Scheduler minimum-user-limit-percent to accept decimal values. Contributed by Benjamin Teke
add c1bf3cb HDFS-16016. BPServiceActor to provide new thread to handle IBR (#2998)
add 839fcf7 HDFS-16068. WebHdfsFileSystem has a possible connection leak in connection with HttpFS (#3104)
add a77bf7c Make upstream aware of 3.3.1 release
add 9a6a11c YARN-10767. Yarn Logs Command retrying on Standby RM for 30 times. Contributed by D M Murali Krishna Reddy.
add 2b304ad HDFS-16073. Remove redundant RPC requests for getFileLinkInfo in ClientNamenodeProtocolTranslatorPB. Contributed by lei w.
add 428478b YARN-10801. Fix Auto Queue template to properly set all configuration properties. Contributed by Andras Gyori
add 6562391 YARN-10813. Set default capacity of root for node labels. Contributed by Andras Gyori
add 9e7c7ad HADOOP-17760. Delete hadoop.ssl.enabled and dfs.https.enable from docs and core-default.xml (#3099)
add 56d2497 HDFS-13671. Namenode deletes large dir slowly caused by FoldedTreeSet#removeAndGet (#3065)
add 1ad674e HDFS-16074. Remove an expensive debug string concatenation (#3107)
add 51991c4 HADOOP-17724. Add Dockerfile for Debian 10 (#3038)
add 643dfd6 HDFS-15842. HDFS mover to emit metrics. (#2738)
add c491f81 HDFS-16078. Remove unused parameters for DatanodeManager.handleLifeline() (#3119). Contributed by tomscut.
add c255feb HDFS-16061. DFTestUtil.waitReplication can produce false positives (#3095). Contributed by Ahmed Hussein.
add d9eccb1 HDFS-16080. RBF: Invoking method in all locations should break the loop after successful result (#3121). Contributed by Viraj Jasani.
add bdc9c88 HDFS-16079. Improve the block state change log (#3120). Contributed by tomscut.
add c488abb HDFS-16075. Use empty array constants present in StorageType and DatanodeInfo to avoid creating redundant objects (#3115)
add 6e11461 MAPREDUCE-7354. Use empty array constant present in TaskCompletionEvent to avoid creating redundant objects (#3123)
add 10b79a2 HDFS-14575. LeaseRenewer#daemon threads leak in DFSClient. Contributed by Renukaprasad C.
add d9fbb3c HDFS-16082. Atomic operations on exceptionsSinceLastBalance and failedTimesSinceLastSuccessfulBalance in Balancer (#3127)
add 11d144d YARN-10826. [UI2] Upgrade Node.js to v12.22.1. (#3128)
add ef5dbc7 HADOOP-17766. CI for Debian 10 (#3129)
add fdef2b4 HDFS-16076. Avoid using slow DataNodes for reading by sorting locations (#3117)
add f78b6d4 HDFS-16085. Move the getPermissionChecker out of the read lock (#3134). Contributed by tomscut.
add 581f43d HADOOP-17769. Upgrade JUnit to 4.13.2. fixes TestBlockRecovery (#3130). Contributed by Ahmed Hussein.
add 5b7f68a HADOOP-17771. S3AFS creation fails "Unable to find a region via the region provider chain." (#3133)
add 0d07837 HDFS-16044. Fix getListing call getLocatedBlocks even source is a directory. Contributed by ludun.
add 0934e78 YARN-10780. Optimise retrieval of configured node labels in CS queues. Contributed by Andras Gyori.
add 748570b Revert "HDFS-16044. Fix getListing call getLocatedBlocks even source is a directory. Contributed by ludun."
add ed5d10e HADOOP-17764. S3AInputStream read does not re-open the input stream on the second read retry attempt (#3109)
add 7c7d02e YARN-10824. Title not set for JHS and NM webpages. Contributed by Bilwa S T.
add 3788fe5 HDFS-13916. Distcp SnapshotDiff to support WebHDFS. Contributed by Xun REN.
add 6b4987f YARN-10820. Make GetClusterNodesRequestPBImpl thread safe. Contributed by Swathi Chandrashekar.
add dc6f456 YARN-10834. Intra-queue preemption: apps that don't use defined custom resource won't be preempted. Contributed by Eric Payne.
add 93a1685 HDFS-16092. Avoid creating LayoutFlags redundant objects (#3150)
add 95454d8 HDFS-16089. Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor (#3146)
add c005195 HADOOP-17777. Update clover-maven-plugin version from 3.3.0 to 4.4.1 (#3152)
add 4cac6ec HDFS-16096. Delete useless method DirectoryWithQuotaFeature#setQuota (#3156)
add 20a4b1a HADOOP-17331. [JDK 16] TestDNS fails (#2884)
add d2e7003 HDFS-16090. Fine grained lock for datanodeNetworkCounts (#3148)
add 7c999e2 HADOOP-17778. CI for Centos 8 (#3151)
add 56c7ada HDFS-16086. Add volume information to datanode log for tracing (#3136)
add 52b9319 HDFS-16104. Remove unused parameter and fix java doc for DiskBalancerCLI (#3160). Contributed by tomscut.
add 9be1733 HDFS-16099. Make bpServiceToActive to be volatile. (#3159)
add 390f860 HDFS-16106. Fix flaky unit test TestDFSShell (#3168)
add ea259f2 HADOOP-17774. S3A bytesRead FS statistic showing twice the correct value (#3144)
add 35570e4 HADOOP-17290. ABFS: Add Identifiers to Client Request Header (#2520)
add b87bac1 HDFS-16109. Fix flaky some unit tests since they offen timeout (#3172)
add da575a5 HDFS-16108. Fix incorrect log placeholders used in JournalNodeSyncer (#3169)
add 93ad7c3 HADOOP-17250 Lot of short reads can be merged with readahead. (#3110)
add f639fbc HADOOP-17402. Add GCS config to the core-site (#2638)
add a5db683 HADOOP-17749. Remove lock contention in SelectorPool of SocketIOWithTimeout (#3080)
add a447cfd HDFS-16110. Remove unused method reportChecksumFailure in DFSClient (#3174)
add c85a183 HDFS-16101. Remove unuse variable and IoException in ProvidedStorageMap. Contributed by lei w.
add c42441a HADOOP-17775. Remove JavaScript package from Docker environment. (#3137)
add 3f4221e MAPREDUCE-7351 - CleanupJob during handle of SIGTERM signal (#3176)
add 9381071 HDFS-16114. the balancer parameters print error (#3179)
add 7581413 MAPREDUCE-7353: Mapreduce job fails when NM is stopped. Contributed by Bilwa S T (BilwaST)
add b4c2647 HDFS-16088. Standby NameNode process getLiveDatanodeStorageReport req… (#3140)
add 618c921 HADOOP-17788. Replace IOUtils#closeQuietly usages by Hadoop's own utility (#3171)
add 6450c1e HDFS-16121. Iterative snapshot diff report can generate duplicate records for creates, deletes and Renames. (#3188)
add b6c06c4 HDFS-15796. ConcurrentModificationException error happens on NameNode occasionally. Contributed by Daniel Ma
add 6ac5d8a HADOOP-17794. Add a sample configuration to use ZKDelegationTokenSecretManager in Hadoop KMS (#3190)
add ba325a8 HADOOP-17793. Better token validation (#3189)
add fef53aa HDFS-16122. Fix DistCpContext#toString() (#3191). Contributed by tomscut.
add c81f82e HADOOP-12665. Document hadoop.security.token.service.use_ip. (#3187)
add 177d906 HADOOP-17770 WASB : Support disabling buffered reads in positional reads (#3149)
add e95c325 MAPREDUCE-7356. Remove some duplicate dependencies from mapreduce-client's child poms (#3193). Contributed by Viraj Jasani.
add ea90c51 HDFS-15785. Datanode to support using DNS to resolve nameservices to IP addresses to get list of namenodes. (#2639)
add 1dd03cc HADOOP-17028. ViewFS should initialize mounted target filesystems lazily. Contributed by Abhishek Das (#2260)
add 87e0000 HADOOP-17672.Remove an invalid comment content in the FileContext class. (#2961)
add df44178 HADOOP-17795. Provide fallbacks for callqueue.impl and scheduler.impl (#3192)
add 632f64c YARN-10456. RM PartitionQueueMetrics records are named QueueMetrics in Simon metrics registry. Contributed by Eric Payne.
add 4bb25c8 HDFS-15650. Make the socket timeout for computing checksum of striped blocks configurable (#2414)
add d0ee065 HADOOP-16272. Upgrade HikariCP to 4.0.3 (#3204)
add f6f105c HADOOP-17803. Remove WARN logging from LoggingAuditor when executing a request outside an audit span (#3207)
add 997d749 HADOOP-17801. No error message reported when bucket doesn't exist in S3AFS (#3202)
add 4700271 HDFS-16127. Improper pipeline close recovery causes a permanent write failure or data loss. Contributed by Kihwal Lee.
new 0e7d216 INodeMap with PartitionedGSet and per-partition locking.
new 1829b58 Add namespace key for INode. (shv)
new b1e2c07 HDFS-16125. [FGL] Fix the iterator for PartitionedGSet. Contributed by Xing Lin. (#3197)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9a60e53)
\
N -- N -- N refs/heads/fgl (b1e2c07)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BUILDING.txt | 2 +-
LICENSE-binary | 10 +-
dev-support/Jenkinsfile | 152 +-
dev-support/bin/checkcompatibility.py | 2 +-
dev-support/bin/create-release | 2 +-
dev-support/bin/hadoop.sh | 3 +-
dev-support/docker/Dockerfile | 155 +-
dev-support/docker/Dockerfile_aarch64 | 135 +-
dev-support/docker/Dockerfile_centos_7 | 79 +
dev-support/docker/Dockerfile_centos_8 | 111 +
dev-support/docker/Dockerfile_debian_10 | 105 +
dev-support/docker/README.md | 74 +
dev-support/docker/pkg-resolver/check_platform.py | 50 +
dev-support/docker/pkg-resolver/install-boost.sh | 56 +
dev-support/docker/pkg-resolver/install-cmake.sh | 53 +
.../docker/pkg-resolver/install-common-pkgs.sh | 22 +
dev-support/docker/pkg-resolver/install-epel.sh | 49 +
.../docker/pkg-resolver/install-hadolint.sh | 35 +
.../docker/pkg-resolver/install-intel-isa-l.sh | 58 +
dev-support/docker/pkg-resolver/install-maven.sh | 49 +
.../docker/pkg-resolver/install-protobuf.sh | 57 +
.../docker/pkg-resolver/install-spotbugs.sh | 50 +
dev-support/docker/pkg-resolver/install-yasm.sh | 49 +
.../docker/pkg-resolver/install-zstandard.sh | 53 +
dev-support/docker/pkg-resolver/packages.json | 348 +
dev-support/docker/pkg-resolver/platforms.json | 7 +
dev-support/docker/pkg-resolver/resolve.py | 61 +
dev-support/jenkins.sh | 212 +
hadoop-client-modules/hadoop-client-api/pom.xml | 4 +
.../org/apache/hadoop/fs/obs/OBSCommonUtils.java | 2 +-
.../apache/hadoop/fs/obs/OBSObjectBucketUtils.java | 9 +-
.../security/authentication/util/KerberosUtil.java | 7 +-
hadoop-common-project/hadoop-common/pom.xml | 49 +-
.../java/org/apache/hadoop/conf/Configuration.java | 79 +-
.../java/org/apache/hadoop/crypto/CryptoCodec.java | 2 +-
.../java/org/apache/hadoop/fs/BlockLocation.java | 2 +-
.../apache/hadoop/fs/CommonConfigurationKeys.java | 24 +
.../java/org/apache/hadoop/fs/ContentSummary.java | 27 +
.../org/apache/hadoop/fs/DelegateToFileSystem.java | 7 +-
.../java/org/apache/hadoop/fs/FileContext.java | 5 -
.../main/java/org/apache/hadoop/fs/FileUtil.java | 18 +-
.../src/main/java/org/apache/hadoop/fs/Path.java | 26 +-
.../main/java/org/apache/hadoop/fs/QuotaUsage.java | 12 +-
.../org/apache/hadoop/fs/audit/AuditConstants.java | 108 +
.../hadoop/fs/audit/AuditStatisticNames.java | 40 +-
.../apache/hadoop/fs/audit/CommonAuditContext.java | 288 +
.../org/apache/hadoop/fs/audit}/package-info.java | 14 +-
.../org/apache/hadoop/fs/permission/AclStatus.java | 2 +-
.../org/apache/hadoop/fs/permission/AclUtil.java | 2 +-
.../org/apache/hadoop/fs/shell/AclCommands.java | 3 +-
.../java/org/apache/hadoop/fs/shell/Command.java | 3 +
.../java/org/apache/hadoop/fs/shell/Count.java | 12 +-
.../java/org/apache/hadoop/fs/shell/PathData.java | 18 +-
.../hadoop/fs/statistics/IOStatisticsLogging.java | 31 +
.../hadoop/fs/statistics/StoreStatisticNames.java | 10 +
.../fs/statistics/impl/EmptyIOStatisticsStore.java | 182 +
.../fs/statistics/impl/IOStatisticsBinding.java | 42 +
.../apache/hadoop/fs/store}/LogExactlyOnce.java | 12 +-
.../fs/store/audit/ActiveThreadSpanSource.java | 34 +-
.../hadoop/fs/store/audit/AuditEntryPoint.java | 55 +
.../apache/hadoop/fs/store/audit/AuditSpan.java | 109 +
.../hadoop/fs/store/audit/AuditSpanSource.java | 47 +-
.../hadoop/fs/store/audit/AuditingFunctions.java | 121 +
.../fs/store/audit/HttpReferrerAuditHeader.java | 503 +
.../hadoop/fs/store/audit}/package-info.java | 14 +-
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 79 +-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 101 +-
.../fs/viewfs/ViewFileSystemOverloadScheme.java | 11 +-
.../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 38 +-
.../main/java/org/apache/hadoop/ha/NodeFencer.java | 2 +-
.../java/org/apache/hadoop/http/HttpServer2.java | 36 +-
.../org/apache/hadoop/ipc/DecayRpcScheduler.java | 19 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine.java | 215 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine2.java | 63 +-
.../src/main/java/org/apache/hadoop/ipc/RPC.java | 9 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 84 +-
.../hadoop/metrics2/impl/MBeanInfoBuilder.java | 3 +-
.../hadoop/metrics2/impl/MetricsCollectorImpl.java | 3 +-
.../metrics2/impl/MetricsRecordBuilderImpl.java | 3 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 2 +-
.../apache/hadoop/metrics2/lib/MutableRates.java | 2 +-
.../metrics2/lib/MutableRatesWithAggregation.java | 2 +-
.../org/apache/hadoop/metrics2/util/Servers.java | 3 +-
.../src/main/java/org/apache/hadoop/net/DNS.java | 13 +-
.../hadoop/net/DomainNameResolverFactory.java | 6 +-
.../org/apache/hadoop/net/SocketIOWithTimeout.java | 103 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 4 +-
.../security/ssl/FileMonitoringTimerTask.java | 58 +-
.../security/ssl/ReloadingX509KeystoreManager.java | 2 +-
.../security/ssl/ReloadingX509TrustManager.java | 4 +-
.../org/apache/hadoop/security/ssl/SSLFactory.java | 12 +-
.../org/apache/hadoop/security/token/Token.java | 6 +-
.../hadoop/service/launcher/ServiceLauncher.java | 2 +
.../org/apache/hadoop/util/ChunkedArrayList.java | 1 -
.../main/java/org/apache/hadoop/util/ExitUtil.java | 3 +-
.../org/apache/hadoop/util/JvmPauseMonitor.java | 3 -
.../java/org/apache/hadoop/util/LatchLock.java | 4 +-
.../main/java/org/apache/hadoop/util/Lists.java | 235 +
.../org/apache/hadoop/util/PartitionedGSet.java | 35 +-
.../src/main/java/org/apache/hadoop/util/Sets.java | 377 +
.../java/org/apache/hadoop/util/ToolRunner.java | 5 +
.../main/java/org/apache/hadoop/util/ZKUtil.java | 1 -
.../hadoop/util/functional/RemoteIterators.java | 34 +-
.../src/main/resources/core-default.xml | 81 +-
.../hadoop-common/src/site/markdown/SecureMode.md | 3 +-
.../site/markdown/release/3.3.1/CHANGELOG.3.3.1.md | 750 +
.../markdown/release/3.3.1/RELEASENOTES.3.3.1.md | 293 +
.../hadoop/ipc/protobuf/TestProtosLegacy.java | 9892 ++++++
.../ipc/protobuf/TestRpcServiceProtosLegacy.java | 3313 ++
.../hadoop/conf/TestCommonConfigurationFields.java | 8 +-
.../org/apache/hadoop/conf/TestConfiguration.java | 124 +-
.../apache/hadoop/conf/TestReconfiguration.java | 2 +-
.../crypto/key/TestKeyProviderCryptoExtension.java | 3 +-
.../apache/hadoop/crypto/key/TestValueQueue.java | 2 +-
.../crypto/key/kms/TestKMSClientProvider.java | 3 +-
.../kms/TestLoadBalancingKMSClientProvider.java | 5 +-
.../hadoop/fs/FileSystemContractBaseTest.java | 8 +-
.../org/apache/hadoop/fs/TestBlockLocation.java | 2 +-
.../java/org/apache/hadoop/fs/TestDefaultUri.java | 22 +-
.../hadoop/fs/TestFileSystemStorageStatistics.java | 6 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 24 +-
.../java/org/apache/hadoop/fs/TestFsShellCopy.java | 17 +-
.../java/org/apache/hadoop/fs/TestFsShellList.java | 15 +-
.../org/apache/hadoop/fs/TestFsShellTouch.java | 105 +-
.../org/apache/hadoop/fs/TestLocalFileSystem.java | 8 +-
.../hadoop/fs/TestLocalFileSystemPermission.java | 9 +-
.../hadoop/fs/audit/TestCommonAuditContext.java | 161 +
.../fs/contract/AbstractFSContractTestBase.java | 6 +-
.../hadoop/fs/contract/ContractTestUtils.java | 10 +-
.../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 3 +-
.../apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 59 +-
.../java/org/apache/hadoop/fs/shell/TestCount.java | 29 +-
.../hadoop/fs/shell/TestPrintableString.java | 8 +-
.../org/apache/hadoop/fs/shell/find/TestAnd.java | 3 +-
.../hadoop/fs/shell/find/TestFilterExpression.java | 3 +-
.../org/apache/hadoop/fs/shell/find/TestFind.java | 3 +-
.../org/apache/hadoop/fs/shell/find/TestIname.java | 3 +-
.../org/apache/hadoop/fs/shell/find/TestName.java | 3 +-
.../org/apache/hadoop/fs/shell/find/TestPrint.java | 3 +-
.../apache/hadoop/fs/shell/find/TestPrint0.java | 3 +-
.../apache/hadoop/fs/shell/find/TestResult.java | 4 +-
.../hadoop/fs/viewfs/TestRegexMountPoint.java | 12 +-
.../apache/hadoop/fs/viewfs/TestViewFsConfig.java | 3 +-
.../hadoop/fs/viewfs/ViewFileSystemBaseTest.java | 68 +-
.../java/org/apache/hadoop/ha/DummyHAService.java | 2 +-
.../java/org/apache/hadoop/ha/TestNodeFencer.java | 3 +-
.../apache/hadoop/ha/TestShellCommandFencer.java | 6 +-
.../apache/hadoop/ha/TestZKFailoverController.java | 3 +-
.../org/apache/hadoop/http/TestHttpServer.java | 11 +-
.../hadoop/io/compress/TestDecompressorStream.java | 21 +-
.../zstd/TestZStandardCompressorDecompressor.java | 36 +-
.../apache/hadoop/io/erasurecode/TestECSchema.java | 3 +-
.../io/erasurecode/coder/TestRSErasureCoder.java | 4 +-
.../hadoop/io/erasurecode/coder/TestXORCoder.java | 4 +-
.../hadoop/io/retry/TestDefaultRetryPolicy.java | 18 +-
.../apache/hadoop/ipc/TestCallQueueManager.java | 7 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 152 +-
.../hadoop/ipc/TestProtoBufRpcServerHandoff.java | 2 +-
.../hadoop/metrics2/source/TestJvmMetrics.java | 3 +-
.../test/java/org/apache/hadoop/net/TestDNS.java | 43 +-
.../hadoop/net/TestMockDomainNameResolver.java | 14 +-
.../apache/hadoop/net/TestSocketIOWithTimeout.java | 79 +
.../java/org/apache/hadoop/security/TestKDiag.java | 3 +-
.../org/apache/hadoop/security/TestKDiagNoKDC.java | 3 +-
.../alias/TestCredentialProviderFactory.java | 17 +-
.../TestDefaultImpersonationProvider.java | 4 +-
.../apache/hadoop/security/ssl/TestSSLFactory.java | 16 +
.../TestZKDelegationTokenSecretManager.java | 3 +-
...egationTokenAuthenticationHandlerWithMocks.java | 3 +-
.../hadoop/service/TestServiceOperations.java | 7 +-
.../launcher/AbstractServiceLauncherTestBase.java | 3 +-
.../org/apache/hadoop/test/GenericTestUtils.java | 19 +-
.../apache/hadoop/test/PlatformAssumptions.java | 2 +-
.../apache/hadoop/test/UnitTestcaseTimeLimit.java | 5 +-
.../hadoop/util/TestApplicationClassLoader.java | 1 -
.../org/apache/hadoop/util/TestCrcComposer.java | 3 +-
.../java/org/apache/hadoop/util/TestCrcUtil.java | 3 +-
.../apache/hadoop/util/TestDirectBufferPool.java | 2 -
.../hadoop/util/TestDiskCheckerWithDiskIo.java | 5 +-
.../java/org/apache/hadoop/util/TestLists.java | 102 +
.../org/apache/hadoop/util/TestNativeCrc32.java | 34 +-
.../apache/hadoop/util/TestPartitionedGSet.java | 270 +
.../apache/hadoop/util/TestReflectionUtils.java | 6 +-
.../java/org/apache/hadoop/util/TestShell.java | 3 +-
.../org/apache/hadoop/util/TestSysInfoLinux.java | 9 +-
.../java/org/apache/hadoop/util/TestWinUtils.java | 31 +-
.../hadoop-common/src/test/proto/test_legacy.proto | 101 +
.../src/test/proto/test_rpc_service_legacy.proto | 79 +
.../hadoop/crypto/key/kms/server/KMSAudit.java | 2 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 56 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 6 +-
hadoop-common-project/hadoop-registry/pom.xml | 1 -
.../registry/client/impl/zk/RegistrySecurity.java | 2 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 41 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 6 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 5 +-
.../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 51 +
.../java/org/apache/hadoop/hdfs/DataStreamer.java | 16 +-
.../apache/hadoop/hdfs/DistributedFileSystem.java | 5 +-
.../org/apache/hadoop/hdfs/FileChecksumHelper.java | 3 +-
.../java/org/apache/hadoop/hdfs/XAttrHelper.java | 2 +-
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 +
.../org/apache/hadoop/hdfs/client/HdfsUtils.java | 13 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 12 +
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 34 +-
.../apache/hadoop/hdfs/protocol/LocatedBlock.java | 3 +-
.../apache/hadoop/hdfs/protocol/LocatedBlocks.java | 2 +-
.../hadoop/hdfs/protocol/ReencryptionStatus.java | 2 +-
.../hdfs/protocol/datatransfer/PipelineAck.java | 2 +-
.../datatransfer/sasl/SaslDataTransferClient.java | 2 +-
.../ClientNamenodeProtocolTranslatorPB.java | 7 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 2 +-
.../hdfs/shortcircuit/DomainSocketFactory.java | 4 +-
.../org/apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +-
.../apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 +-
.../hadoop/hdfs/client/impl/TestLeaseRenewer.java | 87 +
.../ha/TestRequestHedgingProxyProvider.java | 2 +-
.../hadoop/hdfs/web/TestURLConnectionFactory.java | 2 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 3 +-
.../apache/hadoop/fs/http/server/FSOperations.java | 28 +-
.../fs/http/server/HttpFSParametersProvider.java | 22 +-
.../apache/hadoop/fs/http/server/HttpFSServer.java | 9 +-
.../org/apache/hadoop/lib/wsrs/Parameters.java | 3 +-
.../apache/hadoop/lib/wsrs/ParametersProvider.java | 2 +-
.../hadoop/fs/http/client/BaseTestHttpFSWith.java | 3 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 62 -
.../lib/service/security/DummyGroupMapping.java | 2 +-
.../hadoop-hdfs-native-client/src/CMakeLists.txt | 3 +-
.../libhdfs-tests/test_libhdfs_mini_stress.c | 4 +-
.../native/libhdfspp/lib/rpc/cyrus_sasl_engine.cc | 2 +-
.../libhdfspp/lib/x-platform/c-api/syscall.cc | 22 +-
.../libhdfspp/lib/x-platform/c-api/syscall.h | 3 +
.../main/native/libhdfspp/lib/x-platform/syscall.h | 29 +
.../libhdfspp/lib/x-platform/syscall_linux.cc | 11 +
.../libhdfspp/lib/x-platform/syscall_windows.cc | 30 +
.../src/main/native/libhdfspp/tests/CMakeLists.txt | 13 +-
.../native/libhdfspp/tests/bad_datanode_test.cc | 1 -
.../native/libhdfspp/tests/configuration_test.cc | 23 +-
.../native/libhdfspp/tests/configuration_test.h | 64 +-
.../native/libhdfspp/tests/hdfs_builder_test.cc | 4 +-
.../libhdfspp/tests/hdfs_configuration_test.cc | 16 +-
.../tests/x-platform/syscall_common_test.cc | 18 +
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 6 +
.../federation/metrics/FederationRPCMBean.java | 15 +
.../federation/metrics/FederationRPCMetrics.java | 10 +
.../federation/resolver/MountTableResolver.java | 80 +-
.../server/federation/resolver/RemoteLocation.java | 14 +
.../federation/router/ConnectionContext.java | 57 +-
.../federation/router/ConnectionManager.java | 53 +-
.../server/federation/router/ConnectionPool.java | 67 +-
.../federation/router/RouterFederationRename.java | 117 +-
.../server/federation/router/RouterHttpServer.java | 3 +
.../router/RouterNetworkTopologyServlet.java | 69 +
.../server/federation/router/RouterRpcClient.java | 32 +-
.../src/main/webapps/router/explorer.html | 24 +-
.../src/main/webapps/router/explorer.js | 50 +-
.../src/main/webapps/router/federationhealth.html | 1 +
.../src/site/markdown/HDFSRouterFederation.md | 5 +-
.../hdfs/server/federation/MockNamenode.java | 2 +-
.../federation/router/TestConnectionManager.java | 28 +-
.../server/federation/router/TestRouterAdmin.java | 3 +-
.../router/TestRouterFederationRename.java | 185 +-
.../router/TestRouterFederationRenameBase.java | 203 +
.../TestRouterFederationRenameInKerberosEnv.java | 299 +
.../TestRouterFederationRenamePermission.java | 246 +
.../router/TestRouterNetworkTopologyServlet.java | 210 +
.../router/TestRouterPolicyProvider.java | 3 +-
.../server/federation/router/TestRouterRpc.java | 2 +-
.../router/TestRouterRpcMultiDestination.java | 2 +-
.../server/federation/router/TestRouterTrash.java | 297 +
.../federation/router/TestRouterUserMappings.java | 2 +-
.../dev-support/jdiff/Apache_Hadoop_HDFS_3.3.1.xml | 835 +
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 27 +-
.../main/java/org/apache/hadoop/hdfs/DFSUtil.java | 128 +-
.../main/java/org/apache/hadoop/hdfs/HAUtil.java | 2 +-
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 6 +
.../apache/hadoop/hdfs/protocol/LayoutFlags.java | 15 +-
.../datatransfer/sasl/SaslDataTransferServer.java | 2 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 5 +-
.../hdfs/qjournal/client/QuorumJournalManager.java | 2 +-
.../hadoop/hdfs/qjournal/server/JournalNode.java | 2 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 8 +-
.../token/block/BlockTokenSecretManager.java | 5 +-
.../delegation/DelegationTokenSecretManager.java | 2 +-
.../hdfs/server/aliasmap/InMemoryAliasMap.java | 22 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 22 +-
.../hdfs/server/balancer/BalancerParameters.java | 4 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 1 +
.../hdfs/server/balancer/NameNodeConnector.java | 108 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 189 +-
.../blockmanagement/BlockInfoContiguous.java | 29 +-
.../server/blockmanagement/BlockInfoStriped.java | 33 +-
.../hdfs/server/blockmanagement/BlockManager.java | 466 +-
.../BlockPlacementPolicyDefault.java | 4 +-
.../blockmanagement/BlockStoragePolicySuite.java | 2 +-
.../hdfs/server/blockmanagement/BlocksMap.java | 66 +-
.../blockmanagement/DatanodeAdminManager.java | 4 +-
.../server/blockmanagement/DatanodeManager.java | 83 +-
.../blockmanagement/DatanodeStorageInfo.java | 127 +-
.../blockmanagement/PendingDataNodeMessages.java | 2 +-
.../PendingReconstructionBlocks.java | 2 +-
.../server/blockmanagement/ProvidedStorageMap.java | 4 +-
.../server/blockmanagement/SlowDiskTracker.java | 2 +-
.../hdfs/server/common/HdfsServerConstants.java | 28 +-
.../org/apache/hadoop/hdfs/server/common/Util.java | 10 +-
.../hdfs/server/datanode/BPOfferService.java | 6 +-
.../hdfs/server/datanode/BPServiceActor.java | 71 +-
.../hdfs/server/datanode/BlockChecksumHelper.java | 2 +-
.../hdfs/server/datanode/BlockPoolManager.java | 14 +-
.../server/datanode/BlockPoolSliceStorage.java | 2 +-
.../hadoop/hdfs/server/datanode/BlockReceiver.java | 9 +-
.../hadoop/hdfs/server/datanode/BlockSender.java | 4 +-
.../apache/hadoop/hdfs/server/datanode/DNConf.java | 15 +
.../hadoop/hdfs/server/datanode/DataNode.java | 32 +-
.../hadoop/hdfs/server/datanode/DataStorage.java | 3 +-
.../hadoop/hdfs/server/datanode/DataXceiver.java | 11 +-
.../hdfs/server/datanode/DataXceiverServer.java | 10 +-
.../hdfs/server/datanode/DirectoryScanner.java | 4 +-
.../hdfs/server/datanode/FileIoProvider.java | 10 +-
.../hadoop/hdfs/server/datanode/Replica.java | 7 +
.../hdfs/server/datanode/ShortCircuitRegistry.java | 6 +-
.../datanode/checker/DatasetVolumeChecker.java | 2 +-
.../erasurecode/StripedBlockReconstructor.java | 7 +
.../server/datanode/fsdataset/FsDatasetSpi.java | 7 +-
.../fsdataset/impl/FsDatasetAsyncDiskService.java | 3 +-
.../datanode/fsdataset/impl/FsDatasetCache.java | 10 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 16 +-
.../fsdataset/impl/MappableBlockLoader.java | 7 +-
.../fsdataset/impl/MemoryMappableBlockLoader.java | 6 +-
.../impl/NativePmemMappableBlockLoader.java | 14 +-
.../fsdataset/impl/PmemMappableBlockLoader.java | 6 +-
.../datanode/fsdataset/impl/PmemVolumeManager.java | 4 +-
.../server/datanode/fsdataset/impl/ReplicaMap.java | 81 +-
.../server/datanode/metrics/DataNodeMetrics.java | 6 +
.../hdfs/server/diskbalancer/command/Command.java | 7 +-
.../server/diskbalancer/command/ReportCommand.java | 5 +-
.../org/apache/hadoop/hdfs/server/mover/Mover.java | 34 +-
.../hadoop/hdfs/server/mover/MoverMetrics.java | 83 +
.../hadoop/hdfs/server/mover/package-info.java} | 17 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 8 +-
.../hdfs/server/namenode/AclTransformation.java | 10 +-
.../hadoop/hdfs/server/namenode/BackupImage.java | 2 +-
.../hadoop/hdfs/server/namenode/CacheManager.java | 11 +-
.../hadoop/hdfs/server/namenode/Checkpointer.java | 11 +-
.../server/namenode/DirectoryWithQuotaFeature.java | 20 +-
.../hdfs/server/namenode/EditLogOutputStream.java | 11 +
.../server/namenode/EncryptionZoneManager.java | 13 +-
.../hadoop/hdfs/server/namenode/FSDirAppendOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSDirAttrOp.java | 3 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 4 +-
.../hdfs/server/namenode/FSDirErasureCodingOp.java | 5 +-
.../namenode/FSDirSatisfyStoragePolicyOp.java | 3 +-
.../hadoop/hdfs/server/namenode/FSDirXAttrOp.java | 7 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 42 +-
.../hdfs/server/namenode/FSEditLogAsync.java | 7 +-
.../hadoop/hdfs/server/namenode/FSEditLogOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 2 +-
.../server/namenode/FSImageFormatProtobuf.java | 2 +-
.../FSImageTransactionalStorageInspector.java | 7 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 21 +-
.../hdfs/server/namenode/FileJournalManager.java | 2 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 4 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 2 +-
.../hadoop/hdfs/server/namenode/JournalSet.java | 24 +-
.../hadoop/hdfs/server/namenode/LeaseManager.java | 15 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 2 +-
.../server/namenode/NNStorageRetentionManager.java | 5 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +-
.../hdfs/server/namenode/NameNodeHttpServer.java | 2 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +-
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 7 +-
.../server/namenode/NetworkTopologyServlet.java | 9 +-
.../hdfs/server/namenode/ReencryptionUpdater.java | 7 +-
.../hdfs/server/namenode/SecondaryNameNode.java | 2 +-
.../hdfs/server/namenode/TransferFsImage.java | 2 +-
.../server/namenode/XAttrPermissionFilter.java | 2 +-
.../server/namenode/ha/StandbyCheckpointer.java | 3 +-
.../snapshot/DirectorySnapshottableFeature.java | 2 +-
.../hdfs/server/namenode/snapshot/Snapshot.java | 30 +-
.../namenode/snapshot/SnapshotDiffListingInfo.java | 6 +-
.../server/namenode/snapshot/SnapshotManager.java | 2 +-
.../server/namenode/top/metrics/TopMetrics.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 63 +-
.../hdfs/server/protocol/BlockReportContext.java | 10 +-
.../hdfs/server/protocol/DatanodeProtocol.java | 1 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 51 +-
.../apache/hadoop/hdfs/tools/DiskBalancerCLI.java | 6 +-
.../tools/offlineImageViewer/FSImageLoader.java | 14 +-
.../offlineImageViewer/PBImageTextWriter.java | 44 +-
.../tools/offlineImageViewer/PBImageXmlWriter.java | 7 +-
.../org/apache/hadoop/hdfs/util/FoldedTreeSet.java | 1285 -
.../java/org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +-
.../src/main/proto/DatanodeProtocol.proto | 3 +-
.../src/main/resources/hdfs-default.xml | 66 +-
.../src/main/webapps/hdfs/explorer.html | 23 +-
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 46 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 6 +-
.../hadoop/fs/viewfs/TestViewFileSystemHdfs.java | 73 +
...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 6 +-
.../fs/viewfs/TestViewFileSystemWithAcls.java | 2 +-
.../apache/hadoop/fs/viewfs/TestViewFsHdfs.java | 78 +
.../hadoop/fs/viewfs/TestViewFsWithAcls.java | 2 +-
.../apache/hadoop/hdfs/AdminStatesBaseTest.java | 2 +-
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 53 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 4 +-
.../org/apache/hadoop/hdfs/MiniDFSNNTopology.java | 2 +-
.../hadoop/hdfs/TestBatchedListDirectories.java | 2 +-
.../apache/hadoop/hdfs/TestBlockStoragePolicy.java | 5 +-
.../org/apache/hadoop/hdfs/TestCrcCorruption.java | 2 +-
.../apache/hadoop/hdfs/TestDFSClientRetries.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSFinalize.java | 3 +-
.../org/apache/hadoop/hdfs/TestDFSInputStream.java | 4 +-
.../org/apache/hadoop/hdfs/TestDFSRollback.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestDFSShell.java | 4 +-
.../java/org/apache/hadoop/hdfs/TestDFSUtil.java | 3 +-
.../org/apache/hadoop/hdfs/TestDatanodeReport.java | 17 +-
.../org/apache/hadoop/hdfs/TestDecommission.java | 69 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +-
.../hadoop/hdfs/TestErasureCodingExerciseAPIs.java | 2 +-
.../org/apache/hadoop/hdfs/TestExtendedAcls.java | 2 +-
.../apache/hadoop/hdfs/TestHDFSPolicyProvider.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestHdfsAdmin.java | 3 +-
.../org/apache/hadoop/hdfs/TestLeaseRecovery.java | 6 +-
.../apache/hadoop/hdfs/TestMaintenanceState.java | 2 +-
.../org/apache/hadoop/hdfs/TestPersistBlocks.java | 3 +
.../java/org/apache/hadoop/hdfs/TestQuota.java | 2 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 2 +-
.../hadoop/hdfs/TestReplaceDatanodeOnFailure.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestSafeMode.java | 2 +-
.../hadoop/hdfs/TestSafeModeWithStripedFile.java | 2 +-
.../hdfs/TestTrashWithSecureEncryptionZones.java | 2 +-
.../hdfs/client/impl/BlockReaderTestUtil.java | 4 +-
.../hadoop/hdfs/net/TestDFSNetworkTopology.java | 6 +-
.../hadoop/hdfs/protocol/TestBlockListAsLongs.java | 4 +-
.../hadoop/hdfs/protocol/TestLocatedBlock.java | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 2 +-
.../hadoop/hdfs/qjournal/MiniJournalCluster.java | 3 +-
.../apache/hadoop/hdfs/qjournal/QJMTestUtil.java | 3 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 6 +-
.../qjournal/client/TestQuorumJournalManager.java | 2 +-
.../client/TestQuorumJournalManagerUnit.java | 2 +-
.../hdfs/qjournal/server/TestJournalNodeSync.java | 2 +-
.../hdfs/security/token/block/TestBlockToken.java | 2 +-
.../hadoop/hdfs/server/balancer/TestBalancer.java | 8 +-
.../hdfs/server/balancer/TestBalancerRPCDelay.java | 8 +-
.../balancer/TestBalancerWithHANameNodes.java | 123 +
.../hdfs/server/blockmanagement/TestBlockInfo.java | 88 +
.../server/blockmanagement/TestBlockManager.java | 94 +-
.../blockmanagement/TestBlockReportLease.java | 5 +-
.../blockmanagement/TestDatanodeManager.java | 9 +-
.../blockmanagement/TestRBWBlockInvalidation.java | 2 +-
...tReconstructStripedBlocksWithRackAwareness.java | 9 +-
.../TestReplicationPolicyExcludeSlowNodes.java | 8 +-
.../blockmanagement/TestSortLocatedBlock.java | 312 +
.../hadoop/hdfs/server/common/TestJspHelper.java | 19 +
.../impl/TestInMemoryLevelDBAliasMapClient.java | 2 +-
.../hdfs/server/datanode/SimulatedFSDataset.java | 7 +-
.../hdfs/server/datanode/TestBPOfferService.java | 9 +-
.../TestBlockHasMultipleReplicasOnSameDN.java | 7 +-
.../hdfs/server/datanode/TestBlockPoolManager.java | 60 +-
.../datanode/TestDataNodeHotSwapVolumes.java | 5 +-
.../hdfs/server/datanode/TestDataNodeMetrics.java | 3 +-
.../datanode/TestDataXceiverLazyPersistHint.java | 4 +-
.../hdfs/server/datanode/TestDatanodeRegister.java | 12 +-
.../hdfs/server/datanode/TestDirectoryScanner.java | 52 +-
.../hadoop/hdfs/server/datanode/TestDiskError.java | 2 +-
.../datanode/TestIncrementalBlockReports.java | 24 +-
.../hdfs/server/datanode/TestLargeBlockReport.java | 5 +-
.../TestNNHandlesBlockReportPerStorage.java | 2 +-
.../datanode/TestNNHandlesCombinedBlockReport.java | 2 +-
.../hdfs/server/datanode/TestRefreshNamenodes.java | 2 +-
.../datanode/extdataset/ExternalDatasetImpl.java | 2 +-
.../datanode/extdataset/ExternalReplica.java | 6 +
.../extdataset/ExternalReplicaInPipeline.java | 6 +
.../fsdataset/impl/LazyPersistTestCase.java | 23 +-
.../datanode/fsdataset/impl/TestFsDatasetImpl.java | 46 +-
.../datanode/fsdataset/impl/TestFsVolumeList.java | 2 +-
.../fsdataset/impl/TestScrLazyPersistFiles.java | 7 +-
.../fsdataset/impl/TestSpaceReservation.java | 7 +-
.../web/webhdfs/TestDataNodeUGIProvider.java | 4 +-
.../command/TestDiskBalancerCommand.java | 3 +-
.../apache/hadoop/hdfs/server/mover/TestMover.java | 57 +
.../hadoop/hdfs/server/namenode/FSAclBaseTest.java | 2 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 9 +-
.../hdfs/server/namenode/FSXAttrBaseTest.java | 2 +-
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 36 +-
.../hdfs/server/namenode/TestAclConfigFlag.java | 6 +-
.../server/namenode/TestAclTransformation.java | 9 +-
.../hdfs/server/namenode/TestAddStripedBlocks.java | 4 +-
.../hdfs/server/namenode/TestAuditLogger.java | 2 +-
.../hdfs/server/namenode/TestBackupNode.java | 4 +-
.../TestBlockPlacementPolicyRackFaultTolerant.java | 4 +-
.../hdfs/server/namenode/TestCheckpoint.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../hadoop/hdfs/server/namenode/TestEditLog.java | 7 +-
.../hdfs/server/namenode/TestEditLogRace.java | 66 +-
.../hdfs/server/namenode/TestFSDirectory.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFSImage.java | 4 +-
.../hdfs/server/namenode/TestFSImageWithAcl.java | 4 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 5 +-
.../namenode/TestINodeAttributeProvider.java | 4 +-
.../hdfs/server/namenode/TestLeaseManager.java | 3 +-
.../hdfs/server/namenode/TestListOpenFiles.java | 31 +
.../namenode/TestNNStorageRetentionManager.java | 15 +-
.../hdfs/server/namenode/TestNameNodeRecovery.java | 7 +-
.../namenode/TestReconstructStripedBlocks.java | 2 +-
.../hadoop/hdfs/server/namenode/ha/HATestUtil.java | 12 +
.../server/namenode/ha/TestBootstrapStandby.java | 2 +-
.../namenode/ha/TestConsistentReadsObserver.java | 54 +
.../hdfs/server/namenode/ha/TestDNFencing.java | 9 +-
.../namenode/ha/TestEditLogsDuringFailover.java | 3 +-
.../hdfs/server/namenode/ha/TestHASafeMode.java | 7 +-
.../hdfs/server/namenode/ha/TestObserverNode.java | 118 +
.../server/namenode/ha/TestPipelinesFailover.java | 2 +-
.../server/namenode/ha/TestStandbyCheckpoints.java | 38 +-
.../namenode/ha/TestStandbyInProgressTail.java | 9 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 4 +-
.../snapshot/TestFileWithSnapshotFeature.java | 7 +-
.../namenode/snapshot/TestNestedSnapshots.java | 13 +-
.../namenode/snapshot/TestSnapshotDiffReport.java | 75 +-
.../org/apache/hadoop/hdfs/tools/TestDFSAdmin.java | 34 +-
...stViewFileSystemOverloadSchemeWithDFSAdmin.java | 10 +-
...ViewFileSystemOverloadSchemeWithFSCommands.java | 4 +-
.../offlineImageViewer/TestOfflineImageViewer.java | 98 +-
.../TestOfflineImageViewerForAcl.java | 11 +-
.../apache/hadoop/hdfs/util/FoldedTreeSetTest.java | 644 -
.../org/apache/hadoop/hdfs/web/TestJsonUtil.java | 3 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 35 +-
.../hadoop/security/TestRefreshUserMappings.java | 2 +-
.../src/test/resources/testErasureCodingConf.xml | 4 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 33 +-
.../hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 +-
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 7 +-
.../apache/hadoop/mapreduce/v2/app/MockJobs.java | 2 +-
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 30 +
.../mapreduce/v2/app/webapp/TestAMWebServices.java | 2 +-
.../hadoop-mapreduce-client-core/pom.xml | 5 -
.../mapreduce/counters/CounterGroupFactory.java | 2 +-
.../mapreduce/lib/input/FileInputFormat.java | 5 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 8 +-
.../hadoop/mapreduce/util/CountersStrings.java | 3 +-
.../apache/hadoop/mapred/TestFileInputFormat.java | 3 +-
.../java/org/apache/hadoop/mapreduce/TestJob.java | 2 +-
.../hadoop/mapreduce/TestJobMonitorAndPrint.java | 13 +-
.../mapreduce/lib/input/TestFileInputFormat.java | 8 +-
.../apache/hadoop/mapreduce/v2/hs/UnparsedJob.java | 2 +-
.../hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java | 1 +
.../v2/hs/webapp/TestHsWebServicesLogs.java | 2 +-
.../hadoop/mapred/TestMRTimelineEventHandling.java | 3 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 16 -
.../src/main/native/src/lib/primitives.h | 4 +
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 2 +-
.../mapred/uploader/TestFrameworkUploader.java | 4 +-
hadoop-maven-plugins/pom.xml | 4 +-
.../maven/plugin/resourcegz/ResourceGzMojo.java | 4 +-
hadoop-project-dist/pom.xml | 2 +-
hadoop-project/pom.xml | 15 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 12 +
.../fs/s3a/CredentialInitializationException.java | 9 +
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 77 +-
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 38 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 131 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 110 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1412 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 53 +-
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 74 +-
.../apache/hadoop/fs/s3a/S3ALocatedFileStatus.java | 1 +
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 30 +-
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 5 +
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 72 +-
.../hadoop/fs/s3a/UnknownStoreException.java | 24 +-
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 290 +-
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 18 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 296 +
.../apache/hadoop/fs/s3a/api}/package-info.java | 19 +-
.../fs/s3a/audit/AWSAuditEventCallbacks.java | 160 +
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 296 +
.../AuditFailureException.java} | 29 +-
.../hadoop/fs/s3a/audit/AuditIntegration.java | 137 +
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 92 +
.../apache/hadoop/fs/s3a/audit/AuditSpanS3A.java | 16 +-
.../hadoop/fs/s3a/audit/OperationAuditor.java | 73 +
.../fs/s3a/audit/OperationAuditorOptions.java | 74 +
.../hadoop/fs/s3a/audit/S3AAuditConstants.java | 104 +
.../apache/hadoop/fs/s3a/audit/S3LogParser.java | 309 +
.../org/apache/hadoop/fs/s3a/audit/S3LogVerbs.java | 51 +
.../fs/s3a/audit/impl/AbstractAuditSpanImpl.java | 99 +
.../s3a/audit/impl/AbstractOperationAuditor.java | 121 +
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 773 +
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 432 +
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 185 +
.../hadoop/fs/s3a/audit/impl/NoopAuditor.java | 99 +
.../apache/hadoop/fs/s3a/audit/impl/NoopSpan.java | 109 +
.../s3a/audit/impl/S3AInternalAuditConstants.java | 29 +-
.../hadoop/fs/s3a/audit/impl}/package-info.java | 13 +-
.../apache/hadoop/fs/s3a/audit/package-info.java | 41 +
.../fs/s3a/auth/AssumedRoleCredentialProvider.java | 2 +-
.../apache/hadoop/fs/s3a/auth/RolePolicies.java | 2 +-
.../hadoop/fs/s3a/commit/AbstractS3ACommitter.java | 70 +-
.../hadoop/fs/s3a/commit/CommitOperations.java | 31 +-
.../apache/hadoop/fs/s3a/commit/CommitUtils.java | 24 +
.../fs/s3a/commit/InternalCommitterConstants.java | 6 +
.../fs/s3a/commit/MagicCommitIntegration.java | 15 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 2 +-
.../apache/hadoop/fs/s3a/commit/staging/Paths.java | 4 +-
.../fs/s3a/commit/staging/StagingCommitter.java | 2 +
.../hadoop/fs/s3a/impl/AbstractStoreOperation.java | 40 +-
.../hadoop/fs/s3a/impl/CallableSupplier.java | 52 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 1 +
.../hadoop/fs/s3a/impl/ContextAccessors.java | 21 +-
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 17 +-
.../fs/s3a/impl/ExecutingStoreOperation.java | 40 +-
.../fs/s3a/impl/GetContentSummaryOperation.java | 208 +
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 44 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 15 +
.../fs/s3a/impl/ListingOperationCallbacks.java | 26 +-
.../apache/hadoop/fs/s3a/impl/MkdirOperation.java | 184 +
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 24 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 695 +
.../apache/hadoop/fs/s3a/impl/StoreContext.java | 60 +-
.../hadoop/fs/s3a/impl/StoreContextBuilder.java | 18 +-
.../fs/s3a/s3guard/DumpS3GuardDynamoTable.java | 37 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 6 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 57 +-
.../apache/hadoop/fs/s3a/select/SelectBinding.java | 29 +-
.../impl/ForwardingIOStatisticsStore.java | 186 +
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 9 +-
.../src/site/markdown/tools/hadoop-aws/auditing.md | 389 +
.../tools/hadoop-aws/auditing_architecture.md | 323 +
.../src/site/markdown/tools/hadoop-aws/index.md | 18 +
.../tools/hadoop-aws/troubleshooting_s3a.md | 648 +-
.../tools/hadoop-aws/audit-architecture.png | Bin 0 -> 55701 bytes
.../fs/contract/s3a/ITestS3AContractSeek.java | 2 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 46 +
.../hadoop/fs/s3a/ITestS3AEmptyDirectory.java | 7 +-
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 190 +
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 15 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 23 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 4 +-
.../hadoop/fs/s3a/ITestS3ARemoteFileChanged.java | 2 +-
.../hadoop/fs/s3a/ITestS3GuardEmptyDirs.java | 45 +-
.../hadoop/fs/s3a/ITestS3GuardListConsistency.java | 2 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 44 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 57 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 46 -
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 2 +-
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 11 +-
.../hadoop/fs/s3a/TestS3AExceptionTranslation.java | 8 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 209 +
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 213 +
.../hadoop/fs/s3a/audit/AccessCheckingAuditor.java | 53 +
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 123 +
.../fs/s3a/audit/ITestAuditAccessChecks.java | 201 +
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 135 +
.../fs/s3a/audit/SimpleAWSRequestHandler.java | 54 +
.../hadoop/fs/s3a/audit/TestAuditIntegration.java | 216 +
.../fs/s3a/audit/TestAuditSpanLifecycle.java | 133 +
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 323 +
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 203 +
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 3 +-
.../hadoop/fs/s3a/commit/AbstractCommitITest.java | 12 +-
.../fs/s3a/commit/AbstractITCommitProtocol.java | 3 +-
.../fs/s3a/commit/ITestCommitOperations.java | 5 +-
.../hadoop/fs/s3a/commit/TestMagicCommitPaths.java | 2 +-
.../commit/integration/ITestS3ACommitterMRJob.java | 2 +-
.../fs/s3a/commit/staging/StagingTestBase.java | 2 +-
.../s3a/commit/staging/TestStagingCommitter.java | 2 +-
.../staging/TestStagingPartitionedFileListing.java | 2 +-
.../staging/TestStagingPartitionedTaskCommit.java | 4 +-
.../apache/hadoop/fs/s3a/impl/ITestXAttrCost.java | 3 +-
.../hadoop/fs/s3a/impl/TestHeaderProcessing.java | 27 +-
.../hadoop/fs/s3a/impl/TestNetworkBinding.java | 2 +-
.../fs/s3a/impl/TestPartialDeleteFailures.java | 18 +-
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 185 +
.../fs/s3a/performance/AbstractS3ACostTest.java | 33 +-
.../performance/ITestDirectoryMarkerListing.java | 13 +-
.../s3a/performance/ITestS3AMiscOperationCost.java | 143 +
.../fs/s3a/performance/ITestS3AMkdirCost.java | 185 +
.../fs/s3a/performance/OperationCostValidator.java | 3 +-
.../fs/s3a/s3guard/ITestDynamoDBMetadataStore.java | 2 +-
.../hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java | 2 -
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 47 +-
.../fs/s3a/s3guard/TestPathOrderComparators.java | 12 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 3 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 137 +-
.../statistics/ITestS3AFileSystemStatistic.java | 75 +
.../s3a/test/MinimalListingOperationCallbacks.java | 10 +-
.../hadoop-aws/src/test/resources/core-site.xml | 13 +
.../hadoop-aws/src/test/resources/log4j.properties | 6 +
.../dev-support/testrun-scripts/runtests.sh | 7 +
hadoop-tools/hadoop-azure/pom.xml | 4 +-
.../fs/azure/AzureNativeFileSystemStore.java | 107 +-
.../hadoop/fs/azure/BlockBlobInputStream.java | 45 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 64 +-
.../hadoop/fs/azure/NativeFileSystemStore.java | 4 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 82 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 232 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 264 +-
.../fs/azurebfs/constants/ConfigurationKeys.java | 14 +
.../fs/azurebfs/constants/FSOperationType.java | 60 +
.../constants/FileSystemConfigurations.java | 4 +-
.../hadoop/fs/azurebfs/services/AbfsClient.java | 136 +-
.../fs/azurebfs/services/AbfsHttpOperation.java | 15 +-
.../fs/azurebfs/services/AbfsInputStream.java | 118 +-
.../azurebfs/services/AbfsInputStreamContext.java | 15 +
.../hadoop/fs/azurebfs/services/AbfsLease.java | 32 +-
.../services/AbfsListStatusRemoteIterator.java | 7 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 47 +-
.../fs/azurebfs/services/AbfsRestOperation.java | 19 +-
.../fs/azurebfs/services/ListingSupport.java | 10 +-
.../hadoop/fs/azurebfs/services/ReadBuffer.java | 10 +
.../fs/azurebfs/services/ReadBufferManager.java | 6 +-
.../fs/azurebfs/services/ReadBufferWorker.java | 3 +-
.../apache/hadoop/fs/azurebfs/utils/Listener.java | 17 +-
.../hadoop/fs/azurebfs/utils/TracingContext.java | 185 +
.../fs/azurebfs/utils/TracingHeaderFormat.java | 14 +-
.../hadoop-azure/src/site/markdown/abfs.md | 24 +-
.../hadoop-azure/src/site/markdown/index.md | 11 +
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 56 +
.../fs/azure/ITestOutputStreamSemantics.java | 19 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 62 +-
.../apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 7 +-
.../fs/azurebfs/ITestAbfsDurationTrackers.java | 3 +-
.../fs/azurebfs/ITestAbfsIdentityTransformer.java | 2 +-
.../azurebfs/ITestAbfsInputStreamStatistics.java | 32 +-
.../ITestAbfsListStatusRemoteIterator.java | 25 +-
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 51 +-
.../hadoop/fs/azurebfs/ITestAbfsStatistics.java | 12 +
.../azurebfs/ITestAzureBlobFileSystemAppend.java | 13 +
.../ITestAzureBlobFileSystemAttributes.java | 19 +-
.../ITestAzureBlobFileSystemAuthorization.java | 18 +-
.../ITestAzureBlobFileSystemBackCompat.java | 8 +-
.../ITestAzureBlobFileSystemCheckAccess.java | 6 +-
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 30 +-
.../ITestAzureBlobFileSystemDelegationSAS.java | 6 +-
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 25 +-
.../ITestAzureBlobFileSystemFileStatus.java | 2 +-
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 25 +
.../fs/azurebfs/ITestAzureBlobFileSystemLease.java | 34 +-
.../ITestAzureBlobFileSystemListStatus.java | 5 +
.../fs/azurebfs/ITestAzureBlobFileSystemMkDir.java | 11 +-
.../fs/azurebfs/ITestAzureBlobFileSystemOauth.java | 12 +-
.../ITestAzureBlobFileSystemPermission.java | 4 +-
.../ITestAzureBlobFileSystemRandomRead.java | 7 +-
.../azurebfs/ITestAzureBlobFileSystemRename.java | 13 +-
...zureBlobFileSystemStoreListStatusWithRange.java | 3 +-
.../fs/azurebfs/ITestAzureBlobFilesystemAcl.java | 160 +-
.../fs/azurebfs/ITestCustomerProvidedKey.java | 89 +-
.../fs/azurebfs/ITestFileSystemProperties.java | 32 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 33 +-
.../hadoop/fs/azurebfs/ITestSharedKeyAuth.java | 3 +-
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 10 +-
.../TestAbfsConfigurationFieldsValidation.java | 2 +
.../fs/azurebfs/TestAccountConfiguration.java | 49 +-
.../hadoop/fs/azurebfs/TestTracingContext.java | 201 +
.../contract/ITestAbfsFileSystemContractSeek.java | 238 +
.../fs/azurebfs/services/ITestAbfsInputStream.java | 8 +-
.../services/ITestAbfsInputStreamReadFooter.java | 7 +-
.../ITestAbfsInputStreamSmallFileReads.java | 7 +-
.../fs/azurebfs/services/TestAbfsInputStream.java | 57 +-
.../fs/azurebfs/services/TestAbfsOutputStream.java | 145 +-
.../fs/azurebfs/utils/TracingHeaderValidator.java | 152 +
.../test/resources/azure-auth-keys.xml.template | 3 +-
.../java/org/apache/hadoop/tools/CopyListing.java | 2 +-
.../apache/hadoop/tools/CopyListingFileStatus.java | 2 +-
.../org/apache/hadoop/tools/DistCpContext.java | 2 +-
.../java/org/apache/hadoop/tools/DistCpSync.java | 74 +-
.../org/apache/hadoop/tools/SimpleCopyListing.java | 2 +-
.../apache/hadoop/tools/mapred/CopyCommitter.java | 6 +-
.../org/apache/hadoop/tools/TestDistCpSync.java | 162 +-
.../tools/contract/AbstractContractDistCpTest.java | 56 +
.../hadoop/tools/mapred/TestCopyCommitter.java | 45 +
.../apache/hadoop/tools/util/TestDistCpUtils.java | 11 +-
.../hadoop-dynamometer-blockgen/pom.xml | 2 +-
.../hadoop-dynamometer-infra/pom.xml | 2 +-
.../tools/dynamometer/ApplicationMaster.java | 2 +-
.../apache/hadoop/tools/dynamometer/Client.java | 2 +-
.../tools/dynamometer/TestDynamometerInfra.java | 2 +-
.../hadoop-dynamometer-workload/pom.xml | 5 +
.../workloadgenerator/CreateFileMapper.java | 2 +-
.../workloadgenerator/audit/AuditReplayMapper.java | 2 +-
.../tools/fedbalance/TestDistCpProcedure.java | 2 +-
.../procedure/TestBalanceProcedureScheduler.java | 2 +-
.../hadoop/metrics2/impl/TestKafkaMetrics.java | 2 +-
.../src/main/data/2jobs2min-rumen-jh.json | 2 -
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +-
.../resource/TestPlacementConstraintParser.java | 3 +-
.../distributedshell/ApplicationMaster.java | 7 +-
.../yarn/applications/distributedshell/Client.java | 9 +-
.../distributedshell/Log4jPropertyHelper.java | 13 +-
.../hadoop/yarn/service/webapp/ApiServer.java | 7 +-
.../apache/hadoop/yarn/service/TestApiServer.java | 2 +-
.../yarn/service/client/TestApiServiceClient.java | 3 +-
.../hadoop/yarn/service/utils/ServiceApiUtil.java | 2 +-
.../apache/hadoop/yarn/service/MockServiceAM.java | 2 +-
.../TestDefaultUpgradeComponentsFinder.java | 2 +-
.../yarn/service/TestYarnNativeServices.java | 6 +-
.../yarn/service/client/TestServiceClient.java | 2 +-
.../component/instance/TestComponentInstance.java | 2 +-
.../provider/TestAbstractProviderService.java | 2 +-
.../hadoop/yarn/service/utils/TestFilterUtils.java | 2 +-
.../hadoop/yarn/client/api/impl/NMClientImpl.java | 2 +-
.../org/apache/hadoop/yarn/client/cli/LogsCLI.java | 4 +-
.../hadoop/yarn/client/cli/NodeAttributesCLI.java | 5 +-
.../client/TestNoHaRMFailoverProxyProvider.java | 20 +-
.../yarn/client/TestRMFailoverProxyProvider.java | 25 +-
.../yarn/client/api/impl/TestYarnClientImpl.java | 4 +-
.../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 14 +-
.../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 30 -
.../impl/pb/GetApplicationsRequestPBImpl.java | 28 +-
.../impl/pb/GetClusterNodesRequestPBImpl.java | 6 +-
.../impl/pb/GetNodesToLabelsResponsePBImpl.java | 3 +-
.../pb/ApplicationSubmissionContextPBImpl.java | 30 +-
.../yarn/client/api/impl/TimelineClientImpl.java | 25 +
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 4 +
.../apache/hadoop/yarn/event/EventDispatcher.java | 4 +
.../yarn/logaggregation/AggregatedLogFormat.java | 2 +-
.../hadoop/yarn/logaggregation/LogCLIHelpers.java | 4 +-
.../ifile/IndexedFileAggregatedLogsBlock.java | 5 +-
.../nodelabels/store/op/RemoveClusterLabelOp.java | 2 +-
.../impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java | 3 +-
.../hadoop/yarn/util/ProcfsBasedProcessTree.java | 2 +-
.../org/apache/hadoop/yarn/util/RMHAUtils.java | 3 +-
.../util/resource/DominantResourceCalculator.java | 5 +-
.../apache/hadoop/yarn/webapp/ResponseInfo.java | 2 +-
.../java/org/apache/hadoop/yarn/webapp/WebApp.java | 4 +-
.../apache/hadoop/yarn/webapp/hamlet/Hamlet.java | 30561 -------------------
.../hadoop/yarn/webapp/hamlet/HamletGen.java | 451 -
.../hadoop/yarn/webapp/hamlet/HamletImpl.java | 387 -
.../hadoop/yarn/webapp/hamlet/HamletSpec.java | 3103 --
.../hadoop/yarn/webapp/hamlet2/HamletGen.java | 3 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 26 +-
.../apache/hadoop/yarn/webapp/view/JQueryUI.java | 7 +-
.../hadoop/yarn/webapp/view/TwoColumnLayout.java | 7 +-
.../src/main/resources/yarn-default.xml | 6 +
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 12 +-
.../impl/pb/TestGetApplicationsRequestPBImpl.java | 74 +
.../pb/TestApplicationSubmissionContextPBImpl.java | 74 +
.../yarn/client/api/impl/TestTimelineClient.java | 39 +
.../TestLogAggregationIndexedFileController.java | 8 +-
.../hadoop/yarn/nodelabels/NodeLabelTestBase.java | 2 +-
.../nodelabels/TestCommonNodeLabelsManager.java | 2 +-
.../CustomResourceTypesConfigurationProvider.java | 2 +-
.../yarn/webapp/hamlet/TestParseSelector.java | 57 -
.../webapp/{hamlet => hamlet2}/TestHamlet.java | 84 +-
.../webapp/{hamlet => hamlet2}/TestHamletImpl.java | 75 +-
.../yarn/webapp/hamlet2/TestParseSelector.java | 57 +
.../hadoop-yarn-server-common/pom.xml | 2 +-
.../api/protocolrecords/TestProtocolRecords.java | 3 +-
.../containermanager/ContainerManagerImpl.java | 6 +-
.../application/ApplicationImpl.java | 3 +
.../linux/resources/gpu/GpuResourceAllocator.java | 8 +-
.../logaggregation/AppLogAggregatorImpl.java | 6 +-
.../deviceframework/DeviceMappingManager.java | 10 +-
.../resourceplugin/fpga/FpgaDiscoverer.java | 2 +-
.../resourceplugin/gpu/GpuDiscoverer.java | 10 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 10 +
.../server/nodemanager/webapp/NMWebServices.java | 4 +-
.../yarn/server/nodemanager/webapp/NodePage.java | 2 +-
.../container-executor/impl/container-executor.c | 130 +-
.../src/main/native/container-executor/impl/main.c | 4 +-
.../container-executor/impl/runc/runc_reap.c | 6 +-
.../src/main/native/container-executor/impl/util.c | 10 +-
.../container-executor/impl/utils/docker-util.c | 4 +-
.../container-executor/impl/utils/string-utils.c | 1 +
.../test/test-container-executor.c | 41 +
.../server/nodemanager/TestContainerExecutor.java | 2 +-
.../containermanager/TestAuxServices.java | 2 +-
.../containermanager/TestContainerManager.java | 3 +-
.../TestContainerManagerRecovery.java | 6 +-
.../container/TestResourceMappings.java | 9 +-
.../launcher/TestContainerLaunch.java | 4 +-
.../resources/gpu/TestGpuResourceAllocator.java | 3 +-
.../localizer/TestResourceLocalizationService.java | 3 +-
.../resourceplugin/com/nec/TestNECVEPlugin.java | 2 +-
.../com/nec/TestVEDeviceDiscoverer.java | 3 +-
.../fpga/TestIntelFpgaOpenclPlugin.java | 3 +-
.../resourceplugin/gpu/TestGpuResourcePlugin.java | 2 +-
.../gpu/TestNvidiaDockerV1CommandPlugin.java | 2 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 2 +-
.../metrics/TestNodeManagerMetrics.java | 14 +-
.../TestConfigurationNodeLabelsProvider.java | 4 +-
.../server/resourcemanager/ClusterMetrics.java | 57 +
.../yarn/server/resourcemanager/RMAppManager.java | 26 +-
.../yarn/server/resourcemanager/RMServerUtils.java | 6 +-
.../server/resourcemanager/ResourceManager.java | 62 +-
.../monitor/SchedulingMonitorManager.java | 7 +-
.../capacity/FifoIntraQueuePreemptionPlugin.java | 5 +-
.../MappingRuleValidationContextImpl.java | 2 +-
.../scheduler/AppSchedulingInfo.java | 2 +
.../resourcemanager/scheduler/QueueMetrics.java | 6 +-
.../resourcemanager/scheduler/SchedulerUtils.java | 12 +-
.../scheduler/activities/ActivitiesManager.java | 5 +-
.../scheduler/capacity/AbstractCSQueue.java | 58 +-
.../scheduler/capacity/AutoCreatedLeafQueue.java | 20 +-
.../capacity/AutoCreatedQueueTemplate.java | 14 +-
.../scheduler/capacity/CSQueueUtils.java | 12 +-
.../scheduler/capacity/CapacityScheduler.java | 109 +-
.../CapacitySchedulerAutoQueueHandler.java | 140 -
.../capacity/CapacitySchedulerConfiguration.java | 49 +-
.../capacity/CapacitySchedulerQueueManager.java | 240 +
.../scheduler/capacity/ConfiguredNodeLabels.java | 77 +
.../scheduler/capacity/LeafQueue.java | 38 +-
.../scheduler/capacity/ManagedParentQueue.java | 16 +-
.../scheduler/capacity/ParentQueue.java | 65 +-
.../scheduler/capacity/PlanQueue.java | 40 +-
.../QueueConfigurationAutoRefreshPolicy.java | 2 +-
.../scheduler/capacity/ReservationQueue.java | 2 +-
.../scheduler/capacity/UsersManager.java | 30 +-
.../capacity/conf/ZKConfigurationStore.java | 25 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../processor/PlacementConstraintProcessor.java | 3 +-
.../scheduler/fair/FSSchedulerNode.java | 10 +-
.../scheduler/fair/FairScheduler.java | 14 +-
.../scheduler/fair/FairSchedulerConfiguration.java | 7 +-
.../scheduler/fair/QueueManager.java | 4 -
.../fair/allocation/AllocationFileParser.java | 2 +-
.../fair/converter/QueuePlacementConverter.java | 2 +-
.../webapp/ApplicationsRequestBuilder.java | 2 +-
.../webapp/CapacitySchedulerPage.java | 21 +-
.../webapp/MetricsOverviewTable.java | 8 +
.../server/resourcemanager/webapp/RMAppBlock.java | 2 +-
.../resourcemanager/webapp/RMWebAppFilter.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
.../resourcemanager/webapp/dao/AppAttemptInfo.java | 32 +-
.../dao/AutoQueueTemplatePropertiesInfo.java | 43 +-
.../webapp/dao/CapacitySchedulerInfo.java | 3 +
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 4 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 6 +-
.../webapp/dao/ClusterMetricsInfo.java | 21 +
.../dao/helper/CapacitySchedulerInfoHelper.java | 17 +
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
.../server/resourcemanager/TestAppManager.java | 14 +-
.../resourcemanager/TestClientRMService.java | 6 +-
.../server/resourcemanager/TestClusterMetrics.java | 17 +
.../yarn/server/resourcemanager/TestRMRestart.java | 8 +-
.../applicationsmanager/MockAsm.java | 3 +-
.../applicationsmanager/TestAMRestart.java | 16 +-
...lCapacityPreemptionPolicyIntraQueueWithDRF.java | 83 +
.../nodelabels/TestNodeAttributesManager.java | 2 +-
.../placement/MockQueueHierarchyBuilder.java | 2 +-
.../placement/TestPlacementManager.java | 2 +-
.../placement/csmappingrule/TestMappingRule.java | 2 +-
.../csmappingrule/TestMappingRuleMatchers.java | 2 +-
.../recovery/TestZKRMStateStore.java | 9 +-
.../rmapp/TestRMAppTransitions.java | 8 +-
.../scheduler/TestAbstractYarnScheduler.java | 5 +-
.../scheduler/TestPartitionQueueMetrics.java | 8 +
.../scheduler/TestSchedulerUtils.java | 6 +-
.../capacity/CapacitySchedulerTestBase.java | 2 +-
.../scheduler/capacity/TestApplicationLimits.java | 6 +-
.../capacity/TestAutoCreatedQueueTemplate.java | 11 +
.../scheduler/capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerAmbiguousLeafs.java | 123 +
.../TestCapacitySchedulerAutoQueueCreation.java | 6 +-
.../TestCapacitySchedulerLazyPreemption.java | 2 +-
.../TestCapacitySchedulerMaxParallelApps.java | 3 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 66 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 2 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 32 +-
.../scheduler/capacity/TestLeafQueue.java | 105 +
.../capacity/TestNodeLabelContainerAllocation.java | 2 +-
.../scheduler/capacity/TestParentQueue.java | 14 +-
.../scheduler/capacity/TestReservationQueue.java | 5 +
.../scheduler/capacity/TestUsersManager.java | 109 +
.../scheduler/capacity/TestUtils.java | 6 +-
.../TestWorkPreservingRMRestartForNodeLabel.java | 2 +-
.../constraint/TestAllocationTagsNamespace.java | 2 +-
.../TestPlacementConstraintManagerService.java | 2 +-
.../scheduler/fair/FairSchedulerTestBase.java | 2 +-
.../scheduler/fair/TestAppRunnability.java | 6 +-
.../scheduler/fair/TestFSLeafQueue.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 62 +-
.../scheduler/fair/TestFairSchedulerFairShare.java | 3 +
.../scheduler/fair/TestQueueManager.java | 9 +-
.../fair/allocationfile/AllocationFileQueue.java | 2 +-
.../AllocationFileQueuePlacementPolicy.java | 2 +-
.../AllocationFileQueuePlacementRule.java | 2 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 2 +-
.../fair/converter/TestFSQueueConverter.java | 6 +-
.../converter/TestQueuePlacementConverter.java | 3 +-
.../webapp/ActivitiesTestUtils.java | 2 +-
.../webapp/TestApplicationsRequestBuilder.java | 2 +-
.../resourcemanager/webapp/TestNodesPage.java | 2 +-
.../webapp/TestRMWebServiceAppsNodelabel.java | 2 +-
.../resourcemanager/webapp/TestRMWebServices.java | 2 +-
.../webapp/TestRMWebServicesAppAttempts.java | 40 +
.../webapp/TestRMWebServicesApps.java | 2 +-
.../webapp/TestRMWebServicesCapacitySched.java | 14 +-
...estRMWebServicesCapacitySchedDynamicConfig.java | 57 +-
...RMWebServicesDelegationTokenAuthentication.java | 17 +-
.../TestRMWebServicesForCSWithPartitions.java | 4 +-
.../FairSchedulerJsonVerifications.java | 2 +-
.../FairSchedulerXmlVerifications.java | 2 +-
.../TestRMWebServicesFairScheduler.java | 2 +-
...ebServicesFairSchedulerCustomResourceTypes.java | 5 +-
.../helper/ResourceRequestsJsonVerifications.java | 5 +-
.../helper/ResourceRequestsXmlVerifications.java | 7 +-
.../server/timeline/EntityGroupPlugInForTest.java | 2 +-
.../cosmosdb/CosmosDBDocumentStoreReader.java | 2 +-
.../TestTimelineReaderWebServicesHBaseStorage.java | 56 +-
.../pom.xml | 6 +
.../reader/TimelineReaderWebServicesUtils.java | 4 +-
.../collector/TestTimelineCollector.java | 2 +-
.../reader/TestTimelineReaderWebServicesUtils.java | 3 +-
.../yarn/server/webproxy/TestProxyUriUtils.java | 3 +-
.../src/site/markdown/FairScheduler.md | 4 +-
.../src/site/markdown/SecureContainer.md | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 2 +-
.../main/webapp/app/templates/yarn-app/info.hbs | 2 +-
.../hadoop-yarn-ui/src/main/webapp/bower.json | 4 +-
.../src/main/webapp/config/environment.js | 4 +
.../src/main/webapp/ember-cli-build.js | 2 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 3 +
.../hadoop-yarn-ui/src/main/webapp/yarn.lock | 483 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 +
pom.xml | 32 +-
1021 files changed, 43708 insertions(+), 43519 deletions(-)
create mode 100644 dev-support/docker/Dockerfile_centos_7
create mode 100644 dev-support/docker/Dockerfile_centos_8
create mode 100644 dev-support/docker/Dockerfile_debian_10
create mode 100644 dev-support/docker/README.md
create mode 100644 dev-support/docker/pkg-resolver/check_platform.py
create mode 100644 dev-support/docker/pkg-resolver/install-boost.sh
create mode 100644 dev-support/docker/pkg-resolver/install-cmake.sh
create mode 100644 dev-support/docker/pkg-resolver/install-common-pkgs.sh
create mode 100644 dev-support/docker/pkg-resolver/install-epel.sh
create mode 100644 dev-support/docker/pkg-resolver/install-hadolint.sh
create mode 100644 dev-support/docker/pkg-resolver/install-intel-isa-l.sh
create mode 100644 dev-support/docker/pkg-resolver/install-maven.sh
create mode 100644 dev-support/docker/pkg-resolver/install-protobuf.sh
create mode 100644 dev-support/docker/pkg-resolver/install-spotbugs.sh
create mode 100644 dev-support/docker/pkg-resolver/install-yasm.sh
create mode 100644 dev-support/docker/pkg-resolver/install-zstandard.sh
create mode 100644 dev-support/docker/pkg-resolver/packages.json
create mode 100644 dev-support/docker/pkg-resolver/platforms.json
create mode 100644 dev-support/docker/pkg-resolver/resolve.py
create mode 100644 dev-support/jenkins.sh
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java
copy hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditStatisticNames.java (56%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit}/package-info.java (77%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsStore.java
copy {hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store}/LogExactlyOnce.java (81%)
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/LogExactlyOnce.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/ActiveThreadSpanSource.java (57%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditEntryPoint.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditSpan.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/UnknownStoreException.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditSpanSource.java (55%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/HttpReferrerAuditHeader.java
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit}/package-info.java (72%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Lists.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java
create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.1/CHANGELOG.3.3.1.md
create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.1/RELEASENOTES.3.3.1.md
create mode 100644 hadoop-common-project/hadoop-common/src/test/arm-java/org/apache/hadoop/ipc/protobuf/TestProtosLegacy.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/arm-java/org/apache/hadoop/ipc/protobuf/TestRpcServiceProtosLegacy.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLists.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPartitionedGSet.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/proto/test_legacy.proto
create mode 100644 hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service_legacy.proto
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterNetworkTopologyServlet.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameBase.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenameInKerberosEnv.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederationRenamePermission.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNetworkTopologyServlet.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterTrash.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.1.xml
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/MoverMetrics.java
copy hadoop-hdfs-project/{hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h => hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/package-info.java} (65%)
delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/FoldedTreeSet.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedBlock.java
delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/FoldedTreeSetTest.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api}/package-info.java (61%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/LogExactlyOnce.java => audit/AuditFailureException.java} (55%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/package-info.java => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditSpanS3A.java (75%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditorOptions.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3LogParser.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3LogVerbs.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/AbstractAuditSpanImpl.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/AbstractOperationAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopSpan.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/package-info.java => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/S3AInternalAuditConstants.java (56%)
rename {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl}/package-info.java (77%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/package-info.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/ForwardingIOStatisticsStore.java
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md
create mode 100644 hadoop-tools/hadoop-aws/src/site/resources/tools/hadoop-aws/audit-architecture.png
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AInputStreamRetry.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AccessCheckingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSRequestHandler.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/ITestS3AFileSystemStatistic.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FSOperationType.java
copy hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/Listener.java (64%)
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
copy hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/syscall.h => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java (71%)
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/Hamlet.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/HamletGen.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/HamletImpl.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/hamlet/HamletSpec.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet/TestParseSelector.java
rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/{hamlet => hamlet2}/TestHamlet.java (65%)
rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/{hamlet => hamlet2}/TestHamletImpl.java (55%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/hamlet2/TestParseSelector.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfiguredNodeLabels.java
rename hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteSkipTrashParam.java => hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AutoQueueTemplatePropertiesInfo.java (54%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUsersManager.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HDFS-16125. [FGL] Fix the iterator for
PartitionedGSet. Contributed by Xing Lin. (#3197)
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b1e2c07379e7d46686f570220621a5e94d05e795
Author: Xing Lin <xi...@linkedin.com>
AuthorDate: Fri Jul 16 13:04:59 2021 -0700
HDFS-16125. [FGL] Fix the iterator for PartitionedGSet. Contributed by Xing Lin. (#3197)
---
.../java/org/apache/hadoop/util/LatchLock.java | 4 +-
.../org/apache/hadoop/util/PartitionedGSet.java | 35 ++-
.../apache/hadoop/util/TestPartitionedGSet.java | 270 +++++++++++++++++++++
.../hadoop/hdfs/server/namenode/INodeMap.java | 4 +-
4 files changed, 300 insertions(+), 13 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java
index 41e33da..fd98391 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java
@@ -30,7 +30,7 @@ public abstract class LatchLock<C> {
protected abstract boolean isReadTopLocked();
/** @return true topLock is locked for write by any thread */
protected abstract boolean isWriteTopLocked();
- protected abstract void readTopdUnlock();
+ protected abstract void readTopUnlock();
protected abstract void writeTopUnlock();
protected abstract boolean hasReadChildLock();
@@ -46,7 +46,7 @@ public abstract class LatchLock<C> {
// Public APIs to use with the class
public void readLock() {
readChildLock();
- readTopdUnlock();
+ readTopUnlock();
}
public void readUnlock() {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
index 7ebb1b3..f3569cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
@@ -24,7 +24,7 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
-
+import java.util.NoSuchElementException;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
@@ -79,8 +79,7 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
public PartitionedGSet(final int capacity,
final Comparator<? super K> comparator,
- final LatchLock<?> latchLock,
- final E rootKey) {
+ final LatchLock<?> latchLock) {
this.partitions = new TreeMap<K, PartitionEntry>(comparator);
this.latchLock = latchLock;
// addNewPartition(rootKey).put(rootKey);
@@ -275,17 +274,36 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
* modifying other partitions, while iterating through the current one.
*/
private class EntryIterator implements Iterator<E> {
- private final Iterator<K> keyIterator;
+ private Iterator<K> keyIterator;
private Iterator<E> partitionIterator;
public EntryIterator() {
keyIterator = partitions.keySet().iterator();
- K curKey = partitions.firstKey();
- partitionIterator = getPartition(curKey).iterator();
+
+ if (!keyIterator.hasNext()) {
+ partitionIterator = null;
+ return;
+ }
+
+ K firstKey = keyIterator.next();
+ partitionIterator = partitions.get(firstKey).iterator();
}
@Override
public boolean hasNext() {
+
+ // Special case: an iterator was created for an empty PartitionedGSet.
+ // Check whether new partitions have been added since then.
+ if (partitionIterator == null) {
+ if (partitions.size() == 0) {
+ return false;
+ } else {
+ keyIterator = partitions.keySet().iterator();
+ K nextKey = keyIterator.next();
+ partitionIterator = partitions.get(nextKey).iterator();
+ }
+ }
+
while(!partitionIterator.hasNext()) {
if(!keyIterator.hasNext()) {
return false;
@@ -298,9 +316,8 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
@Override
public E next() {
- while(!partitionIterator.hasNext()) {
- K curKey = keyIterator.next();
- partitionIterator = getPartition(curKey).iterator();
+ if (!hasNext()) {
+ throw new NoSuchElementException("No more elements in this set.");
}
return partitionIterator.next();
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPartitionedGSet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPartitionedGSet.java
new file mode 100644
index 0000000..9ae772c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPartitionedGSet.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Testing {@link PartitionedGSet} */
+public class TestPartitionedGSet {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestPartitionedGSet.class);
+ private static final int ELEMENT_NUM = 100;
+
+ /**
+ * Generate positive random numbers for testing. We want to use only positive
+ * numbers because the smallest partition used in testing is 0.
+ *
+ * @param length
+ * number of random numbers to be generated.
+ *
+ * @param randomSeed
+ * seed to be used for random number generator.
+ *
+ * @return
+ * An array of Integers
+ */
+ private static ArrayList<Integer> getRandomList(int length, int randomSeed) {
+ Random random = new Random(randomSeed);
+ ArrayList<Integer> list = new ArrayList<Integer>(length);
+ for (int i = 0; i < length; i++) {
+ list.add(random.nextInt(Integer.MAX_VALUE));
+ }
+ return list;
+ }
+
+ private static class TestElement implements LinkedElement {
+ private final int val;
+ private LinkedElement next;
+
+ TestElement(int val) {
+ this.val = val;
+ this.next = null;
+ }
+
+ public int getVal() {
+ return val;
+ }
+
+ @Override
+ public void setNext(LinkedElement next) {
+ this.next = next;
+ }
+
+ @Override
+ public LinkedElement getNext() {
+ return next;
+ }
+ }
+
+ private static class TestElementComparator implements Comparator<TestElement>
+ {
+ @Override
+ public int compare(TestElement e1, TestElement e2) {
+ if (e1 == null || e2 == null) {
+ throw new NullPointerException("Cannot compare null elements");
+ }
+
+ return e1.getVal() - e2.getVal();
+ }
+ }
+
+ protected ReentrantReadWriteLock topLock =
+ new ReentrantReadWriteLock(false);
+ /**
+ * We are NOT testing any concurrent access to a PartitionedGSet here.
+ */
+ private class NoOpLock extends LatchLock<ReentrantReadWriteLock> {
+ private ReentrantReadWriteLock childLock;
+
+ public NoOpLock() {
+ childLock = new ReentrantReadWriteLock(false);
+ }
+
+ @Override
+ protected boolean isReadTopLocked() {
+ return topLock.getReadLockCount() > 0 || isWriteTopLocked();
+ }
+
+ @Override
+ protected boolean isWriteTopLocked() {
+ return topLock.isWriteLocked();
+ }
+
+ @Override
+ protected void readTopUnlock() {
+ topLock.readLock().unlock();
+ }
+
+ @Override
+ protected void writeTopUnlock() {
+ topLock.writeLock().unlock();
+ }
+
+ @Override
+ protected boolean hasReadChildLock() {
+ return childLock.getReadLockCount() > 0 || hasWriteChildLock();
+ }
+
+ @Override
+ protected void readChildLock() {
+ childLock.readLock().lock();
+ }
+
+ @Override
+ protected void readChildUnlock() {
+ childLock.readLock().unlock();
+ }
+
+ @Override
+ protected boolean hasWriteChildLock() {
+ return childLock.isWriteLockedByCurrentThread();
+ }
+
+ @Override
+ protected void writeChildLock() {
+ childLock.writeLock().lock();
+ }
+
+ @Override
+ protected void writeChildUnlock() {
+ childLock.writeLock().unlock();
+ }
+
+ @Override
+ protected LatchLock<ReentrantReadWriteLock> clone() {
+ return new NoOpLock();
+ }
+ }
+
+ /**
+ * Test iterator for a PartitionedGSet with no partitions.
+ */
+ @Test(timeout=60000)
+ public void testIteratorForNoPartition() {
+ PartitionedGSet<TestElement, TestElement> set =
+ new PartitionedGSet<TestElement, TestElement>(
+ 16, new TestElementComparator(), new NoOpLock());
+
+ topLock.readLock().lock();
+ int count = 0;
+ Iterator<TestElement> iter = set.iterator();
+ while( iter.hasNext() ) {
+ iter.next();
+ count ++;
+ }
+ topLock.readLock().unlock();
+ Assert.assertEquals(0, count);
+ }
+
+ /**
+ * Test iterator for a PartitionedGSet with empty partitions.
+ */
+ @Test(timeout=60000)
+ public void testIteratorForEmptyPartitions() {
+ PartitionedGSet<TestElement, TestElement> set =
+ new PartitionedGSet<TestElement, TestElement>(
+ 16, new TestElementComparator(), new NoOpLock());
+
+ set.addNewPartition(new TestElement(0));
+ set.addNewPartition(new TestElement(1000));
+ set.addNewPartition(new TestElement(2000));
+
+ topLock.readLock().lock();
+ int count = 0;
+ Iterator<TestElement> iter = set.iterator();
+ while( iter.hasNext() ) {
+ iter.next();
+ count ++;
+ }
+ topLock.readLock().unlock();
+ Assert.assertEquals(0, count);
+ }
+
+ /**
+ * Test whether the iterator can return the same number of elements as stored
+ * into the PartitionedGSet.
+ */
+ @Test(timeout=60000)
+ public void testIteratorCountElements() {
+ ArrayList<Integer> list = getRandomList(ELEMENT_NUM, 123);
+ PartitionedGSet<TestElement, TestElement> set =
+ new PartitionedGSet<TestElement, TestElement>(
+ 16, new TestElementComparator(), new NoOpLock());
+
+ set.addNewPartition(new TestElement(0));
+ set.addNewPartition(new TestElement(1000));
+ set.addNewPartition(new TestElement(2000));
+
+ topLock.writeLock().lock();
+ for (Integer i : list) {
+ set.put(new TestElement(i));
+ }
+ topLock.writeLock().unlock();
+
+ topLock.readLock().lock();
+ int count = 0;
+ Iterator<TestElement> iter = set.iterator();
+ while( iter.hasNext() ) {
+ iter.next();
+ count ++;
+ }
+ topLock.readLock().unlock();
+ Assert.assertEquals(ELEMENT_NUM, count);
+ }
+
+ /**
+ * Test iterator when it is created before partitions/elements are
+ * added to the PartitionedGSet.
+ */
+ @Test(timeout=60000)
+ public void testIteratorAddElementsAfterIteratorCreation() {
+ PartitionedGSet<TestElement, TestElement> set =
+ new PartitionedGSet<TestElement, TestElement>(
+ 16, new TestElementComparator(), new NoOpLock());
+
+ // Create the iterator before partitions are added.
+ Iterator<TestElement> iter = set.iterator();
+
+ set.addNewPartition(new TestElement(0));
+ set.addNewPartition(new TestElement(1000));
+ set.addNewPartition(new TestElement(2000));
+
+ // Added one element
+ topLock.writeLock().lock();
+ set.put(new TestElement(2500));
+ topLock.writeLock().unlock();
+
+ topLock.readLock().lock();
+ int count = 0;
+ while( iter.hasNext() ) {
+ iter.next();
+ count ++;
+ }
+ topLock.readLock().unlock();
+ Assert.assertEquals(1, count);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index 3b07dce..a0253b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -121,7 +121,7 @@ public class INodeMap {
}
@Override
- protected void readTopdUnlock() {
+ protected void readTopUnlock() {
namesystem.getFSLock().readUnlock("INodeMap", null, false);
}
@@ -194,7 +194,7 @@ public class INodeMap {
// Compute the map capacity by allocating 1% of total memory
int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
this.map = new PartitionedGSet<>(capacity, new INodeKeyComparator(),
- new INodeMapLock(), rootDir);
+ new INodeMapLock());
// Pre-populate initial empty partitions
PartitionedGSet<INode, INodeWithAdditionalFields> pgs =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: INodeMap with PartitionedGSet and per-partition
locking.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0e7d216134cc0c93b4ed5861430ef79ce0a9c7ab
Author: Konstantin V Shvachko <sh...@apache.org>
AuthorDate: Fri May 7 17:47:37 2021 -0700
INodeMap with PartitionedGSet and per-partition locking.
---
.../java/org/apache/hadoop/util/LatchLock.java | 64 +++++
.../org/apache/hadoop/util/PartitionedGSet.java | 263 +++++++++++++++++++++
.../hadoop/hdfs/server/namenode/FSDirMkdirOp.java | 92 ++++++-
.../hadoop/hdfs/server/namenode/FSDirectory.java | 2 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 29 ++-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 9 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 96 +++++++-
.../hadoop/hdfs/server/namenode/INodeMap.java | 148 ++++++++++--
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +
.../hadoop/hdfs/server/namenode/TestINodeFile.java | 39 ++-
10 files changed, 682 insertions(+), 62 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java
new file mode 100644
index 0000000..41e33da
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LatchLock.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+/**
+ * LatchLock controls two hierarchical Read/Write locks:
+ * the topLock and the childLock.
+ * Typically an operation starts with the topLock already acquired.
+ * To acquire child lock LatchLock will
+ * first acquire the childLock, and then release the topLock.
+ */
+public abstract class LatchLock<C> {
+ // Interfaces methods to be defined for subclasses
+ /** @return true topLock is locked for read by any thread */
+ protected abstract boolean isReadTopLocked();
+ /** @return true topLock is locked for write by any thread */
+ protected abstract boolean isWriteTopLocked();
+ protected abstract void readTopdUnlock();
+ protected abstract void writeTopUnlock();
+
+ protected abstract boolean hasReadChildLock();
+ protected abstract void readChildLock();
+ protected abstract void readChildUnlock();
+
+ protected abstract boolean hasWriteChildLock();
+ protected abstract void writeChildLock();
+ protected abstract void writeChildUnlock();
+
+ protected abstract LatchLock<C> clone();
+
+ // Public APIs to use with the class
+ public void readLock() {
+ readChildLock();
+ readTopdUnlock();
+ }
+
+ public void readUnlock() {
+ readChildUnlock();
+ }
+
+ public void writeLock() {
+ writeChildLock();
+ writeTopUnlock();
+ }
+
+ public void writeUnlock() {
+ writeChildUnlock();
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
new file mode 100644
index 0000000..4b0cdc9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
+
+/**
+ * An implementation of {@link GSet}, which splits a collection of elements
+ * into partitions each corresponding to a range of keys.
+ *
+ * This class does not support null element.
+ *
+ * This class is backed up by LatchLock for hierarchical synchronization.
+ *
+ * @param <K> Key type for looking up the elements
+ * @param <E> Element type, which must be
+ * (1) a subclass of K, and
+ * (2) implementing {@link LinkedElement} interface.
+ */
+@InterfaceAudience.Private
+public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
+
+ private static final int DEFAULT_PARTITION_CAPACITY = 2027;
+
+ /**
+ * An ordered map of contiguous segments of elements.
+ * Each key in the map represent the smallest key in the mapped segment,
+ * so that all elements in this segment are >= the mapping key,
+ * but are smaller then the next key in the map.
+ * Elements within a partition do not need to be ordered.
+ */
+ private final NavigableMap<K, PartitionEntry> partitions;
+ private LatchLock<?> latchLock;
+
+ /**
+ * The number of elements in the set.
+ */
+ protected volatile int size;
+
+ /**
+ * A single partition of the {@link PartitionedGSet}.
+ * Consists of a hash table {@link LightWeightGSet} and a lock, which
+ * controls access to this partition independently on the other ones.
+ */
+ private class PartitionEntry extends LightWeightGSet<K, E> {
+ private final LatchLock<?> partLock;
+
+ PartitionEntry(int defaultPartitionCapacity) {
+ super(defaultPartitionCapacity);
+ this.partLock = latchLock.clone();
+ }
+ }
+
+ public PartitionedGSet(final int capacity,
+ final Comparator<? super K> comparator,
+ final LatchLock<?> latchLock,
+ final E rootKey) {
+ this.partitions = new TreeMap<K, PartitionEntry>(comparator);
+ this.latchLock = latchLock;
+ addNewPartition(rootKey).put(rootKey);
+ this.size = 1;
+ }
+
+ /**
+ * Creates new empty partition.
+ * @param key
+ * @return
+ */
+ private PartitionEntry addNewPartition(final K key) {
+ PartitionEntry lastPart = null;
+ if(size > 0)
+ lastPart = partitions.lastEntry().getValue();
+
+ PartitionEntry newPart =
+ new PartitionEntry(DEFAULT_PARTITION_CAPACITY);
+ // assert size == 0 || newPart.partLock.isWriteTopLocked() :
+ // "Must hold write Lock: key = " + key;
+ partitions.put(key, newPart);
+
+ LOG.debug("Total GSet size = {}", size);
+ LOG.debug("Number of partitions = {}", partitions.size());
+ LOG.debug("Previous partition size = {}",
+ lastPart == null ? 0 : lastPart.size());
+
+ return newPart;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ protected PartitionEntry getPartition(final K key) {
+ Entry<K, PartitionEntry> partEntry = partitions.floorEntry(key);
+ if(partEntry == null) {
+ return null;
+ }
+ PartitionEntry part = partEntry.getValue();
+ if(part == null) {
+ throw new IllegalStateException("Null partition for key: " + key);
+ }
+ assert size == 0 || part.partLock.isReadTopLocked() ||
+ part.partLock.hasReadChildLock() : "Must hold read Lock: key = " + key;
+ return part;
+ }
+
+ @Override
+ public boolean contains(final K key) {
+ PartitionEntry part = getPartition(key);
+ if(part == null) {
+ return false;
+ }
+ return part.contains(key);
+ }
+
+ @Override
+ public E get(final K key) {
+ PartitionEntry part = getPartition(key);
+ if(part == null) {
+ return null;
+ }
+ LOG.debug("get key: {}", key);
+ // part.partLock.readLock();
+ return part.get(key);
+ }
+
+ @Override
+ public E put(final E element) {
+ K key = element;
+ PartitionEntry part = getPartition(key);
+ if(part == null) {
+ throw new HadoopIllegalArgumentException("Illegal key: " + key);
+ }
+ assert size == 0 || part.partLock.isWriteTopLocked() ||
+ part.partLock.hasWriteChildLock() :
+ "Must hold write Lock: key = " + key;
+ LOG.debug("put key: {}", key);
+ PartitionEntry newPart = addNewPartitionIfNeeded(part, key);
+ if(newPart != part) {
+ newPart.partLock.writeChildLock();
+ part = newPart;
+ }
+ E result = part.put(element);
+ if(result == null) { // new element
+ size++;
+ }
+ return result;
+ }
+
+ private PartitionEntry addNewPartitionIfNeeded(
+ PartitionEntry curPart, K key) {
+ if(curPart.size() < DEFAULT_PARTITION_CAPACITY * 1.1
+ || curPart.contains(key)) {
+ return curPart;
+ }
+ return addNewPartition(key);
+ }
+
+ @Override
+ public E remove(final K key) {
+ PartitionEntry part = getPartition(key);
+ if(part == null) {
+ return null;
+ }
+ E result = part.remove(key);
+ if(result != null) {
+ size--;
+ }
+ return result;
+ }
+
+ @Override
+ public void clear() {
+ LOG.error("Total GSet size = {}", size);
+ LOG.error("Number of partitions = {}", partitions.size());
+ // assert latchLock.hasWriteTopLock() : "Must hold write topLock";
+ // SHV May need to clear all partitions?
+ partitions.clear();
+ size = 0;
+ }
+
+ @Override
+ public Collection<E> values() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return new EntryIterator();
+ }
+
+ /**
+ * Iterator over the elements in the set.
+ * Iterates first by keys, then inside the partition
+ * corresponding to the key.
+ *
+ * Modifications are tracked by the underlying collections. We allow
+ * modifying other partitions, while iterating through the current one.
+ */
+ private class EntryIterator implements Iterator<E> {
+ private final Iterator<K> keyIterator;
+ private Iterator<E> partitionIterator;
+
+ public EntryIterator() {
+ keyIterator = partitions.keySet().iterator();
+ K curKey = partitions.firstKey();
+ partitionIterator = getPartition(curKey).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(partitionIterator.hasNext()) {
+ return true;
+ }
+ return keyIterator.hasNext();
+ }
+
+ @Override
+ public E next() {
+ if(!partitionIterator.hasNext()) {
+ K curKey = keyIterator.next();
+ partitionIterator = getPartition(curKey).iterator();
+ }
+ return partitionIterator.next();
+ }
+ }
+
+ public void latchWriteLock(K[] keys) {
+ // getPartition(parent).partLock.writeChildLock();
+ LatchLock<?> pLock = null;
+ for(K key : keys) {
+ pLock = getPartition(key).partLock;
+ pLock.writeChildLock();
+ }
+ assert pLock != null : "pLock is null";
+ pLock.writeTopUnlock();
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index da324fb..c8c6277 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -69,18 +69,18 @@ class FSDirMkdirOp {
// create multiple inodes.
fsn.checkFsObjectLimit();
- // Ensure that the user can traversal the path by adding implicit
- // u+wx permission to all ancestor directories.
- INodesInPath existing =
- createParentDirectories(fsd, iip, permissions, false);
- if (existing != null) {
- existing = createSingleDirectory(
- fsd, existing, iip.getLastLocalName(), permissions);
+ // create all missing directories along the path,
+ // but don't add them to the INodeMap yet
+ permissions = addImplicitUwx(permissions, permissions); // SHV !!!
+ INode[] missing = createPathDirectories(fsd, iip, permissions);
+ iip = iip.getExistingINodes();
+ // switch the locks
+ fsd.getINodeMap().latchWriteLock(iip, missing);
+ // Add missing inodes to the INodeMap
+ for(INode dir : missing) {
+ iip = addSingleDirectory(fsd, iip, dir, permissions);
+ assert iip != null : "iip should not be null";
}
- if (existing == null) {
- throw new IOException("Failed to create directory: " + src);
- }
- iip = existing;
}
return fsd.getAuditFileInfo(iip);
} finally {
@@ -132,6 +132,7 @@ class FSDirMkdirOp {
if (missing == 0) { // full path exists, return parents.
existing = iip.getParentINodesInPath();
} else if (missing > 1) { // need to create at least one ancestor dir.
+ FSNamesystem.LOG.error("missing = " + missing);
// Ensure that the user can traversal the path by adding implicit
// u+wx permission to all ancestor directories.
PermissionStatus basePerm = inheritPerms
@@ -143,6 +144,13 @@ class FSDirMkdirOp {
for (int i = existing.length(); existing != null && i <= last; i++) {
byte[] component = iip.getPathComponent(i);
existing = createSingleDirectory(fsd, existing, component, perm);
+ if(existing == null) {
+ FSNamesystem.LOG.error("unprotectedMkdir returned null for "
+ + iip.getPath() + " for " + new String(component) + " i = " + i);
+ // Somebody already created the parent. Recalculate existing
+ existing = INodesInPath.resolve(fsd.getRoot(), iip.getPathComponents());
+ i = existing.length() - 1;
+ }
}
}
return existing;
@@ -228,5 +236,67 @@ class FSDirMkdirOp {
}
return iip;
}
+
+ private static INode createDirectoryINode(FSDirectory fsd,
+ INodesInPath parent, byte[] name, PermissionStatus permission)
+ throws FileAlreadyExistsException {
+ assert fsd.hasReadLock();
+ assert parent.getLastINode() != null;
+ if (!parent.getLastINode().isDirectory()) {
+ throw new FileAlreadyExistsException("Parent path is not a directory: " +
+ parent.getPath() + " " + DFSUtil.bytes2String(name));
+ }
+ final INodeDirectory dir = new INodeDirectory(
+ fsd.allocateNewInodeId(), name, permission, now());
+ return dir;
+ }
+
+ private static INode[] createPathDirectories(FSDirectory fsd,
+ INodesInPath iip, PermissionStatus perm)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ INodesInPath existing = iip.getExistingINodes();
+ assert existing != null : "existing should not be null";
+ int numMissing = iip.length() - existing.length();
+ if (numMissing == 0) { // full path exists
+ return new INode[0];
+ }
+
+ // create the missing directories along the path
+ INode[] missing = new INode[numMissing];
+ final int last = iip.length();
+ for (int i = existing.length(); i < last; i++) {
+ byte[] component = iip.getPathComponent(i);
+ missing[i - existing.length()] =
+ createDirectoryINode(fsd, existing, component, perm);
+ }
+ return missing;
+ }
+
+ private static INodesInPath addSingleDirectory(FSDirectory fsd,
+ INodesInPath existing, INode dir, PermissionStatus perm)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ INodesInPath iip = fsd.addLastINode(existing, dir, perm.getPermission(), true);
+ if (iip == null) {
+ FSNamesystem.LOG.debug("somebody already created {} on path {}", dir, existing.getPath());
+ final INodeDirectory parent = existing.getLastINode().asDirectory();
+ dir = parent.getChild(dir.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID);
+ return INodesInPath.append(existing, dir, dir.getLocalNameBytes());
+ }
+ existing = iip;
+ assert dir.equals(existing.getLastINode()) : "dir is not the last INode";
+
+ // Directory creation also count towards FilesCreated
+ // to match count of FilesDeleted metric.
+ NameNode.getNameNodeMetrics().incrFilesCreated();
+
+ assert dir.getPermissionStatus().getGroupName() != null :
+ "GroupName is null for " + existing.getPath();
+ String cur = existing.getPath();
+ fsd.getEditLog().logMkDir(cur, dir);
+ NameNode.stateChangeLog.debug("mkdirs: created directory {}", cur);
+ return existing;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 497aa84..b5e68a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -317,7 +317,7 @@ public class FSDirectory implements Closeable {
FSDirectory(FSNamesystem ns, Configuration conf) throws IOException {
this.inodeId = new INodeId();
rootDir = createRoot(ns);
- inodeMap = INodeMap.newInstance(rootDir);
+ inodeMap = INodeMap.newInstance(rootDir, ns);
this.isPermissionEnabled = conf.getBoolean(
DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,
DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index f7749ce..1305438 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -177,18 +177,23 @@ public class FSImage implements Closeable {
void format(FSNamesystem fsn, String clusterId, boolean force)
throws IOException {
- long fileCount = fsn.getFilesTotal();
- // Expect 1 file, which is the root inode
- Preconditions.checkState(fileCount == 1,
- "FSImage.format should be called with an uninitialized namesystem, has " +
- fileCount + " files");
- NamespaceInfo ns = NNStorage.newNamespaceInfo();
- LOG.info("Allocated new BlockPoolId: " + ns.getBlockPoolID());
- ns.clusterID = clusterId;
-
- storage.format(ns);
- editLog.formatNonFileJournals(ns, force);
- saveFSImageInAllDirs(fsn, 0);
+ fsn.readLock();
+ try {
+ long fileCount = fsn.getFilesTotal();
+ // Expect 1 file, which is the root inode
+ Preconditions.checkState(fileCount == 1,
+ "FSImage.format should be called with an uninitialized namesystem, has " +
+ fileCount + " files");
+ NamespaceInfo ns = NNStorage.newNamespaceInfo();
+ LOG.info("Allocated new BlockPoolId: " + ns.getBlockPoolID());
+ ns.clusterID = clusterId;
+
+ storage.format(ns);
+ editLog.formatNonFileJournals(ns, force);
+ saveFSImageInAllDirs(fsn, 0);
+ } finally {
+ fsn.readUnlock();
+ }
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7ccaae9..a8fb490 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1753,7 +1753,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public void readUnlock(String opName,
Supplier<String> lockReportInfoSupplier) {
- this.fsLock.readUnlock(opName, lockReportInfoSupplier);
+ this.fsLock.readUnlock(opName, lockReportInfoSupplier, true);
}
@Override
@@ -1786,7 +1786,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
@Override
public boolean hasWriteLock() {
- return this.fsLock.isWriteLockedByCurrentThread();
+ return this.fsLock.isWriteLockedByCurrentThread() ||
+ fsLock.haswWriteChildLock();
}
@Override
public boolean hasReadLock() {
@@ -1801,6 +1802,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return this.fsLock.getWriteHoldCount();
}
+ public FSNamesystemLock getFSLock() {
+ return this.fsLock;
+ }
+
/** Lock the checkpoint lock */
public void cpLock() {
this.cpLock.lock();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index b4f479f..f53f10d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -29,6 +32,7 @@ import java.util.function.Supplier;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.namenode.INodeMap.INodeMapLock;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.log.LogThrottlingHelper;
import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
@@ -129,6 +133,32 @@ class FSNamesystemLock {
private static final String OVERALL_METRIC_NAME = "Overall";
+ private final ThreadLocal<Collection<INodeMapLock>> partitionLocks =
+ new ThreadLocal<Collection<INodeMapLock>>() {
+ @Override
+ public Collection<INodeMapLock> initialValue() {
+ return new ArrayList<INodeMapLock>();
+ }
+ };
+
+ void addChildLock(INodeMapLock lock) {
+ partitionLocks.get().add(lock);
+ }
+
+ boolean removeChildLock(INodeMapLock lock) {
+ return partitionLocks.get().remove(lock);
+ }
+
+ boolean haswWriteChildLock() {
+ Iterator<INodeMapLock> iter = partitionLocks.get().iterator();
+ // FSNamesystem.LOG.debug("partitionLocks.size = {}", partitionLocks.get().size());
+ while(iter.hasNext()) {
+ if(iter.next().hasWriteChildLock())
+ return true;
+ }
+ return false;
+ }
+
FSNamesystemLock(Configuration conf,
MutableRatesWithAggregation detailedHoldTimeMetrics) {
this(conf, detailedHoldTimeMetrics, new Timer());
@@ -180,11 +210,29 @@ class FSNamesystemLock {
public void readUnlock(String opName,
Supplier<String> lockReportInfoSupplier) {
+ readUnlock(opName, lockReportInfoSupplier, true);
+ }
+
+ public void readUnlock(String opName,
+ Supplier<String> lockReportInfoSupplier,
+ boolean unlockChildren) {
final boolean needReport = coarseLock.getReadHoldCount() == 1;
final long readLockIntervalNanos =
timer.monotonicNowNanos() - readLockHeldTimeStampNanos.get();
final long currentTimeMs = timer.now();
- coarseLock.readLock().unlock();
+
+ if(getReadHoldCount() > 0) { // Current thread holds the lock
+ // Unlock the top FSNamesystemLock
+ coarseLock.readLock().unlock();
+ }
+
+ if(unlockChildren) { // Also unlock and remove children locks
+ Iterator<INodeMapLock> iter = partitionLocks.get().iterator();
+ while(iter.hasNext()) {
+ iter.next().readChildUnlock();
+ iter.remove();
+ }
+ }
if (needReport) {
addMetric(opName, readLockIntervalNanos, false);
@@ -252,7 +300,7 @@ class FSNamesystemLock {
* FSNamesystemLock#writeUnlock(String, boolean, Supplier)}
*/
public void writeUnlock() {
- writeUnlock(OP_NAME_OTHER, false, null);
+ writeUnlock(OP_NAME_OTHER, false, null, true);
}
/**
@@ -262,7 +310,7 @@ class FSNamesystemLock {
* @param opName Operation name.
*/
public void writeUnlock(String opName) {
- writeUnlock(opName, false, null);
+ writeUnlock(opName, false, null, true);
}
/**
@@ -274,7 +322,7 @@ class FSNamesystemLock {
*/
public void writeUnlock(String opName,
Supplier<String> lockReportInfoSupplier) {
- writeUnlock(opName, false, lockReportInfoSupplier);
+ writeUnlock(opName, false, lockReportInfoSupplier, true);
}
/**
@@ -286,7 +334,7 @@ class FSNamesystemLock {
* for long time will be logged in logs and metrics.
*/
public void writeUnlock(String opName, boolean suppressWriteLockReport) {
- writeUnlock(opName, suppressWriteLockReport, null);
+ writeUnlock(opName, suppressWriteLockReport, null, true);
}
/**
@@ -297,8 +345,9 @@ class FSNamesystemLock {
* for long time will be logged in logs and metrics.
* @param lockReportInfoSupplier The info shown in the lock report
*/
- private void writeUnlock(String opName, boolean suppressWriteLockReport,
- Supplier<String> lockReportInfoSupplier) {
+ public void writeUnlock(String opName, boolean suppressWriteLockReport,
+ Supplier<String> lockReportInfoSupplier,
+ boolean unlockChildren) {
final boolean needReport = !suppressWriteLockReport && coarseLock
.getWriteHoldCount() == 1 && coarseLock.isWriteLockedByCurrentThread();
final long writeLockIntervalNanos =
@@ -329,7 +378,18 @@ class FSNamesystemLock {
longestWriteLockHeldInfo = new LockHeldInfo();
}
- coarseLock.writeLock().unlock();
+ if(this.isWriteLockedByCurrentThread()) { // Current thread holds the lock
+ // Unlock the top FSNamesystemLock
+ coarseLock.writeLock().unlock();
+ }
+
+ if(unlockChildren) { // Unlock and remove children locks
+ Iterator<INodeMapLock> iter = partitionLocks.get().iterator();
+ while(iter.hasNext()) {
+ iter.next().writeChildUnlock();
+ iter.remove();
+ }
+ }
if (needReport) {
addMetric(opName, writeLockIntervalNanos, true);
@@ -355,7 +415,25 @@ class FSNamesystemLock {
public int getWriteHoldCount() {
return coarseLock.getWriteHoldCount();
}
-
+
+ /**
+ * Queries if the write lock is held by any thread.
+ * @return {@code true} if any thread holds the write lock and
+ * {@code false} otherwise
+ */
+ public boolean isReadLocked() {
+ return coarseLock.getReadLockCount() > 0 || isWriteLocked();
+ }
+
+ /**
+ * Queries if the write lock is held by any thread.
+ * @return {@code true} if any thread holds the write lock and
+ * {@code false} otherwise
+ */
+ public boolean isWriteLocked() {
+ return coarseLock.isWriteLocked();
+ }
+
public boolean isWriteLockedByCurrentThread() {
return coarseLock.isWriteLockedByCurrentThread();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index f35949f..88c3233 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -17,44 +17,139 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Comparator;
import java.util.Iterator;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.util.GSet;
+import org.apache.hadoop.util.LatchLock;
import org.apache.hadoop.util.LightWeightGSet;
-
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.PartitionedGSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Storing all the {@link INode}s and maintaining the mapping between INode ID
* and INode.
*/
public class INodeMap {
-
- static INodeMap newInstance(INodeDirectory rootDir) {
- // Compute the map capacity by allocating 1% of total memory
- int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
- GSet<INode, INodeWithAdditionalFields> map =
- new LightWeightGSet<>(capacity);
- map.put(rootDir);
- return new INodeMap(map);
+ public static class INodeIdComparator implements Comparator<INode> {
+ @Override
+ public int compare(INode i1, INode i2) {
+ if (i1 == null || i2 == null) {
+ throw new NullPointerException("Cannot compare null INodesl");
+ }
+ long id1 = i1.getId();
+ long id2 = i2.getId();
+ return id1 < id2 ? -1 : id1 == id2 ? 0 : 1;
+ }
+ }
+
+ public class INodeMapLock extends LatchLock<ReentrantReadWriteLock> {
+ Logger LOG = LoggerFactory.getLogger(INodeMapLock.class);
+
+ private ReentrantReadWriteLock childLock;
+
+ INodeMapLock() {
+ this(null);
+ }
+
+ private INodeMapLock(ReentrantReadWriteLock childLock) {
+ assert namesystem != null : "namesystem is null";
+ this.childLock = childLock;
+ }
+
+ @Override
+ protected boolean isReadTopLocked() {
+ return namesystem.getFSLock().isReadLocked();
+ }
+
+ @Override
+ protected boolean isWriteTopLocked() {
+ return namesystem.getFSLock().isWriteLocked();
+ }
+
+ @Override
+ protected void readTopdUnlock() {
+ namesystem.getFSLock().readUnlock("INodeMap", null, false);
+ }
+
+ @Override
+ protected void writeTopUnlock() {
+ namesystem.getFSLock().writeUnlock("INodeMap", false, null, false);
+ }
+
+ @Override
+ protected boolean hasReadChildLock() {
+ return this.childLock.getReadHoldCount() > 0 || hasWriteChildLock();
+ }
+
+ @Override
+ protected void readChildLock() {
+ // LOG.info("readChildLock: thread = {}, {}", Thread.currentThread().getId(), Thread.currentThread().getName());
+ this.childLock.readLock().lock();
+ namesystem.getFSLock().addChildLock(this);
+ // LOG.info("readChildLock: done");
+ }
+
+ @Override
+ protected void readChildUnlock() {
+ // LOG.info("readChildUnlock: thread = {}, {}", Thread.currentThread().getId(), Thread.currentThread().getName());
+ this.childLock.readLock().unlock();
+ // LOG.info("readChildUnlock: done");
+ }
+
+ @Override
+ protected boolean hasWriteChildLock() {
+ return this.childLock.isWriteLockedByCurrentThread();
+ }
+
+ @Override
+ protected void writeChildLock() {
+ // LOG.info("writeChildLock: thread = {}, {}", Thread.currentThread().getId(), Thread.currentThread().getName());
+ this.childLock.writeLock().lock();
+ namesystem.getFSLock().addChildLock(this);
+ // LOG.info("writeChildLock: done");
+ }
+
+ @Override
+ protected void writeChildUnlock() {
+ // LOG.info("writeChildUnlock: thread = {}, {}", Thread.currentThread().getId(), Thread.currentThread().getName());
+ this.childLock.writeLock().unlock();
+ // LOG.info("writeChildUnlock: done");
+ }
+
+ @Override
+ protected LatchLock<ReentrantReadWriteLock> clone() {
+ return new INodeMapLock(new ReentrantReadWriteLock(false)); // not fair
+ }
+ }
+
+ static INodeMap newInstance(INodeDirectory rootDir,
+ FSNamesystem ns) {
+ return new INodeMap(rootDir, ns);
}
/** Synchronized by external lock. */
private final GSet<INode, INodeWithAdditionalFields> map;
-
+ private FSNamesystem namesystem;
+
public Iterator<INodeWithAdditionalFields> getMapIterator() {
return map.iterator();
}
- private INodeMap(GSet<INode, INodeWithAdditionalFields> map) {
- Preconditions.checkArgument(map != null);
- this.map = map;
+ private INodeMap(INodeDirectory rootDir, FSNamesystem ns) {
+ this.namesystem = ns;
+ // Compute the map capacity by allocating 1% of total memory
+ int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
+ this.map = new PartitionedGSet<>(capacity, new INodeIdComparator(),
+ new INodeMapLock(), rootDir);
}
-
+
/**
* Add an {@link INode} into the {@link INode} map. Replace the old value if
* necessary.
@@ -138,4 +233,27 @@ public class INodeMap {
public void clear() {
map.clear();
}
+
+ public void latchWriteLock(INodesInPath iip, INode[] missing) {
+ assert namesystem.hasReadLock() : "must have namesysem lock";
+ assert iip.length() > 0 : "INodesInPath has 0 length";
+ if(!(map instanceof PartitionedGSet)) {
+ return;
+ }
+ // Locks partitions along the path starting from the first existing parent
+ // Locking is in the hierarchical order
+ INode[] allINodes = new INode[Math.min(1, iip.length()) + missing.length];
+ allINodes[0] = iip.getLastINode();
+ System.arraycopy(missing, 0, allINodes, 1, missing.length);
+ /*
+ // Locks all the partitions along the path in the hierarchical order
+ INode[] allINodes = new INode[iip.length() + missing.length];
+ INode[] existing = iip.getINodesArray();
+ System.arraycopy(existing, 0, allINodes, 0, existing.length);
+ System.arraycopy(missing, 0, allINodes, existing.length, missing.length);
+ */
+
+ ((PartitionedGSet<INode, INodeWithAdditionalFields>)
+ map).latchWriteLock(allINodes);
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index d813375..4982119 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -186,6 +186,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.Whitebox;
+import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
@@ -1997,6 +1998,7 @@ public class DFSTestUtil {
GenericTestUtils.setLogLevel(NameNode.LOG, level);
GenericTestUtils.setLogLevel(NameNode.stateChangeLog, level);
GenericTestUtils.setLogLevel(NameNode.blockStateChangeLog, level);
+ GenericTestUtils.setLogLevel(GSet.LOG, level);
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index b32f8fe..0b21032 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -1017,23 +1017,38 @@ public class TestINodeFile {
final Path dir = new Path("/dir");
hdfs.mkdirs(dir);
- INodeDirectory dirNode = getDir(fsdir, dir);
- INode dirNodeFromNode = fsdir.getInode(dirNode.getId());
- assertSame(dirNode, dirNodeFromNode);
+ cluster.getNamesystem().readLock();
+ try {
+ INodeDirectory dirNode = getDir(fsdir, dir);
+ INode dirNodeFromNode = fsdir.getInode(dirNode.getId());
+ assertSame(dirNode, dirNodeFromNode);
+ } finally {
+ cluster.getNamesystem().readUnlock();
+ }
// set quota to dir, which leads to node replacement
hdfs.setQuota(dir, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
- dirNode = getDir(fsdir, dir);
- assertTrue(dirNode.isWithQuota());
- // the inode in inodeMap should also be replaced
- dirNodeFromNode = fsdir.getInode(dirNode.getId());
- assertSame(dirNode, dirNodeFromNode);
+ cluster.getNamesystem().readLock();
+ try {
+ INodeDirectory dirNode = getDir(fsdir, dir);
+ assertTrue(dirNode.isWithQuota());
+ // the inode in inodeMap should also be replaced
+ INode dirNodeFromNode = fsdir.getInode(dirNode.getId());
+ assertSame(dirNode, dirNodeFromNode);
+ } finally {
+ cluster.getNamesystem().readUnlock();
+ }
hdfs.setQuota(dir, -1, -1);
- dirNode = getDir(fsdir, dir);
- // the inode in inodeMap should also be replaced
- dirNodeFromNode = fsdir.getInode(dirNode.getId());
- assertSame(dirNode, dirNodeFromNode);
+ cluster.getNamesystem().readLock();
+ try {
+ INodeDirectory dirNode = getDir(fsdir, dir);
+ // the inode in inodeMap should also be replaced
+ INode dirNodeFromNode = fsdir.getInode(dirNode.getId());
+ assertSame(dirNode, dirNodeFromNode);
+ } finally {
+ cluster.getNamesystem().readUnlock();
+ }
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: Add namespace key for INode. (shv)
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shv pushed a commit to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1829b58b77341b83c9c73f356fa479d9940f6a24
Author: Konstantin V Shvachko <sh...@apache.org>
AuthorDate: Fri May 7 17:51:58 2021 -0700
Add namespace key for INode. (shv)
---
.../org/apache/hadoop/util/PartitionedGSet.java | 80 ++++++++++++++++++----
.../hadoop/hdfs/server/namenode/FSDirMkdirOp.java | 3 +
.../apache/hadoop/hdfs/server/namenode/INode.java | 40 ++++++++++-
.../hadoop/hdfs/server/namenode/INodeMap.java | 71 +++++++++++++++++--
4 files changed, 176 insertions(+), 18 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
index 4b0cdc9..7ebb1b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
@@ -22,6 +22,7 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.NavigableMap;
+import java.util.Set;
import java.util.TreeMap;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -44,7 +45,8 @@ import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
@InterfaceAudience.Private
public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
- private static final int DEFAULT_PARTITION_CAPACITY = 2027;
+ private static final int DEFAULT_PARTITION_CAPACITY = 65536; // 4096; // 5120; // 2048; // 1027;
+ private static final float DEFAULT_PARTITION_OVERFLOW = 1.8f;
/**
* An ordered map of contiguous segments of elements.
@@ -81,8 +83,11 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
final E rootKey) {
this.partitions = new TreeMap<K, PartitionEntry>(comparator);
this.latchLock = latchLock;
- addNewPartition(rootKey).put(rootKey);
- this.size = 1;
+ // addNewPartition(rootKey).put(rootKey);
+ // this.size = 1;
+ this.size = 0;
+ LOG.info("Partition capacity = {}", DEFAULT_PARTITION_CAPACITY);
+ LOG.info("Partition overflow factor = {}", DEFAULT_PARTITION_OVERFLOW);
}
/**
@@ -90,16 +95,19 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
* @param key
* @return
*/
- private PartitionEntry addNewPartition(final K key) {
+ public PartitionEntry addNewPartition(final K key) {
+ Entry<K, PartitionEntry> lastEntry = partitions.lastEntry();
PartitionEntry lastPart = null;
- if(size > 0)
- lastPart = partitions.lastEntry().getValue();
+ if(lastEntry != null)
+ lastPart = lastEntry.getValue();
PartitionEntry newPart =
new PartitionEntry(DEFAULT_PARTITION_CAPACITY);
// assert size == 0 || newPart.partLock.isWriteTopLocked() :
// "Must hold write Lock: key = " + key;
- partitions.put(key, newPart);
+ PartitionEntry oldPart = partitions.put(key, newPart);
+ assert oldPart == null :
+ "RangeMap already has a partition associated with " + key;
LOG.debug("Total GSet size = {}", size);
LOG.debug("Number of partitions = {}", partitions.size());
@@ -173,7 +181,7 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
private PartitionEntry addNewPartitionIfNeeded(
PartitionEntry curPart, K key) {
- if(curPart.size() < DEFAULT_PARTITION_CAPACITY * 1.1
+ if(curPart.size() < DEFAULT_PARTITION_CAPACITY * DEFAULT_PARTITION_OVERFLOW
|| curPart.contains(key)) {
return curPart;
}
@@ -197,12 +205,56 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
public void clear() {
LOG.error("Total GSet size = {}", size);
LOG.error("Number of partitions = {}", partitions.size());
+ printStats();
// assert latchLock.hasWriteTopLock() : "Must hold write topLock";
// SHV May need to clear all partitions?
partitions.clear();
size = 0;
}
+ private void printStats() {
+ int partSizeMin = Integer.MAX_VALUE, partSizeAvg = 0, partSizeMax = 0;
+ long totalSize = 0;
+ int numEmptyPartitions = 0, numFullPartitions = 0;
+ Collection<PartitionEntry> parts = partitions.values();
+ Set<Entry<K, PartitionEntry>> entries = partitions.entrySet();
+ int i = 0;
+ for(Entry<K, PartitionEntry> e : entries) {
+ PartitionEntry part = e.getValue();
+ int s = part.size;
+ if(s == 0) numEmptyPartitions++;
+ if(s > DEFAULT_PARTITION_CAPACITY) numFullPartitions++;
+ totalSize += s;
+ partSizeMin = (s < partSizeMin ? s : partSizeMin);
+ partSizeMax = (partSizeMax < s ? s : partSizeMax);
+ Class<?> inodeClass = e.getKey().getClass();
+ try {
+ long[] key = (long[]) inodeClass.
+ getMethod("getNamespaceKey", int.class).invoke(e.getKey(), 2);
+ long[] firstKey = new long[0];
+ if(part.iterator().hasNext()) {
+ Object first = part.iterator().next();
+ firstKey = (long[]) inodeClass.getMethod(
+ "getNamespaceKey", int.class).invoke(first, 2);
+ Object parent = inodeClass.
+ getMethod("getParent").invoke(first);
+ long parentId = (parent == null ? 0L :
+ (long) inodeClass.getMethod("getId").invoke(parent));
+ firstKey[0] = parentId;
+ }
+ LOG.error("Partition #{}\t key: {}\t size: {}\t first: {}",
+ i++, key, s, firstKey); // SHV should be info
+ } catch (Exception ex) {
+ LOG.error("Cannot find Method getNamespaceKey() in {}", inodeClass);
+ }
+ }
+ partSizeAvg = (int) (totalSize / parts.size());
+ LOG.error("Partition sizes: min = {}, avg = {}, max = {}, sum = {}",
+ partSizeMin, partSizeAvg, partSizeMax, totalSize);
+ LOG.error("Number of partitions: empty = {}, full = {}",
+ numEmptyPartitions, numFullPartitions);
+ }
+
@Override
public Collection<E> values() {
// TODO Auto-generated method stub
@@ -234,15 +286,19 @@ public class PartitionedGSet<K, E extends K> implements GSet<K, E> {
@Override
public boolean hasNext() {
- if(partitionIterator.hasNext()) {
- return true;
+ while(!partitionIterator.hasNext()) {
+ if(!keyIterator.hasNext()) {
+ return false;
+ }
+ K curKey = keyIterator.next();
+ partitionIterator = getPartition(curKey).iterator();
}
- return keyIterator.hasNext();
+ return partitionIterator.hasNext();
}
@Override
public E next() {
- if(!partitionIterator.hasNext()) {
+ while(!partitionIterator.hasNext()) {
K curKey = keyIterator.next();
partitionIterator = getPartition(curKey).iterator();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index c8c6277..5a40906 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -265,10 +265,13 @@ class FSDirMkdirOp {
// create the missing directories along the path
INode[] missing = new INode[numMissing];
final int last = iip.length();
+ INode parent = existing.getLastINode();
for (int i = existing.length(); i < last; i++) {
byte[] component = iip.getPathComponent(i);
missing[i - existing.length()] =
createDirectoryINode(fsd, existing, component, perm);
+ missing[i - existing.length()].setParent(parent.asDirectory());
+ parent = missing[i - existing.length()];
}
return missing;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index daff95c..42e462e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -577,6 +578,43 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
return name == null? null: DFSUtil.bytes2String(name);
}
+ private long[] namespaceKey;
+
+ /**
+ * Key of an INode.
+ * Defines partitioning of INodes in the INodeMap.
+ *
+ * @param level how many levels to be included in the key
+ * @return
+ */
+ public long[] getNamespaceKey(int level) {
+ if(namespaceKey == null) { // generate the namespace key
+ long[] buf = new long[level];
+ INode cur = this;
+ for(int l = 0; l < level; l++) {
+ long curId = (cur == null) ? 0L : cur.getId();
+ buf[level - l - 1] = curId;
+ cur = (cur == null) ? null : cur.parent;
+ }
+ buf[0] = indexOf(buf);
+ namespaceKey = buf;
+ }
+ return namespaceKey;
+ }
+
+ private final static long LARGE_PRIME = 512927357;
+ public static long indexOf(long[] key) {
+ if(key[key.length-1] == INodeId.ROOT_INODE_ID) {
+ return key[0];
+ }
+ long idx = LARGE_PRIME * key[0];
+ idx = (idx ^ (idx >> 32)) & (INodeMap.NUM_RANGES_STATIC -1);
+ return idx;
+ }
+
+ /**
+ * Key of a snapshot Diff Element
+ */
@Override
public final byte[] getKey() {
return getLocalNameBytes();
@@ -636,7 +674,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
@Override
public String toString() {
- return getLocalName();
+ return getLocalName() + ": " + Arrays.toString(namespaceKey);
}
@VisibleForTesting
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index 88c3233..3b07dce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -29,14 +29,63 @@ import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LatchLock;
import org.apache.hadoop.util.LightWeightGSet;
import org.apache.hadoop.util.PartitionedGSet;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Storing all the {@link INode}s and maintaining the mapping between INode ID
* and INode.
*/
public class INodeMap {
+ static final int NAMESPACE_KEY_DEBTH = 2;
+ static final int NUM_RANGES_STATIC = 256; // power of 2
+
+ public static class INodeKeyComparator implements Comparator<INode> {
+ INodeKeyComparator() {
+ FSDirectory.LOG.info("Namespace key debth = {}", NAMESPACE_KEY_DEBTH);
+ }
+
+ @Override
+ public int compare(INode i1, INode i2) {
+ if (i1 == null || i2 == null) {
+ throw new NullPointerException("Cannot compare null INodes");
+ }
+ long[] key1 = i1.getNamespaceKey(NAMESPACE_KEY_DEBTH);
+ long[] key2 = i2.getNamespaceKey(NAMESPACE_KEY_DEBTH);
+ for(int l = 0; l < NAMESPACE_KEY_DEBTH; l++) {
+ if(key1[l] == key2[l]) continue;
+ return (key1[l] < key2[l] ? -1 : 1);
+ }
+ return 0;
+ }
+ }
+
+ /**
+ * INodeKeyComparator with Hashed Parent
+ *
+ */
+ public static class HPINodeKeyComparator implements Comparator<INode> {
+ HPINodeKeyComparator() {
+ FSDirectory.LOG.info("Namespace key debth = {}", NAMESPACE_KEY_DEBTH);
+ }
+
+ @Override
+ public int compare(INode i1, INode i2) {
+ if (i1 == null || i2 == null) {
+ throw new NullPointerException("Cannot compare null INodes");
+ }
+ long[] key1 = i1.getNamespaceKey(NAMESPACE_KEY_DEBTH);
+ long[] key2 = i2.getNamespaceKey(NAMESPACE_KEY_DEBTH);
+ long key1_0 = INode.indexOf(key1);
+ long key2_0 = INode.indexOf(key2);
+ if(key1_0 != key2_0)
+ return (key1_0 < key2_0 ? -1 : 1);
+ for(int l = 1; l < NAMESPACE_KEY_DEBTH; l++) {
+ if(key1[l] == key2[l]) continue;
+ return (key1[l] < key2[l] ? -1 : 1);
+ }
+ return 0;
+ }
+ }
+
public static class INodeIdComparator implements Comparator<INode> {
@Override
public int compare(INode i1, INode i2) {
@@ -50,8 +99,6 @@ public class INodeMap {
}
public class INodeMapLock extends LatchLock<ReentrantReadWriteLock> {
- Logger LOG = LoggerFactory.getLogger(INodeMapLock.class);
-
private ReentrantReadWriteLock childLock;
INodeMapLock() {
@@ -146,8 +193,22 @@ public class INodeMap {
this.namesystem = ns;
// Compute the map capacity by allocating 1% of total memory
int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
- this.map = new PartitionedGSet<>(capacity, new INodeIdComparator(),
+ this.map = new PartitionedGSet<>(capacity, new INodeKeyComparator(),
new INodeMapLock(), rootDir);
+
+ // Pre-populate initial empty partitions
+ PartitionedGSet<INode, INodeWithAdditionalFields> pgs =
+ (PartitionedGSet<INode, INodeWithAdditionalFields>) map;
+ PermissionStatus perm = new PermissionStatus(
+ "", "", new FsPermission((short) 0));
+ for(int p = 0; p < NUM_RANGES_STATIC; p++) {
+ INodeDirectory key = new INodeDirectory(
+ INodeId.ROOT_INODE_ID, "range key".getBytes(), perm, 0);
+ key.setParent(new INodeDirectory((long)p, null, perm, 0));
+ pgs.addNewPartition(key);
+ }
+
+ map.put(rootDir);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org