You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2025/03/27 12:59:08 UTC
(hadoop) branch HADOOP-19236-02 updated (b44a0e18025 -> e392fad87b2)
This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a change to branch HADOOP-19236-02
in repository https://gitbox.apache.org/repos/asf/hadoop.git
omit b44a0e18025 HADOOP-19236. Add unit tests to VolcanoEngine Cloud TOS File System.
add 6370cd268c6 Revert "HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Impleme…" (#7327)
add b6916ebe905 HADOOP-19401: Improve error message when OS can't identify the current user.
add d9bcee929e7 HADOOP-19289. Upgrade to protobuf-java 3.25.5 (#7072)
add 91535fa7b7d YARN-11759: Fix log statement in RMAppImpl#processNodeUpdate
add a5926668682 HADOOP-19225. Upgrade Jetty to 9.4.57.v20241219 due to CVE-2024-8184 (#7116). Contributed by PJ Fanning.
add c7e1b660879 HADOOP-19232: [ABFS][FNSOverBlob] Implementing Ingress Support with various Fallback Handling (#7272)
add 1ba30d6ca63 YARN-11754. [JDK17] Fix SpotBugs Issues in YARN. (#7317) Contributed by Shilun Fan.
add 44a5cba78ac HADOOP-19407: Update 3.4.1 docs landing page to highlight changes shipped in the release.
add 741bdd636b4 HDFS-17721. RBF: Allow routers to declare IP for admin addr (#7342) Contributed by Felix Nguyen.
add 6d20de1ceda HADOOP-19233: ABFS: [FnsOverBlob] Implementing Rename and Delete APIs over Blob Endpoint (#7265)
add 7459a1f215a HADOOP-19405. hadoop-aws and hadoop-azure tests have stopped running. (#7335)
add eae63ec1827 HADOOP-19443: [ABFS][FnsOverBlob] Updating hadoop-azure Test Scripts and Adding Tests For Metadata APIs (#7344)
add 950b3eb431d MAPREDUCE-7497. mapreduce tests have stopped running. (#7343)
add f1ee35b8a18 HADOOP-19415. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-common Part1. (#7339)
add f900339a210 HADOOP-19404: [ABFS][FNSOverBlob] Update documentation for FNS Blob Onboard (#7341)
add b5cbca37dfc HADOOP-19415. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-common Part2. (#7347)
add 872ebda81d4 MAPREDUCE-7415. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-nativetask. (#7349)
add 57d0979d383 HADOOP-19311: [ABFS] Implement Backoff and Read Footer metrics using IOStatistics Class (#7122)
add acb7a800601 MAPREDUCE-7414. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-hs. (#7354)
add 851b4c38a54 MAPREDUCE-7416. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-shuffle. (#7351)
add d552670b86d MAPREDUCE-7418. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-app. (#7350)
add 73ac0b9e39c HADOOP-19448: [ABFS][FNSOverBlob][Optimizations] Reduce Network Calls In Create and Mkdir Flow (#7353)
add cfe76e45cb6 HADOOP-19415. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-common Part3. (#7355)
add 755fa7482bf HDFS-17724: Set recover.lease.on.close.exception as an instance member in the DfsClientConf.java
add 2550344f4f0 HADOOP-19415. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-common Part4. (#7356)
add 81146fed2f8 HADOOP-19415. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-common Part5. (#7357)
add c5fdc3ba6c2 HADOOP-19431. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-distcp. (#7368)
add 8424c155f11 MAPREDUCE-7420. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-core Part1. (#7363)
add b24996cb96a HDFS-17719. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-hdfs-httpfs Part1. (#7371)
add b4168c37b7a YARN-11760. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-applications-distributedshell. (#7373)
add 14b71596d45 YARN-11761. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-services-core. (#7374)
add 19bd575e32e MAPREDUCE-7421. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-jobclient Part1. (#7358)
add 2291fabdfc4 YARN-11758. [UI2] On the Cluster Metrics page make the Resource Usage… (#7331)
add 2202c95e1d2 HADOOP-19375: Organize JDK version-specific code in IDEA friendly approach
add c4dace07719 HADOOP-19377: Avoid initializing useless HashMap in protocolImplMapArray.
add 6e7511c8ac7 HDFS-12431. Upgrade JUnit from 4 to 5 in hadoop-hdfs Part1. (#7383)
add c42469e41e9 MAPREDUCE-7421. [JDK17] Upgrade Junit 4 to 5 in hadoop-mapreduce-client-jobclient Part2. (#7372)
add f1df74855dc HADOOP-19393. [ABFS] Return FileAlreadyExistsException for UnauthorizedBlobOverwrite Rename Errors (#7312)
add 7cc6b238816 HADOOP-19352. Hadoop OSS Connector adds support for V4 signatures. (#7205)
add 22b8fcd21bc YARN-11767. [UI2] upgrade moment.js to v2.30.1 (#7366)
add 744fe66c826 HADOOP-19425. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-azure Part1. (#7369)
add 06ff1b69c76 YARN-11384: NPE in DelegationTokenRenewer causes all subsequent apps to fail with "Timer already cancelled" (#7404) Contributed by Cheng Pan.
add d2442eeaddc HDFS-17543. [ARR] AsyncUtil makes asynchronous code more concise and easier. (#6868). Contributed by Jian Zhang.
add f10ef7d70a4 HADOOP-19235. IPC client uses CompletableFuture to support asynchronous operations. (#6888)
add 001caf47ee0 HDFS-17544. [ARR] The router client rpc protocol PB supports asynchrony. (#6870). Contributed by Jian Zhang.
add 2b05f6c00a1 HDFS-17545. [ARR] router async rpc client. (#6871). Contributed by Jian Zhang.
add 011e8af9dae HDFS-17594. [ARR] RouterCacheAdmin supports asynchronous rpc. (#6986). Contributed by Archie73.
add 318c09e7083 HDFS-17597. [ARR] RouterSnapshot supports asynchronous rpc. (#6994). Contributed by Wenqi Li.
add b82f097d1e9 HDFS-17595. [ARR] ErasureCoding supports asynchronous rpc. (#6983). Contributed by hfutatzhanghb.
add f15be66fe86 HDFS-17601. [ARR] RouterRpcServer supports asynchronous rpc. (#7108). Contributed by hfutatzhanghb.
add cf471b6b7f9 HDFS-17596. [ARR] RouterStoragePolicy supports asynchronous rpc. (#6988). Contributed by hfutatzhanghb.
add 060f6b866c3 HDFS-17656. [ARR] RouterNamenodeProtocol and RouterUserProtocol supports asynchronous rpc. (#7159). Contributed by Jian Zhang.
add 198288c5c8f HDFS-17659. [ARR]Router Quota supports asynchronous rpc. (#7157). Contributed by hfutatzhanghb.
add 7149d6635e3 HDFS-17672. [ARR] Move asynchronous related classes to the async package. (#7184). Contributed by Jian Zhang.
add ae171846d91 HADOOP-19361. RPC DeferredMetrics bugfix. (#7220). Contributed by hfutatzhanghb.
add e75bec65977 HDFS-17640.[ARR] RouterClientProtocol supports asynchronous rpc. (#7188)
add 2678e59c79b HDFS-17650. [ARR] The router server-side rpc protocol PB supports asynchrony. (#7139). Contributed by hfutatzhanghb.
add f66c89b6571 HDFS-17651.[ARR] Async handler executor isolation (#7244). Contributed by hfutatzhanghb.
add 05ae3e094fb HDFS-17715. [ARR] Add documentation for asynchronous router. (#7318). Contributed by Jian Zhang.
add ec6c08b8df6 HADOOP-19456. Upgrade kafka to 3.9.0 to fix CVE-2024-31141. (#7401) Contributed by Palakur Eshwitha Sai.
add 5067082dcda HADOOP-19354. S3A: S3AInputStream to be created by factory under S3AStore (#7214)
add 0dd9bf82ddc HADOOP-19465. Upgrade to netty 4.1.118 due to CVE-2025-24970 (#7413) Contributed by PJ Fanning.
add ca08047a925 HDFS-17729. Inconsistent mtime in the results of -stat and -ls command due to different TimeZone. (#7405) Contributed by hfutatzhanghb.
add 826c1f59674 HADOOP-19467: [ABFS][FnsOverBlob] Fixing Config Name and Value in Documnatation (#7420)
add 1a81c3b564e HDFS-17725. DataNodeVolumeMetrics and BalancerMetrics class add MetricTag. (#7382) Contributed by Zhaobo Huang.
add acc3e505aa0 HADOOP-19466. fix Dockerfile_windows_10 (#7417)
add 2c694e15cc4 YARN-11756. [UI2] Add the metrics from UI1 scheduler/application queues page to UI2 Queues page (#7324)
add d23ebc45121 YARN-11757. [UI2] Add partition usage overview to the Queues page (#7330)
add 1d736cc6791 HADOOP-19383: upgrade Mina 2.0.27 due to CVE-2024-52046
add bb07ff80656 HADOOP-19464. S3A: Restore Compatibility with EMRFS FileSystem (#7410)
add e6144531de8 YARN-11764. yarn tests have stopped running. (#7345)
add 507086a543c HADOOP-19450: [ABFS] Rename/Create path idempotency client-level resolution (#7364)
add a7871463d52 YARN-11777. [UI2] fix the ASF licence comment tag in the partition-usage.hsb introduced by YARN-11757 (#7435)
add b05c0ce9724 HDFS-17496. DataNode supports more fine-grained dataset lock based on blockid. (#7280). Contributed by hfutatzhanghb.
add cc14236c9d9 HADOOP-19348. Integrate analytics accelerator into S3A. (#7433)
add e5615d89da6 YARN-11780. [UI2] typo in the yarn-queue-partition-capacity-labels.hbs queue status section and improve the title tag wordings (#7442)
add 89f9c5af0c1 YARN-11762. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-globalpolicygenerator. (#7389)
add f1f27e1a503 YARN-11263. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-nodemanager Part1. (#7390)
add 1e139d51327 YARN-11258. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-common. (#7437)
add 56be78c364e HDFS-17745. TestRouterMountTable should reset defaultNSEnable to true after each test method. (#7446). Contributed by hfutatzhanghb.
add 246ebe809c8 HDFS-17731. [ARR] Add unit test for async RouterAdminServer. (#7409). Contributed by hfutatzhanghb.
add 32dad2039ad HDFS-17733. [ARR] Optimize isMultiDestDirectory method using AsyncUtil class (#7415). Contributed by hfutatzhanghb.
add 1c2a92ad9ef HADOOP-19303. VectorIO API: support pass-down of a release() operator (#7418)
add 94daf4ad85c HADOOP-19476. Create python3 symlink needed for mvnsite (#7452)
add 4b9e1dcb3b8 YARN-11782. [Federation] Fix incorrect error messages and improve failure handling. (#7449)
add eda1236219b YARN-11244. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-client. (#7438)
add 7c9811062de YARN-11783. Upgrade wro4j to 1.8.0 (#7457)
add 5054b16b1f7 HDFS-17748. Fix javadoc problems caused by HDFS-17496. (#7460). Contributed by farmmamba.
add e72b2184d1e YARN-11263. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-nodemanager Part2. (#7455)
add f552664d94d YARN-11268. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-timelineservice-documentstore. (#7450)
add 103b0546548 HADOOP-19270 Use stable sort in commandQueue (#7038)
add 6b561d54670 YARN-11785. Race condition in QueueMetrics due to non-thread-safe HashMap causes MetricsException. (#7459) Contributed by Tao Yang.
add d552bb056c3 HADOOP-19445: ABFS: [FnsOverBlob][Tests] Add Tests For Negative Scenarios Identified for Rename Operation (#7386)
add f0430f22f73 HDFS-17753. Fix occasional failure of TestRouterHttpServerXFrame. (#7467) Contributed by farmmamba.
add d7a462f0026 [YARN-11776] Handle NPE in the RMDelegationTokenIdentifier if localServiceAddress is null (#7431) Contributed by Abhey Rana.
add 55e0f0a6385 HADOOP-19480. Upgrades AAL version to 1.0.0. (#7469)
add 972b6e0be4c YARN-11790. TestAmFilter#testProxyUpdate fails in some networks (#7478) Contributed by Chris Nauroth.
add 7ab88fe2626 HDFS-17744. [ARR] getEnclosingRoot RPC adapts to async rpc. (#7445). Contributed by hfutatzhanghb.
add 68651ec71eb HADOOP-19086. Update commons-logging to 1.3.0 (#7472)
add 126c3d4be10 YARN-11793: Remove grizzly-http-* dependencies from Hadoop to avoid transitive inclusions.
add 081d51a3828 Revert "YARN-11793: Remove grizzly-http-* dependencies from Hadoop to avoid transitive inclusions."
add 4baa167f0a0 YARN-11786. Upgrade hadoop-yarn-server-timelineservice-hbase-tests to Support Trunk Compilation and Remove compatible hadoop version. (#7453)
add 833725193f2 HADOOP-19487. Upgrade libopenssl to 3.1.1 for rsync on Windows (#7487)
add 1416c94d7cc HDFS-17750. [ARR] RouterWebHdfsMethods adapts to async rpc. (#7462). Contributed by hfutatzhanghb.
add affb8c36aac HDFS-17746. [ARR] The behavior of throwing exception in getListing should be consistent with sync mode. (#7464). Contributed by hfutatzhanghb.
add 2b32e46f666 HDFS-17754. Add uriparser2 to notices (#7481)
add e0d739a7ff2 HADOOP-19444: ABFS: [FnsOverBlob][Tests] Add Tests For Negative Scenarios Identified for Ingress Operations (#7424)
add 32da0d6d144 HADOOP-19400: Expand specification and contract test coverage for InputStream reads.
add ef9043fcc00 HDFS-17743: Added support for random datanode ordering in getBlockLocations() (#7447)
add 49d4c734b0e YARN-11765. Refactor: Move Clock Class from hadoop-mapreduce-project to hadoop-common-project for Reusability (#7352) Contributed by Jiandan Yang.
add f65e9476f75 HADOOP-19446. ABFS: [FnsOverBlob][Tests] Add Tests For Negative Scenarios Identified for Delete Operation (#7376)
add 7d143b7fb1c YARN-11262. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part1. (#7474)
add baf393d6c6e YARN-11262. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part2. (#7484)
add f910b11614f HDFS-17735. [ARR] LocalResolver#getDatanodesSubcluster adapts to async rpc. (#7422). Contributed by hfutatzhanghb.
add b73796a43c7 HADOOP-19452. Client#call decrease asyncCallCounter incorrectlly when exceptions occur. (#7384). Contributed by hfutatzhanghb.
add cd8f18b71d2 MAPREDUCE-7503. Fix ByteBuf leaks in TestShuffleChannelHandler (#7500) Contributed by Istvan Toth.
add 4d8ac20fd42 HADOOP-19500. Skip tests that require JavaScript engine when it's not available (#7503)
add 9ca1052d00c YARN-11799. Remove PrintStream write counting from TestYarnCLI (#7518)
add 7005bcb4c9a HADOOP-18940. [ABFS] Remove commons IOUtils.close() from AbfsOutputStream (#7506)
add fba01e69556 YARN-11797. Fix tests trying to connect to 0.0.0.0 (#7515) Contributed by Istvan Toth.
add f9f9aeefc6c HADOOP-19506. Fix TestThrottledInputStream when bandwidth is equal to throttle limit (#7517) Contributed by Istvan Toth.
add 71a945ac869 MAPREDUCE-7504. Remove terminally deprecated Thread.suspend from TestMergeManager (#7519) Contributed by Istvan Toth.
add 6e844030c9e YARN-11262. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part3. (#7485)
add 069eff9ecd2 HADOOP-19475. Update Boost to 1.86.0
add af6a64f7631 HADOOP-19499. Handle JDK-8225499 IpAddr.toString() format changes in tests
add 758f3a04638 HDFS-17759. Explicitly depend on jackson-core in hadoop-hdfs (#7508) Contributed by Istvan Toth.
add 1f640aa02a2 YARN-11787. [Federation] Enhance Exception Handling in FederationClientInterceptor#forceKillApplication. (#7465) Contributed by Shilun Fan.
add df5e2054525 HDFS-17713. [ARR] Throtting asynchronous calls for each nameservice. (#7304). Contributed by hfutatzhanghb.
add 26b9affe046 HDFS-17714. MockResolver#removeLocation should also remove src path. (#7313). Contributed by hfutatzhanghb.
add e5c901823d6 YARN-11788 - Remove unused unit tests. (#7476) Contributed by Shilun Fan.
add d26f4fdf3a8 HADOOP-19509. Add a config entry to make IPC.Client checkAsyncCall off by default. (#7521)
add 3e8f1f65bdd HDFS-17720.[JDK17] Upgrade JUnit from 4 to 5 in hadoop-hdfs-rbf Part1.
add 65f656738aa HADOOP-19488 fix the temporary direction creation on Windows (#7511)
add 2af914b5d91 HADOOP-18991 Remove commons-beanutils dependency from Hadoop 3 (#7524) Contributed by Istvan Toth.
add 05167ea06af YARN-11795. Skip BaseFederationPoliciesTest.testReinitilializeBad3() on JVMs where Mockito cannot mock ByteBuffers (#7510) Contributed by Istvan Toth.
add b64ad9b682d HADOOP-19508. Set charsetEncoder in HadoopArchiveLogs (#7520) Contributed by Istvan Toth.
add 68fcde01fba YARN-11262. Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part4. (#7488)
add fb30082948c HDFS-17749. [ARR] Fix wrong results due to the wrong usage of asyncComplete in getListing. (#7466). Contributed by hfutatzhanghb.
add ae9cd7576d9 YARN-11262. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part5. (#7489)
add 488376c13b1 HADOOP-19502. Add RegexpSingleline module to checkstyle.xml. (#7505) Contributed by hfutatzhanghb.
add c94b41848bd Revert "HADOOP-19502. Add RegexpSingleline module to checkstyle.xml. (#7505) …" (#7534)
add 3fd404f2eff HADOOP-19495. Add JDK 24 to Ubuntu 20.04 docker development images
add 26940713fc5 YARN-11796. Accept extra JDK 17 options in tests (#7512) Contributed by Istvan Toth.
add 9f7aa7223f6 HADOOP-19514: SecretManager logs at INFO in bin/hadoop calls
add 18f761785a2 HDFS-17720.[JDK17] Upgrade JUnit from 4 to 5 in hadoop-hdfs-rbf Part2.
add cc9e5485727 YARN-11266. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-timelineservice-hbase-tests. (#7538)
add a314a1d7148 YARN-11798. Precheck request separately to avoid redundant node checks and optimize performance for global scheduler. (#7516) Contributed by Tao Yang.
add 3b8942fd764 YARN-11262. [JDK17] Upgrade JUnit from 4 to 5 in hadoop-yarn-server-resourcemanager Part6. (#7490)
add 140f047b5cd HADOOP-19474: [ABFS][FnsOverBlob] Listing Optimizations to avoid multiple iteration over list response. (#7421)
add 5c10e0d458f HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation. (#7504)
new e392fad87b2 HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation (Part 2: Add unit tests.).
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b44a0e18025)
\
N -- N -- N refs/heads/HADOOP-19236-02 (e392fad87b2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BUILDING.txt | 16 +-
LICENSE-binary | 95 +-
NOTICE-binary | 9 +-
dev-support/docker/Dockerfile | 5 +
dev-support/docker/Dockerfile_aarch64 | 5 +
dev-support/docker/Dockerfile_windows_10 | 27 +-
dev-support/docker/pkg-resolver/install-boost.sh | 14 +-
dev-support/docker/pkg-resolver/packages.json | 2 +
.../hadoop-cloud-storage/pom.xml | 6 -
hadoop-cloud-storage-project/hadoop-tos/pom.xml | 60 +-
.../hadoop-tos/src/main/assembly/hadoop-tos.xml | 43 -
.../org/apache/hadoop/fs/tosfs/RawFileSystem.java | 19 +-
.../org/apache/hadoop/fs/tosfs/TosFileSystem.java | 8 +-
.../apache/hadoop/fs/tosfs/commit/Committer.java | 14 +-
.../apache/hadoop/fs/tosfs/commit/SuccessData.java | 28 +-
.../fs/tosfs/commit/mapred/package-info.java | 16 +-
.../org/apache/hadoop/fs/tosfs/common/Tasks.java | 6 +-
.../apache/hadoop/fs/tosfs/common/ThreadPools.java | 4 +-
.../apache/hadoop/fs/tosfs/conf/package-info.java | 16 +-
.../apache/hadoop/fs/tosfs/object/FileStore.java | 7 +-
.../hadoop/fs/tosfs/object/ObjectOutputStream.java | 14 +-
.../fs/tosfs/object/tos/DelegationClient.java | 11 +-
.../tosfs/object/tos/DelegationClientBuilder.java | 46 +-
.../org/apache/hadoop/fs/tosfs/object/tos/TOS.java | 3 +-
.../tos/auth/AbstractCredentialsProvider.java | 2 +-
.../object/tos/auth/ExpireableCredential.java | 2 +-
.../org/apache/hadoop/fs/tosfs/ops/RenameOp.java | 15 +-
.../apache/hadoop/fs/tosfs/util/CommonUtils.java | 2 +-
.../org/apache/hadoop/fs/tosfs/util/FSUtils.java | 5 +-
.../org/apache/hadoop/fs/tosfs/util/Iterables.java | 2 +-
.../hadoop/fs/tosfs/util/RetryableUtils.java | 2 +-
.../src/site/markdown/cloud-storage/index.md | 17 +-
.../java/org/apache/hadoop/fs/tosfs/TestEnv.java | 6 +-
.../org/apache/hadoop/fs/tosfs/TestRawFSUtils.java | 28 +-
.../apache/hadoop/fs/tosfs/TestRawFileSystem.java | 16 +-
.../apache/hadoop/fs/tosfs/TestTosChecksum.java | 92 +-
.../apache/hadoop/fs/tosfs/TestTosFileSystem.java | 20 +-
.../hadoop/fs/tosfs/commit/BaseJobSuite.java | 127 +-
.../hadoop/fs/tosfs/commit/CommitterTestBase.java | 114 +-
.../apache/hadoop/fs/tosfs/commit/JobSuite.java | 111 +-
.../hadoop/fs/tosfs/commit/MRJobTestBase.java | 81 +-
.../hadoop/fs/tosfs/commit/TestCommitter.java | 1 -
.../apache/hadoop/fs/tosfs/commit/TestMRJob.java | 4 +-
.../fs/tosfs/commit/TestMagicOutputStream.java | 91 +-
.../fs/tosfs/commit/mapred/CommitterTestBase.java | 98 +-
.../hadoop/fs/tosfs/commit/mapred/JobSuite.java | 101 +-
.../hadoop/fs/tosfs/contract/TestChecksum.java | 8 +-
.../hadoop/fs/tosfs/contract/TestCreate.java | 9 +-
.../hadoop/fs/tosfs/contract/TestDistCp.java | 11 +-
.../fs/tosfs/contract/TestGetFileStatus.java | 3 +-
.../apache/hadoop/fs/tosfs/contract/TestOpen.java | 3 +-
.../hadoop/fs/tosfs/contract/TestRename.java | 9 +-
.../hadoop/fs/tosfs/contract/TosContract.java | 1 -
.../fs/tosfs/object/ObjectStorageTestBase.java | 55 +-
.../hadoop/fs/tosfs/object/ObjectTestUtils.java | 43 +-
.../fs/tosfs/object/TestDirectoryStorage.java | 58 +-
.../object/TestObjectMultiRangeInputStream.java | 124 +-
.../fs/tosfs/object/TestObjectOutputStream.java | 120 +-
.../tosfs/object/TestObjectRangeInputStream.java | 58 +-
.../hadoop/fs/tosfs/object/TestObjectStorage.java | 452 ++--
.../tosfs/object/tos/TestChainTOSInputStream.java | 60 +-
.../object/tos/TestDelegationClientBuilder.java | 131 +-
.../fs/tosfs/object/tos/TestTOSInputStream.java | 39 +-
.../fs/tosfs/object/tos/TestTOSObjectStorage.java | 229 +-
.../fs/tosfs/object/tos/TestTOSRetryPolicy.java | 21 +-
.../auth/TestDefaultCredentialsProviderChain.java | 64 +-
.../auth/TestEnvironmentCredentialsProvider.java | 28 +-
.../tos/auth/TestSimpleCredentialsProvider.java | 35 +-
.../apache/hadoop/fs/tosfs/ops/TestBaseFsOps.java | 170 +-
.../apache/hadoop/fs/tosfs/ops/TestBaseOps.java | 61 +-
.../hadoop/fs/tosfs/ops/TestDefaultFsOps.java | 71 +-
.../hadoop/fs/tosfs/ops/TestDirectoryFsOps.java | 54 +-
.../apache/hadoop/fs/tosfs/ops/TestRenameOp.java | 111 +-
.../org/apache/hadoop/fs/tosfs/util/TempFiles.java | 2 +-
.../apache/hadoop/fs/tosfs/util/TestFSUtils.java | 6 +-
.../apache/hadoop/fs/tosfs/util/TestIterables.java | 14 +-
.../hadoop/fs/tosfs/util/TestLazyReload.java | 41 +-
.../org/apache/hadoop/fs/tosfs/util/TestRange.java | 17 +-
.../apache/hadoop/fs/tosfs/util/TestUtility.java | 9 +-
hadoop-common-project/hadoop-annotations/pom.xml | 34 +-
.../ExcludePrivateAnnotationsJDiffDoclet.java | 0
.../ExcludePrivateAnnotationsStandardDoclet.java | 0
.../tools/IncludePublicAnnotationsJDiffDoclet.java | 0
.../IncludePublicAnnotationsStandardDoclet.java | 0
.../classification/tools/RootDocProcessor.java | 0
.../classification/tools/StabilityOptions.java | 0
hadoop-common-project/hadoop-common/pom.xml | 36 +-
.../apache/hadoop/fs/BufferedFSInputStream.java | 8 +
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 12 +-
.../org/apache/hadoop/fs/FSDataInputStream.java | 8 +
.../org/apache/hadoop/fs/PositionedReadable.java | 29 +
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 127 +-
.../org/apache/hadoop/fs/VectoredReadUtils.java | 55 +-
.../apache/hadoop/fs/impl/VectorIOBufferPool.java | 80 +
.../main/java/org/apache/hadoop/fs/shell/Stat.java | 2 -
.../hadoop/fs/statistics/StreamStatisticNames.java | 7 +
.../main/java/org/apache/hadoop/ipc/Client.java | 180 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine.java | 4 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine2.java | 4 +-
.../src/main/java/org/apache/hadoop/ipc/RPC.java | 9 +-
.../org/apache/hadoop/net/NetworkTopology.java | 18 +
.../hadoop/security/UGIExceptionMessages.java | 1 +
.../hadoop/security/UserGroupInformation.java | 6 +
.../hadoop/security/token/SecretManager.java | 4 +-
.../main/java/org/apache/hadoop}/util/Clock.java | 2 +-
.../apache/hadoop/util/ConfigurationHelper.java | 36 +
.../org/apache/hadoop}/util/MonotonicClock.java | 3 +-
.../main/java/org/apache/hadoop/util/RunJar.java | 72 +-
.../java/org/apache/hadoop}/util/SystemClock.java | 7 +-
.../java/org/apache/hadoop}/util/UTCClock.java | 3 +-
.../src/main/resources/core-default.xml | 1 +
.../site/markdown/filesystem/fsdatainputstream.md | 47 +-
.../java/org/apache/hadoop/cli/CLITestHelper.java | 19 +-
.../test/java/org/apache/hadoop/cli/TestCLI.java | 10 +-
.../org/apache/hadoop/conf/TestConfServlet.java | 26 +-
.../org/apache/hadoop/conf/TestConfigRedactor.java | 13 +-
.../org/apache/hadoop/conf/TestConfiguration.java | 138 +-
.../hadoop/conf/TestConfigurationDeprecation.java | 41 +-
.../hadoop/conf/TestConfigurationFieldsBase.java | 42 +-
.../hadoop/conf/TestConfigurationSubclass.java | 14 +-
.../org/apache/hadoop/conf/TestDeprecatedKeys.java | 13 +-
.../org/apache/hadoop/conf/TestGetInstances.java | 6 +-
.../apache/hadoop/conf/TestReconfiguration.java | 132 +-
.../org/apache/hadoop/conf/TestStorageUnit.java | 234 +-
.../hadoop/crypto/CryptoStreamsTestBase.java | 187 +-
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 72 +-
.../crypto/TestCryptoOutputStreamClosing.java | 12 +-
.../apache/hadoop/crypto/TestCryptoStreams.java | 14 +-
.../hadoop/crypto/TestCryptoStreamsForLocalFS.java | 65 +-
.../hadoop/crypto/TestCryptoStreamsNormal.java | 70 +-
.../TestCryptoStreamsWithJceAesCtrCryptoCodec.java | 4 +-
.../TestCryptoStreamsWithJceSm4CtrCryptoCodec.java | 4 +-
...tCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 23 +-
...tCryptoStreamsWithOpensslSm4CtrCryptoCodec.java | 32 +-
.../org/apache/hadoop/crypto/TestCryptoUtils.java | 36 +-
.../apache/hadoop/crypto/TestOpensslCipher.java | 60 +-
.../hadoop/crypto/key/TestCachingKeyProvider.java | 163 +-
.../apache/hadoop/crypto/key/TestKeyProvider.java | 36 +-
.../crypto/key/TestKeyProviderCryptoExtension.java | 149 +-
.../TestKeyProviderDelegationTokenExtension.java | 24 +-
.../hadoop/crypto/key/TestKeyProviderFactory.java | 98 +-
.../org/apache/hadoop/crypto/key/TestKeyShell.java | 20 +-
.../apache/hadoop/crypto/key/TestValueQueue.java | 132 +-
.../crypto/key/kms/TestKMSClientProvider.java | 20 +-
.../kms/TestLoadBalancingKMSClientProvider.java | 27 +-
.../crypto/random/TestOpensslSecureRandom.java | 18 +-
.../hadoop/crypto/random/TestOsSecureRandom.java | 26 +-
.../org/apache/hadoop/fs/TestAfsCheckPath.java | 14 +-
.../java/org/apache/hadoop/fs/TestAvroFSInput.java | 4 +-
.../org/apache/hadoop/fs/TestBlockLocation.java | 15 +-
.../apache/hadoop/fs/TestChecksumFileSystem.java | 30 +-
.../org/apache/hadoop/fs/TestCommandFormat.java | 9 +-
.../org/apache/hadoop/fs/TestContentSummary.java | 63 +-
.../hadoop/fs/TestDFCachingGetSpaceUsed.java | 12 +-
.../org/apache/hadoop/fs/TestDFVariations.java | 51 +-
.../src/test/java/org/apache/hadoop/fs/TestDU.java | 35 +-
.../java/org/apache/hadoop/fs/TestDefaultUri.java | 2 +-
.../apache/hadoop/fs/TestDelegateToFileSystem.java | 7 +-
.../hadoop/fs/TestDelegateToFsCheckPath.java | 2 +-
.../hadoop/fs/TestDelegationTokenRenewer.java | 31 +-
.../java/org/apache/hadoop/fs/TestFileContext.java | 24 +-
.../hadoop/fs/TestFileContextDeleteOnExit.java | 30 +-
.../hadoop/fs/TestFileContextResolveAfs.java | 15 +-
.../java/org/apache/hadoop/fs/TestFileStatus.java | 9 +-
.../hadoop/fs/TestFileSystemCanonicalization.java | 12 +-
.../hadoop/fs/TestFileSystemInitialization.java | 5 +-
.../hadoop/fs/TestFileSystemStorageStatistics.java | 20 +-
.../org/apache/hadoop/fs/TestFileSystemTokens.java | 13 +-
.../org/apache/hadoop/fs/TestFilterFileSystem.java | 31 +-
.../java/org/apache/hadoop/fs/TestFilterFs.java | 2 +-
.../java/org/apache/hadoop/fs/TestFsOptions.java | 4 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 27 +-
.../java/org/apache/hadoop/fs/TestFsShellCopy.java | 18 +-
.../java/org/apache/hadoop/fs/TestFsShellList.java | 28 +-
.../apache/hadoop/fs/TestFsShellReturnCode.java | 62 +-
.../org/apache/hadoop/fs/TestFsShellTouch.java | 10 +-
.../apache/hadoop/fs/TestFsUrlConnectionPath.java | 21 +-
.../hadoop/fs/TestGetFileBlockLocations.java | 13 +-
.../org/apache/hadoop/fs/TestGetSpaceUsed.java | 15 +-
.../org/apache/hadoop/fs/TestGlobExpander.java | 12 +-
.../java/org/apache/hadoop/fs/TestGlobPattern.java | 13 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 6 +-
.../apache/hadoop/fs/TestHarFileSystemBasics.java | 58 +-
.../java/org/apache/hadoop/fs/TestHardLink.java | 18 +-
.../java/org/apache/hadoop/fs/TestListFiles.java | 22 +-
.../apache/hadoop/fs/TestLocalDirAllocator.java | 205 +-
.../org/apache/hadoop/fs/TestLocalFileSystem.java | 99 +-
.../hadoop/fs/TestLocalFileSystemPermission.java | 21 +-
.../apache/hadoop/fs/TestLocatedFileStatus.java | 2 +-
.../java/org/apache/hadoop/fs/TestQuotaUsage.java | 34 +-
.../test/java/org/apache/hadoop/fs/TestTrash.java | 145 +-
.../apache/hadoop/fs/TestTruncatedInputBug.java | 4 +-
.../AbstractContractMultipartUploaderTest.java | 2 +-
.../fs/contract/AbstractContractOpenTest.java | 136 ++
.../contract/AbstractContractVectoredReadTest.java | 65 +-
.../hadoop/fs/contract/ContractTestUtils.java | 53 +
.../apache/hadoop/fs/ftp/TestFTPFileSystem.java | 28 +-
.../apache/hadoop/fs/http/TestHttpFileSystem.java | 8 +-
.../hadoop/fs/impl/TestVectoredReadUtils.java | 48 +
.../org/apache/hadoop/fs/permission/TestAcl.java | 10 +-
.../hadoop/fs/permission/TestFsPermission.java | 10 +-
.../hadoop/fs/protocolPB/TestFSSerialization.java | 4 +-
.../apache/hadoop/fs/sftp/TestSFTPFileSystem.java | 112 +-
.../apache/hadoop/fs/shell/TestAclCommands.java | 81 +-
.../apache/hadoop/fs/shell/TestCommandFactory.java | 11 +-
.../java/org/apache/hadoop/fs/shell/TestCopy.java | 26 +-
.../apache/hadoop/fs/shell/TestCopyFromLocal.java | 36 +-
.../hadoop/fs/shell/TestCopyPreserveFlag.java | 69 +-
.../apache/hadoop/fs/shell/TestCopyToLocal.java | 47 +-
.../java/org/apache/hadoop/fs/shell/TestCount.java | 34 +-
.../org/apache/hadoop/fs/shell/TestCpCommand.java | 47 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 74 +-
.../java/org/apache/hadoop/fs/shell/TestMove.java | 28 +-
.../org/apache/hadoop/fs/shell/TestPathData.java | 65 +-
.../apache/hadoop/fs/shell/TestPathExceptions.java | 6 +-
.../hadoop/fs/shell/TestPrintableString.java | 2 +-
.../java/org/apache/hadoop/fs/shell/TestTail.java | 4 +-
.../apache/hadoop/fs/shell/TestTextCommand.java | 99 +-
.../apache/hadoop/fs/shell/TestXAttrCommands.java | 42 +-
.../org/apache/hadoop/fs/shell/find/TestAnd.java | 17 +-
.../hadoop/fs/shell/find/TestFilterExpression.java | 25 +-
.../org/apache/hadoop/fs/shell/find/TestFind.java | 29 +-
.../org/apache/hadoop/fs/shell/find/TestIname.java | 16 +-
.../org/apache/hadoop/fs/shell/find/TestName.java | 16 +-
.../org/apache/hadoop/fs/shell/find/TestPrint.java | 20 +-
.../apache/hadoop/fs/shell/find/TestPrint0.java | 21 +-
.../apache/hadoop/fs/shell/find/TestResult.java | 4 +-
.../org/apache/hadoop/fs/store/TestDataBlocks.java | 21 +-
.../apache/hadoop/fs/store/TestEtagChecksum.java | 6 +-
.../apache/hadoop/fs/viewfs/TestChRootedFs.java | 155 +-
.../TestRegexMountPointInterceptorFactory.java | 9 +-
...ountPointResolvedDstPathReplaceInterceptor.java | 21 +-
...iewFSOverloadSchemeCentralMountTableConfig.java | 4 +-
.../fs/viewfs/TestViewFileSystemDelegation.java | 9 +-
.../TestViewFileSystemDelegationTokenSupport.java | 8 +-
...iewFileSystemOverloadSchemeLocalFileSystem.java | 43 +-
.../apache/hadoop/fs/viewfs/TestViewFsConfig.java | 54 +-
.../apache/hadoop/fs/viewfs/TestViewFsTrash.java | 11 +-
.../apache/hadoop/fs/viewfs/TestViewFsURIs.java | 2 +-
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 24 +-
.../org/apache/hadoop/ha/DummySharedResource.java | 4 +-
.../java/org/apache/hadoop/ha/MiniZKFCCluster.java | 4 +-
.../apache/hadoop/ha/TestActiveStandbyElector.java | 116 +-
.../apache/hadoop/ha/TestFailoverController.java | 7 +-
.../java/org/apache/hadoop/ha/TestHAAdmin.java | 9 +-
.../org/apache/hadoop/ha/TestHealthMonitor.java | 19 +-
.../java/org/apache/hadoop/ha/TestNodeFencer.java | 20 +-
.../apache/hadoop/ha/TestShellCommandFencer.java | 83 +-
.../apache/hadoop/ha/TestSshFenceByTcpPort.java | 18 +-
.../hadoop/http/HttpServerFunctionalTest.java | 8 +-
.../http/TestAuthenticationSessionCookie.java | 22 +-
.../hadoop/http/TestDisabledProfileServlet.java | 26 +-
.../org/apache/hadoop/http/TestGlobalFilter.java | 2 +-
.../org/apache/hadoop/http/TestHtmlQuoting.java | 22 +-
.../org/apache/hadoop/http/TestHttpCookieFlag.java | 27 +-
.../org/apache/hadoop/http/TestHttpRequestLog.java | 17 +-
.../org/apache/hadoop/http/TestHttpServer.java | 51 +-
.../hadoop/http/TestHttpServerLifecycle.java | 12 +-
.../org/apache/hadoop/http/TestHttpServerLogs.java | 10 +-
.../apache/hadoop/http/TestHttpServerWebapps.java | 2 +-
.../hadoop/http/TestHttpServerWithSpnego.java | 29 +-
.../apache/hadoop/http/TestIsActiveServlet.java | 8 +-
.../org/apache/hadoop/http/TestProfileServlet.java | 10 +-
.../org/apache/hadoop/http/TestSSLHttpServer.java | 17 +-
.../hadoop/http/TestSSLHttpServerConfigs.java | 52 +-
.../org/apache/hadoop/http/TestServletFilter.java | 2 +-
.../hadoop/http/lib/TestStaticUserWebFilter.java | 4 +-
.../org/apache/hadoop/ipc/MiniRPCBenchmark.java | 5 +-
.../java/org/apache/hadoop/ipc/TestAsyncIPC.java | 206 +-
.../apache/hadoop/ipc/TestCallQueueManager.java | 35 +-
.../org/apache/hadoop/ipc/TestCallerContext.java | 41 +-
.../apache/hadoop/ipc/TestDecayRpcScheduler.java | 38 +-
.../org/apache/hadoop/ipc/TestFairCallQueue.java | 36 +-
.../test/java/org/apache/hadoop/ipc/TestIPC.java | 318 +--
.../apache/hadoop/ipc/TestIPCServerResponder.java | 28 +-
.../apache/hadoop/ipc/TestIdentityProviders.java | 6 +-
.../apache/hadoop/ipc/TestMiniRPCBenchmark.java | 2 +-
.../hadoop/ipc/TestMultipleProtocolServer.java | 10 +-
.../apache/hadoop/ipc/TestProcessingDetails.java | 4 +-
.../hadoop/ipc/TestProtoBufRPCCompatibility.java | 27 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 77 +-
.../hadoop/ipc/TestProtoBufRpcServerHandoff.java | 23 +-
.../test/java/org/apache/hadoop/ipc/TestRPC.java | 186 +-
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 8 +-
.../apache/hadoop/ipc/TestRPCCompatibility.java | 14 +-
.../apache/hadoop/ipc/TestRPCServerShutdown.java | 22 +-
.../org/apache/hadoop/ipc/TestRPCWaitForProxy.java | 22 +-
.../org/apache/hadoop/ipc/TestResponseBuffer.java | 4 +-
.../java/org/apache/hadoop/ipc/TestRetryCache.java | 25 +-
.../apache/hadoop/ipc/TestRetryCacheMetrics.java | 5 +-
.../apache/hadoop/ipc/TestReuseRpcConnections.java | 21 +-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 8 +-
.../apache/hadoop/ipc/TestRpcServerHandoff.java | 24 +-
.../org/apache/hadoop/ipc/TestRpcWritable.java | 60 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 207 +-
.../java/org/apache/hadoop/ipc/TestServer.java | 21 +-
.../org/apache/hadoop/ipc/TestSocketFactory.java | 24 +-
.../ipc/TestWeightedRoundRobinMultiplexer.java | 30 +-
.../hadoop/ipc/TestWeightedTimeCostProvider.java | 15 +-
.../TestDecayRpcSchedulerDetailedMetrics.java | 6 +-
.../apache/hadoop/ipc/metrics/TestRpcMetrics.java | 6 +-
.../org/apache/hadoop/jmx/TestJMXJsonServlet.java | 18 +-
.../hadoop/jmx/TestJMXJsonServletNaNFiltered.java | 14 +-
.../apache/hadoop/log/TestLogThrottlingHelper.java | 23 +-
.../hadoop/metrics2/filter/TestPatternFilter.java | 34 +-
.../apache/hadoop/metrics2/impl/ConfigUtil.java | 11 +-
.../hadoop/metrics2/impl/TestGangliaMetrics.java | 11 +-
.../metrics2/impl/TestMetricsCollectorImpl.java | 21 +-
.../hadoop/metrics2/impl/TestMetricsConfig.java | 35 +-
.../metrics2/impl/TestMetricsSourceAdapter.java | 13 +-
.../metrics2/impl/TestMetricsSystemImpl.java | 78 +-
.../hadoop/metrics2/impl/TestMetricsVisitor.java | 39 +-
.../apache/hadoop/metrics2/impl/TestSinkQueue.java | 81 +-
.../apache/hadoop/metrics2/lib/TestInterns.java | 25 +-
.../metrics2/lib/TestMetricsAnnotations.java | 35 +-
.../hadoop/metrics2/lib/TestMetricsRegistry.java | 30 +-
.../hadoop/metrics2/lib/TestMutableMetrics.java | 34 +-
.../metrics2/lib/TestMutableRollingAverages.java | 24 +-
.../apache/hadoop/metrics2/lib/TestUniqNames.java | 4 +-
.../apache/hadoop/metrics2/sink/TestFileSink.java | 12 +-
.../hadoop/metrics2/sink/TestGraphiteMetrics.java | 4 +-
.../metrics2/sink/TestPrometheusMetricsSink.java | 57 +-
.../metrics2/sink/TestRollingFileSystemSink.java | 87 +-
.../hadoop/metrics2/sink/TestStatsDMetrics.java | 24 +-
.../metrics2/sink/ganglia/TestGangliaSink.java | 28 +-
.../hadoop/metrics2/source/TestJvmMetrics.java | 50 +-
.../apache/hadoop/metrics2/util/TestMBeans.java | 13 +-
.../hadoop/metrics2/util/TestMetricsCache.java | 45 +-
.../hadoop/metrics2/util/TestSampleQuantiles.java | 6 +-
.../hadoop/metrics2/util/TestSampleStat.java | 52 +-
.../org/apache/hadoop/net/TestClusterTopology.java | 66 +-
.../test/java/org/apache/hadoop/net/TestDNS.java | 39 +-
.../hadoop/net/TestDNSDomainNameResolver.java | 13 +-
.../hadoop/net/TestMockDomainNameResolver.java | 10 +-
.../java/org/apache/hadoop/net/TestNetUtils.java | 54 +-
.../net/TestNetworkTopologyWithNodeGroup.java | 18 +-
.../apache/hadoop/net/TestScriptBasedMapping.java | 21 +-
.../net/TestScriptBasedMappingWithDependency.java | 23 +-
.../apache/hadoop/net/TestSocketIOWithTimeout.java | 8 +-
.../org/apache/hadoop/net/TestStaticMapping.java | 33 +-
.../org/apache/hadoop/net/TestSwitchMapping.java | 32 +-
.../org/apache/hadoop/net/TestTableMapping.java | 8 +-
.../apache/hadoop/net/unix/TestDomainSocket.java | 117 +-
.../hadoop/net/unix/TestDomainSocketWatcher.java | 36 +-
.../org/apache/hadoop/oncrpc/TestFrameDecoder.java | 19 +-
.../apache/hadoop/oncrpc/TestRpcAcceptedReply.java | 10 +-
.../java/org/apache/hadoop/oncrpc/TestRpcCall.java | 22 +-
.../org/apache/hadoop/oncrpc/TestRpcCallCache.java | 30 +-
.../apache/hadoop/oncrpc/TestRpcDeniedReply.java | 23 +-
.../org/apache/hadoop/oncrpc/TestRpcMessage.java | 18 +-
.../org/apache/hadoop/oncrpc/TestRpcReply.java | 20 +-
.../java/org/apache/hadoop/oncrpc/TestXDR.java | 10 +-
.../hadoop/oncrpc/security/TestCredentialsSys.java | 4 +-
.../hadoop/oncrpc/security/TestRpcAuthInfo.java | 10 +-
.../org/apache/hadoop/portmap/TestPortmap.java | 29 +-
.../hadoop/security/ManualTestKeytabLogins.java | 2 +-
.../hadoop/security/TestAuthenticationFilter.java | 6 +-
.../hadoop/security/TestCompositeGroupMapping.java | 4 +-
.../apache/hadoop/security/TestCredentials.java | 90 +-
.../hadoop/security/TestDoAsEffectiveUser.java | 39 +-
.../apache/hadoop/security/TestGroupFallback.java | 4 +-
.../apache/hadoop/security/TestGroupsCaching.java | 32 +-
.../TestHttpCrossOriginFilterInitializer.java | 15 +-
.../security/TestIngressPortBasedResolver.java | 5 +-
.../hadoop/security/TestJNIGroupsMapping.java | 8 +-
.../java/org/apache/hadoop/security/TestKDiag.java | 34 +-
.../org/apache/hadoop/security/TestKDiagNoKDC.java | 23 +-
.../hadoop/security/TestLdapGroupsMapping.java | 46 +-
.../hadoop/security/TestLdapGroupsMappingBase.java | 6 +-
.../TestLdapGroupsMappingWithBindUserSwitch.java | 10 +-
.../TestLdapGroupsMappingWithFailover.java | 4 +-
.../TestLdapGroupsMappingWithOneQuery.java | 18 +-
.../TestLdapGroupsMappingWithPosixGroup.java | 13 +-
.../apache/hadoop/security/TestNetgroupCache.java | 10 +-
.../hadoop/security/TestNullGroupsMapping.java | 17 +-
.../hadoop/security/TestProxyUserFromEnv.java | 6 +-
.../security/TestRuleBasedLdapGroupsMapping.java | 29 +-
.../apache/hadoop/security/TestSecurityUtil.java | 18 +-
.../hadoop/security/TestShellBasedIdMapping.java | 6 +-
.../security/TestShellBasedUnixGroupsMapping.java | 77 +-
.../hadoop/security/TestUGILoginFromKeytab.java | 176 +-
.../hadoop/security/TestUGIWithExternalKdc.java | 22 +-
.../apache/hadoop/security/TestUGIWithMiniKdc.java | 10 +-
.../apache/hadoop/security/TestUserFromEnv.java | 7 +-
.../security/TestWhitelistBasedResolver.java | 4 +-
.../hadoop/security/alias/TestCredShell.java | 40 +-
.../security/alias/TestCredentialProvider.java | 6 +-
.../alias/TestCredentialProviderFactory.java | 50 +-
.../server/TestProxyUserAuthenticationFilter.java | 6 +-
.../security/authorize/TestAccessControlList.java | 23 +-
.../TestDefaultImpersonationProvider.java | 10 +-
.../security/authorize/TestProxyServers.java | 6 +-
.../hadoop/security/authorize/TestProxyUsers.java | 82 +-
.../authorize/TestServiceAuthorization.java | 6 +-
.../security/http/TestCrossOriginFilter.java | 177 +-
.../http/TestRestCsrfPreventionFilter.java | 2 +-
.../security/http/TestXFrameOptionsFilter.java | 30 +-
.../ssl/TestDelegatingSSLSocketFactory.java | 2 +-
.../security/ssl/TestReloadingX509KeyManager.java | 337 +--
.../ssl/TestReloadingX509TrustManager.java | 46 +-
.../apache/hadoop/security/ssl/TestSSLFactory.java | 110 +-
.../hadoop/security/token/TestDtUtilShell.java | 178 +-
.../apache/hadoop/security/token/TestToken.java | 4 +-
.../token/delegation/TestDelegationToken.java | 50 +-
.../delegation/web/TestDelegationTokenManager.java | 29 +-
.../delegation/web/TestWebDelegationToken.java | 189 +-
.../hadoop/service/TestCompositeService.java | 248 +-
.../hadoop/service/TestServiceOperations.java | 8 +-
.../launcher/AbstractServiceLauncherTestBase.java | 36 +-
.../hadoop/service/launcher/TestServiceConf.java | 13 +-
.../launcher/TestServiceInterruptHandling.java | 14 +-
.../service/launcher/TestServiceLauncher.java | 2 +-
.../TestServiceLauncherCreationFailures.java | 2 +-
.../launcher/TestServiceLauncherInnerMethods.java | 6 +-
.../InitInConstructorLaunchableService.java | 14 +-
.../testservices/LaunchableRunningService.java | 4 +-
.../apache/hadoop/test/TestGenericTestUtils.java | 15 +-
.../org/apache/hadoop/test/TestJUnitSetup.java | 7 +-
.../apache/hadoop/test/TestLambdaTestUtils.java | 18 +-
.../hadoop/test/TestMultithreadedTestUtil.java | 23 +-
.../test/java/org/apache/hadoop/test/TestName.java | 39 +
.../hadoop/test/TestTimedOutTestsListener.java | 21 +-
.../apache/hadoop/test/TimedOutTestsListener.java | 7 +-
.../apache/hadoop/test/UnitTestcaseTimeLimit.java | 9 +-
.../org/apache/hadoop/tools/TestCommandShell.java | 28 +-
.../hadoop/util/TestApplicationClassLoader.java | 32 +-
.../apache/hadoop/util/TestAsyncDiskService.java | 10 +-
.../apache/hadoop/util/TestAutoCloseableLock.java | 8 +-
.../apache/hadoop/util/TestBasicDiskValidator.java | 4 +-
.../apache/hadoop/util/TestCacheableIPList.java | 53 +-
.../apache/hadoop/util/TestChunkedArrayList.java | 49 +-
.../java/org/apache/hadoop/util/TestClassUtil.java | 29 +-
.../java/org/apache/hadoop/util/TestClasspath.java | 16 +-
.../java/org/apache/hadoop/util/TestConfTest.java | 5 +-
.../hadoop/util/TestConfigurationHelper.java | 70 +-
.../org/apache/hadoop/util/TestCpuTimeTracker.java | 16 +-
.../org/apache/hadoop/util/TestCrcComposer.java | 17 +-
.../java/org/apache/hadoop/util/TestCrcUtil.java | 22 +-
.../org/apache/hadoop/util/TestDataChecksum.java | 7 +-
.../apache/hadoop/util/TestDirectBufferPool.java | 8 +-
.../org/apache/hadoop/util/TestDiskChecker.java | 63 +-
.../hadoop/util/TestDiskCheckerWithDiskIo.java | 42 +-
.../hadoop/util/TestDiskValidatorFactory.java | 23 +-
.../org/apache/hadoop/util/TestDurationInfo.java | 25 +-
.../java/org/apache/hadoop/util/TestExitUtil.java | 102 +-
.../apache/hadoop/util/TestFastNumberFormat.java | 13 +-
.../apache/hadoop/util/TestFileBasedIPList.java | 79 +-
.../java/org/apache/hadoop/util/TestFindClass.java | 6 +-
.../test/java/org/apache/hadoop/util/TestGSet.java | 66 +-
.../hadoop/util/TestGenericOptionsParser.java | 79 +-
.../org/apache/hadoop/util/TestGenericsUtil.java | 36 +-
.../apache/hadoop/util/TestHostsFileReader.java | 65 +-
.../apache/hadoop/util/TestHttpExceptionUtils.java | 81 +-
.../apache/hadoop/util/TestIdentityHashStore.java | 64 +-
.../org/apache/hadoop/util/TestIndexedSort.java | 44 +-
.../apache/hadoop/util/TestInstrumentedLock.java | 43 +-
.../hadoop/util/TestInstrumentedReadWriteLock.java | 57 +-
.../java/org/apache/hadoop/util/TestJarFinder.java | 14 +-
.../apache/hadoop/util/TestLightWeightCache.java | 32 +-
.../apache/hadoop/util/TestLightWeightGSet.java | 17 +-
.../hadoop/util/TestLightWeightResizableGSet.java | 9 +-
.../org/apache/hadoop/util/TestLineReader.java | 27 +-
.../java/org/apache/hadoop/util/TestLists.java | 61 +-
.../org/apache/hadoop/util/TestMachineList.java | 27 +-
.../apache/hadoop/util/TestNativeCodeLoader.java | 5 +-
.../org/apache/hadoop/util/TestNativeCrc32.java | 103 +-
.../hadoop/util/TestNativeLibraryChecker.java | 5 +-
.../java/org/apache/hadoop/util/TestOptions.java | 19 +-
.../org/apache/hadoop/util/TestPreconditions.java | 2 +-
.../java/org/apache/hadoop/util/TestProgress.java | 15 +-
.../java/org/apache/hadoop/util/TestProtoUtil.java | 6 +-
.../org/apache/hadoop/util/TestPureJavaCrc32.java | 7 +-
.../org/apache/hadoop/util/TestPureJavaCrc32C.java | 6 +-
.../hadoop/util/TestReadWriteDiskValidator.java | 30 +-
.../apache/hadoop/util/TestReflectionUtils.java | 20 +-
.../java/org/apache/hadoop/util/TestRunJar.java | 69 +-
.../java/org/apache/hadoop/util/TestShell.java | 59 +-
.../hadoop/util/TestShutdownHookManager.java | 82 +-
.../hadoop/util/TestShutdownThreadsHelper.java | 18 +-
.../org/apache/hadoop/util/TestSignalLogger.java | 16 +-
.../java/org/apache/hadoop/util/TestStopWatch.java | 20 +-
.../org/apache/hadoop/util/TestStringInterner.java | 5 +-
.../org/apache/hadoop/util/TestStringUtils.java | 125 +-
.../org/apache/hadoop/util/TestSysInfoLinux.java | 6 +-
.../org/apache/hadoop/util/TestSysInfoWindows.java | 17 +-
.../test/java/org/apache/hadoop/util/TestTime.java | 4 +-
.../org/apache/hadoop/util/TestVersionUtil.java | 5 +-
.../apache/hadoop/util/TestWeakReferenceMap.java | 38 +-
.../java/org/apache/hadoop/util/TestWinUtils.java | 95 +-
.../java/org/apache/hadoop/util/TestXMLUtils.java | 107 +-
.../java/org/apache/hadoop/util/TestZKUtil.java | 9 +-
.../hadoop/util/bloom/BloomFilterCommonTester.java | 108 +-
.../apache/hadoop/util/bloom/TestBloomFilters.java | 39 +-
.../util/curator/TestSecureZKCuratorManager.java | 75 +-
.../hadoop/util/curator/TestZKCuratorManager.java | 31 +-
.../hadoop/util/dynamic/TestDynConstructors.java | 31 +-
.../apache/hadoop/util/dynamic/TestDynMethods.java | 117 +-
.../hadoop/util/functional/TestFunctionalIO.java | 2 +-
.../hadoop/util/functional/TestLazyReferences.java | 2 +-
.../util/functional/TestRemoteIterators.java | 2 +-
.../java/org/apache/hadoop/util/hash/TestHash.java | 61 +-
.../apache/hadoop/hdfs/DFSStripedOutputStream.java | 7 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 10 +
.../ClientNamenodeProtocolTranslatorPB.java | 28 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 20 +
.../hadoop/fs/http/client/BaseTestHttpFSWith.java | 287 ++-
.../client/TestHttpFSFWithSWebhdfsFileSystem.java | 10 +-
.../client/TestHttpFSFWithWebhdfsFileSystem.java | 6 +-
.../TestHttpFSFileSystemLocalFileSystem.java | 11 +-
.../client/TestHttpFSWithHttpFSFileSystem.java | 6 +-
.../server/TestCheckUploadContentTypeFilter.java | 2 +-
.../fs/http/server/TestHttpFSAccessControlled.java | 28 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 351 +--
.../fs/http/server/TestHttpFSServerNoACLs.java | 32 +-
.../fs/http/server/TestHttpFSServerNoXAttrs.java | 28 +-
.../fs/http/server/TestHttpFSServerWebServer.java | 30 +-
.../fs/http/server/TestHttpFSWithKerberos.java | 37 +-
.../hadoop/lib/lang/TestRunnableCallable.java | 17 +-
.../org/apache/hadoop/lib/lang/TestXException.java | 6 +-
.../apache/hadoop/lib/server/TestBaseService.java | 8 +-
.../org/apache/hadoop/lib/server/TestServer.java | 80 +-
.../hadoop/lib/server/TestServerConstructor.java | 37 +-
.../hadoop/TestFileSystemAccessService.java | 45 +-
.../TestInstrumentationService.java | 10 +-
.../service/scheduler/TestSchedulerService.java | 4 +-
.../lib/service/security/TestGroupsService.java | 24 +-
.../hadoop/lib/servlet/TestHostnameFilter.java | 6 +-
.../apache/hadoop/lib/servlet/TestMDCFilter.java | 8 +-
.../hadoop/lib/servlet/TestServerWebApp.java | 39 +-
.../java/org/apache/hadoop/lib/util/TestCheck.java | 102 +-
.../hadoop/lib/util/TestConfigurationUtils.java | 6 +-
.../hadoop/lib/wsrs/TestInputStreamEntity.java | 4 +-
.../hadoop/lib/wsrs/TestJSONMapProvider.java | 8 +-
.../apache/hadoop/lib/wsrs/TestJSONProvider.java | 8 +-
.../java/org/apache/hadoop/lib/wsrs/TestParam.java | 6 +-
.../java/org/apache/hadoop/test/HFSTestCase.java | 7 +-
.../java/org/apache/hadoop/test/HTestCase.java | 17 +-
.../java/org/apache/hadoop/test/TestDirHelper.java | 48 +-
.../apache/hadoop/test/TestExceptionHelper.java | 63 +-
.../org/apache/hadoop/test/TestHFSTestCase.java | 39 +-
.../java/org/apache/hadoop/test/TestHTestCase.java | 27 +-
.../org/apache/hadoop/test/TestHdfsHelper.java | 55 +-
.../org/apache/hadoop/test/TestJettyHelper.java | 57 +-
.../src/main/native/libhdfspp/CMakeLists.txt | 2 +-
.../uriparser2/uriparser2/uriparser/UriQuery.c | 1 +
.../src/main/native/libhdfspp/tools/CMakeLists.txt | 2 +-
.../hdfs/protocolPB/AsyncRpcProtocolPBUtil.java | 162 ++
...ientNamenodeProtocolServerSideTranslatorPB.java | 1769 ++++++++++++++
.../RouterClientProtocolTranslatorPB.java} | 1495 +++++++-----
...UserMappingsProtocolServerSideTranslatorPB.java | 59 +
.../RouterGetUserMappingsProtocolTranslatorPB.java | 52 +
...terNamenodeProtocolServerSideTranslatorPB.java} | 275 +--
.../RouterNamenodeProtocolTranslatorPB.java | 270 +++
...UserMappingsProtocolServerSideTranslatorPB.java | 83 +
...terRefreshUserMappingsProtocolTranslatorPB.java | 64 +
.../hadoop/hdfs/protocolPB/package-info.java | 19 +-
.../RouterAsyncRpcFairnessPolicyController.java | 91 +
.../metrics/FederationRPCPerformanceMonitor.java | 20 +-
.../federation/metrics/NamenodeBeanMetrics.java | 6 +-
.../hdfs/server/federation/metrics/RBFMetrics.java | 10 +-
.../MultipleDestinationMountTableResolver.java | 7 +-
.../federation/resolver/order/LocalResolver.java | 18 +-
.../server/federation/router/ConnectionPool.java | 16 +-
.../router/MountTableRefresherService.java | 16 +-
.../hdfs/server/federation/router/Quota.java | 11 +-
.../server/federation/router/RBFConfigKeys.java | 22 +
.../hdfs/server/federation/router/Router.java | 2 +-
.../federation/router/RouterAdminServer.java | 6 +-
.../server/federation/router/RouterCacheAdmin.java | 36 +-
.../federation/router/RouterClientProtocol.java | 137 +-
.../federation/router/RouterFederationRename.java | 2 +-
.../federation/router/RouterHeartbeatService.java | 9 +-
.../router/RouterQuotaUpdateService.java | 10 +
.../server/federation/router/RouterRpcClient.java | 577 +++--
.../server/federation/router/RouterRpcServer.java | 510 +++-
.../federation/router/RouterStateIdContext.java | 13 +-
.../federation/router/RouterWebHdfsMethods.java | 12 +-
.../federation/router/ThreadLocalContext.java | 101 +
.../router/async/AsyncErasureCoding.java | 255 ++
.../server/federation/router/async/AsyncQuota.java | 101 +
.../router/async/RouterAsyncCacheAdmin.java | 109 +
.../router/async/RouterAsyncClientProtocol.java | 1167 +++++++++
.../router/async/RouterAsyncNamenodeProtocol.java | 198 ++
.../router/async/RouterAsyncRpcClient.java | 629 +++++
.../router/async/RouterAsyncSnapshot.java | 264 ++
.../router/async/RouterAsyncStoragePolicy.java | 90 +
.../router/async/RouterAsyncUserProtocol.java | 132 +
.../federation/router/async/package-info.java | 37 +-
.../router/async/utils/ApplyFunction.java | 89 +
.../federation/router/async/utils/Async.java | 115 +
.../router/async/utils/AsyncApplyFunction.java | 162 ++
.../router/async/utils/AsyncBiFunction.java | 83 +
.../router/async/utils/AsyncCatchFunction.java | 174 ++
.../router/async/utils/AsyncForEachRun.java | 185 ++
.../federation/router/async/utils/AsyncRun.java | 74 +
.../federation/router/async/utils/AsyncUtil.java | 411 ++++
.../router/async/utils/CatchFunction.java | 120 +
.../router/async/utils/FinallyFunction.java | 96 +
.../router/async/utils/package-info.java | 35 +
.../server/federation/store/StateStoreUtils.java | 16 +
.../src/main/resources/hdfs-rbf-default.xml | 68 +
.../src/site/markdown/HDFSRouterFederation.md | 34 +-
.../fs/contract/router/RouterHDFSContract.java | 4 +-
.../fs/contract/router/SecurityConfUtil.java | 6 +-
.../contract/router/web/RouterWebHDFSContract.java | 4 +-
.../protocolPB/TestAsyncRpcProtocolPBUtil.java | 123 +
.../hdfs/protocolPB/TestClientProtocol.java} | 21 +-
.../TestClientProtocolServerSideTranslatorPB.java | 88 +
.../protocolPB/TestClientProtocolTranslatorPB.java | 80 +
.../TestRouterClientSideTranslatorPB.java | 244 ++
.../hdfs/rbfbalance/TestMountTableProcedure.java | 20 +-
.../server/federation/FederationTestUtils.java | 17 +-
.../server/federation/MiniRouterDFSCluster.java | 4 +-
.../hdfs/server/federation/MockResolver.java | 1 +
...roportionRouterRpcFairnessPolicyController.java | 6 +-
...estRouterAsyncRpcFairnessPolicyController.java} | 143 +-
.../fairness/TestRouterHandlersFairness.java | 73 +-
.../TestRouterRefreshFairnessPolicyController.java | 16 +-
.../TestRouterRpcFairnessPolicyController.java | 11 +-
.../server/federation/metrics/TestMetricsBase.java | 16 +-
.../metrics/TestNameserviceRPCMetrics.java | 14 +-
.../server/federation/metrics/TestRBFMetrics.java | 12 +-
.../metrics/TestRouterClientMetrics.java | 14 +-
.../resolver/TestFederationNamespaceInfo.java | 2 +-
.../resolver/TestInitializeMountTableResolver.java | 10 +-
.../resolver/TestMountTableResolver.java | 16 +-
.../resolver/TestMultipleDestinationResolver.java | 10 +-
.../federation/resolver/TestNamenodeResolver.java | 32 +-
.../resolver/order/TestAvailableSpaceResolver.java | 8 +-
.../resolver/order/TestLeaderFollowerResolver.java | 4 +-
.../resolver/order/TestLocalResolver.java | 4 +-
.../federation/router/TestConnectionManager.java | 38 +-
.../server/federation/router/TestDFSRouter.java | 10 +-
.../federation/router/TestDisableNameservices.java | 24 +-
.../federation/router/TestDisableRouterQuota.java | 18 +-
.../federation/router/TestFederationUtil.java | 4 +-
.../router/TestNoNamenodesAvailableLongTime.java | 12 +-
.../federation/router/TestObserverWithRouter.java | 136 +-
.../router/TestRenewLeaseWithSameINodeId.java | 12 +-
.../hdfs/server/federation/router/TestRouter.java | 14 +-
.../server/federation/router/TestRouterAdmin.java | 109 +-
.../federation/router/TestRouterAdminCLI.java | 152 +-
.../router/TestRouterAdminGenericRefresh.java | 35 +-
.../federation/router/TestRouterAllResolver.java | 28 +-
.../router/TestRouterClientRejectOverload.java | 97 +-
.../federation/router/TestRouterFaultTolerant.java | 60 +-
.../router/TestRouterFederatedState.java | 4 +-
.../router/TestRouterFederationRename.java | 24 +-
.../TestRouterFederationRenameInKerberosEnv.java | 22 +-
.../TestRouterFederationRenamePermission.java | 18 +-
.../server/federation/router/TestRouterFsck.java | 22 +-
.../router/TestRouterHeartbeatService.java | 18 +-
.../router/TestRouterHttpServerXFrame.java | 14 +-
.../router/TestRouterMissingFolderMulti.java | 12 +-
.../federation/router/TestRouterMountTable.java | 96 +-
.../router/TestRouterMountTableCacheRefresh.java | 102 +-
.../TestRouterMountTableCacheRefreshSecure.java | 23 +-
.../TestRouterMountTableWithoutDefaultNS.java | 22 +-
.../federation/router/TestRouterMultiRack.java | 14 +-
.../router/TestRouterNamenodeHeartbeat.java | 64 +-
.../router/TestRouterNamenodeMonitoring.java | 28 +-
.../router/TestRouterNamenodeWebScheme.java | 16 +-
.../router/TestRouterNetworkTopologyServlet.java | 10 +-
.../router/TestRouterPolicyProvider.java | 41 +-
.../server/federation/router/TestRouterQuota.java | 22 +-
.../federation/router/TestRouterQuotaManager.java | 16 +-
.../router/TestRouterRPCClientRetries.java | 25 +-
...erRPCMultipleDestinationMountTableResolver.java | 69 +-
...tRouterRefreshSuperUserGroupsConfiguration.java | 14 +-
.../federation/router/TestRouterRetryCache.java | 16 +-
.../server/federation/router/TestRouterRpc.java | 89 +-
.../router/TestRouterRpcMultiDestination.java | 36 +-
.../federation/router/TestRouterRpcSingleNS.java | 18 +-
.../TestRouterRpcStoragePolicySatisfier.java | 12 +-
.../federation/router/TestRouterSafemode.java | 32 +-
.../server/federation/router/TestRouterTrash.java | 24 +-
.../federation/router/TestRouterUserMappings.java | 30 +-
.../router/TestRouterWebHdfsMethods.java | 21 +-
.../router/TestRouterWithSecureStartup.java | 16 +-
.../server/federation/router/TestSafeMode.java | 10 +-
.../router/async/RouterAsyncProtocolTestBase.java | 165 ++
.../router/async/TestAsyncRouterAdmin.java | 119 +
.../router/async/TestRouterAsyncCacheAdmin.java | 102 +
.../async/TestRouterAsyncClientProtocol.java | 144 ++
.../router/async/TestRouterAsyncErasureCoding.java | 206 ++
.../router/async/TestRouterAsyncMountTable.java | 67 +
.../async/TestRouterAsyncNamenodeProtocol.java | 126 +
.../router/async/TestRouterAsyncQuota.java | 168 ++
...ncRPCMultipleDestinationMountTableResolver.java | 202 ++
.../router/async/TestRouterAsyncRpc.java | 83 +
.../router/async/TestRouterAsyncRpcClient.java | 319 +++
.../async/TestRouterAsyncRpcMultiDestination.java | 73 +
.../router/async/TestRouterAsyncRpcServer.java | 97 +
.../router/async/TestRouterAsyncSnapshot.java | 115 +
.../router/async/TestRouterAsyncStoragePolicy.java | 66 +
.../router/async/TestRouterAsyncUserProtocol.java | 48 +
.../async/TestRouterAsyncWebHdfsMethods.java | 54 +
.../federation/router/async/utils/AsyncClass.java | 249 ++
.../federation/router/async/utils/BaseClass.java | 66 +
.../federation/router/async/utils/SyncClass.java | 194 ++
.../router/async/utils/TestAsyncUtil.java | 275 +++
.../TestSQLDelegationTokenSecretManagerImpl.java | 86 +-
.../security/TestRouterHttpDelegationToken.java | 19 +-
.../security/TestRouterSecurityManager.java | 44 +-
.../store/FederationStateStoreTestUtils.java | 2 +-
.../federation/store/TestStateStoreBase.java | 14 +-
.../store/TestStateStoreDisabledNameservice.java | 10 +-
.../store/TestStateStoreMembershipState.java | 20 +-
.../federation/store/TestStateStoreMountTable.java | 18 +-
.../store/TestStateStoreRouterState.java | 18 +-
.../store/driver/TestStateStoreDriverBase.java | 40 +-
.../store/driver/TestStateStoreFile.java | 69 +-
.../store/driver/TestStateStoreFileBase.java | 6 +-
.../store/driver/TestStateStoreFileSystem.java | 74 +-
.../store/driver/TestStateStoreMySQL.java | 14 +-
.../federation/store/driver/TestStateStoreZK.java | 20 +-
.../store/records/TestMembershipState.java | 4 +-
.../federation/store/records/TestMountTable.java | 14 +-
.../federation/store/records/TestRouterState.java | 4 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 30 +-
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +
...ientNamenodeProtocolServerSideTranslatorPB.java | 163 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 8 +-
.../hdfs/server/balancer/BalancerMetrics.java | 5 +
.../server/blockmanagement/DatanodeManager.java | 19 +-
.../hdfs/server/common/DataNodeLockManager.java | 9 +-
.../datanode/DataNodeLayoutSubLockStrategy.java | 27 +-
.../hdfs/server/datanode/DataSetLockManager.java | 38 +-
.../server/datanode/DataSetSubLockStrategy.java | 32 +-
.../hadoop/hdfs/server/datanode/DatanodeUtil.java | 34 +-
.../datanode/fsdataset/DataNodeVolumeMetrics.java | 9 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 118 +-
.../src/main/resources/hdfs-default.xml | 9 +
.../org/apache/hadoop/hdfs/TestListFilesInDFS.java | 8 +-
.../hadoop/hdfs/crypto/TestHdfsCryptoStreams.java | 16 +-
.../hdfs/server/balancer/TestBalancerService.java | 13 +
.../server/datanode/TestDataNodeVolumeMetrics.java | 4 +
.../server/datanode/TestDataSetLockManager.java | 11 +
.../datanode/fsdataset/impl/TestFsDatasetImpl.java | 7 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 11 +-
.../org/apache/hadoop/net/TestNetworkTopology.java | 36 +
.../apache/hadoop/hdfs/TestDFSClientFailover.java | 0
.../hadoop-mapreduce-client-app/pom.xml | 9 +
.../apache/hadoop/mapred/MapTaskAttemptImpl.java | 2 +-
.../hadoop/mapred/ReduceTaskAttemptImpl.java | 2 +-
.../apache/hadoop/mapreduce/v2/app/AppContext.java | 2 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 4 +-
.../v2/app/TaskAttemptFinishingMonitor.java | 2 +-
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 2 +-
.../hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 +-
.../mapreduce/v2/app/job/impl/MapTaskImpl.java | 2 +-
.../mapreduce/v2/app/job/impl/ReduceTaskImpl.java | 2 +-
.../mapreduce/v2/app/job/impl/TaskAttemptImpl.java | 2 +-
.../hadoop/mapreduce/v2/app/job/impl/TaskImpl.java | 2 +-
.../mapreduce/v2/app/rm/RMContainerAllocator.java | 2 +-
.../v2/app/speculate/DefaultSpeculator.java | 2 +-
.../hadoop/mapred/TestLocalContainerLauncher.java | 21 +-
.../mapred/TestTaskAttemptFinishingMonitor.java | 8 +-
.../hadoop/mapred/TestTaskAttemptListenerImpl.java | 46 +-
.../org/apache/hadoop/mapred/TestYarnChild.java | 13 +-
.../hadoop/mapreduce/jobhistory/TestEvents.java | 22 +-
.../jobhistory/TestJobHistoryEventHandler.java | 268 +-
.../mapreduce/jobhistory/TestJobSummary.java | 15 +-
.../v2/api/records/TestTaskAttemptReport.java | 8 +-
.../mapreduce/v2/api/records/TestTaskReport.java | 8 +-
.../org/apache/hadoop/mapreduce/v2/app/MRApp.java | 77 +-
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 13 +-
.../hadoop/mapreduce/v2/app/MockAppContext.java | 2 +-
.../hadoop/mapreduce/v2/app/TestAMInfos.java | 14 +-
.../v2/app/TestCheckpointPreemptionPolicy.java | 11 +-
.../apache/hadoop/mapreduce/v2/app/TestFail.java | 49 +-
.../hadoop/mapreduce/v2/app/TestFetchFailure.java | 160 +-
.../mapreduce/v2/app/TestJobEndNotifier.java | 130 +-
.../apache/hadoop/mapreduce/v2/app/TestKill.java | 85 +-
.../v2/app/TestKillAMPreemptionPolicy.java | 2 +-
.../apache/hadoop/mapreduce/v2/app/TestMRApp.java | 123 +-
.../v2/app/TestMRAppComponentDependencies.java | 16 +-
.../hadoop/mapreduce/v2/app/TestMRAppMaster.java | 92 +-
.../mapreduce/v2/app/TestMRClientService.java | 96 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 207 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 45 +-
.../mapreduce/v2/app/TestStagingCleanup.java | 44 +-
.../mapreduce/v2/app/TestTaskHeartbeatHandler.java | 29 +-
.../v2/app/commit/TestCommitterEventHandler.java | 45 +-
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 172 +-
.../v2/app/job/impl/TestMapReduceChildJVM.java | 92 +-
.../v2/app/job/impl/TestShuffleProvider.java | 16 +-
.../mapreduce/v2/app/job/impl/TestTaskAttempt.java | 319 ++-
.../job/impl/TestTaskAttemptContainerRequest.java | 32 +-
.../mapreduce/v2/app/job/impl/TestTaskImpl.java | 32 +-
.../v2/app/launcher/TestContainerLauncher.java | 65 +-
.../v2/app/launcher/TestContainerLauncherImpl.java | 19 +-
.../v2/app/local/TestLocalContainerAllocator.java | 31 +-
.../mapreduce/v2/app/metrics/TestMRAppMetrics.java | 9 +-
.../mapreduce/v2/app/rm/TestRMCommunicator.java | 11 +-
.../v2/app/rm/TestRMContainerAllocator.java | 515 ++--
.../v2/app/rm/TestResourceCalculatorUtils.java | 21 +-
.../v2/app/speculate/TestDataStatistics.java | 49 +-
.../forecast/TestSimpleExponentialForecast.java | 14 +-
.../mapreduce/v2/app/webapp/TestAMWebApp.java | 56 +-
.../mapreduce/v2/app/webapp/TestAMWebServices.java | 47 +-
.../v2/app/webapp/TestAMWebServicesAttempt.java | 13 +-
.../v2/app/webapp/TestAMWebServicesAttempts.java | 88 +-
.../v2/app/webapp/TestAMWebServicesJobConf.java | 41 +-
.../v2/app/webapp/TestAMWebServicesJobs.java | 161 +-
.../v2/app/webapp/TestAMWebServicesTasks.java | 92 +-
.../mapreduce/v2/app/webapp/TestAppController.java | 18 +-
.../hadoop/mapreduce/v2/app/webapp/TestBlocks.java | 10 +-
.../hadoop-mapreduce-client-core/pom.xml | 10 +
.../jobhistory/MapAttemptFinishedEvent.java | 2 +-
.../jobhistory/ReduceAttemptFinishedEvent.java | 2 +-
.../jobhistory/TaskAttemptFinishedEvent.java | 2 +-
.../TaskAttemptUnsuccessfulCompletionEvent.java | 2 +-
.../mapreduce/jobhistory/TaskFailedEvent.java | 2 +-
.../mapreduce/jobhistory/TaskFinishedEvent.java | 2 +-
.../apache/hadoop/mapred/TestClusterStatus.java | 20 +-
.../org/apache/hadoop/mapred/TestCounters.java | 90 +-
.../apache/hadoop/mapred/TestFileInputFormat.java | 116 +-
.../hadoop/mapred/TestFileOutputCommitter.java | 43 +-
.../org/apache/hadoop/mapred/TestIndexCache.java | 10 +-
.../apache/hadoop/mapred/TestJobAclsManager.java | 22 +-
.../java/org/apache/hadoop/mapred/TestJobConf.java | 91 +-
.../apache/hadoop/mapred/TestJobEndNotifier.java | 21 +-
.../java/org/apache/hadoop/mapred/TestJobInfo.java | 15 +-
.../apache/hadoop/mapred/TestJobQueueClient.java | 9 +-
.../apache/hadoop/mapred/TestLineRecordReader.java | 131 +-
.../mapred/TestLocatedFileStatusFetcher.java | 19 +-
.../hadoop/mapred/TestMapFileOutputFormat.java | 10 +-
.../java/org/apache/hadoop/mapred/TestMapTask.java | 71 +-
.../java/org/apache/hadoop/mapred/TestMaster.java | 4 +-
.../apache/hadoop/mapred/TestOldMethodsJobID.java | 32 +-
.../java/org/apache/hadoop/mapred/TestQueue.java | 28 +-
.../apache/hadoop/mapred/TestSkipBadRecords.java | 12 +-
.../java/org/apache/hadoop/mapred/TestTask.java | 22 +-
.../java/org/apache/hadoop/mapred/TestTaskLog.java | 15 +-
.../hadoop/mapred/TestTaskProgressReporter.java | 29 +-
.../mapred/lib/TestCombineFileRecordReader.java | 6 +-
.../hadoop/mapred/lib/db/TestDBInputFormat.java | 29 +-
.../org/apache/hadoop/mapreduce/TestCluster.java | 10 +-
.../hadoop/mapreduce/TestContextFactory.java | 8 +-
.../java/org/apache/hadoop/mapreduce/TestJob.java | 23 +-
.../hadoop/mapreduce/TestJobMonitorAndPrint.java | 15 +-
.../hadoop/mapreduce/TestJobResourceUploader.java | 73 +-
.../TestJobResourceUploaderWithSharedCache.java | 16 +-
.../hadoop/mapreduce/TestJobSubmissionFiles.java | 39 +-
.../apache/hadoop/mapreduce/TestShufflePlugin.java | 17 +-
.../org/apache/hadoop/mapreduce/TestTaskID.java | 218 +-
.../mapreduce/checkpoint/TestFSCheckpointID.java | 2 +-
.../checkpoint/TestFSCheckpointService.java | 16 +-
.../TestClientDistributedCacheManager.java | 79 +-
.../mapreduce/filecache/TestDistributedCache.java | 78 +-
.../jobhistory/TestHistoryViewerPrinter.java | 32 +-
.../hadoop/mapreduce/lib/db/DriverForTest.java | 5 +-
.../hadoop/mapreduce/lib/db/TestDbClasses.java | 26 +-
.../hadoop/mapreduce/lib/db/TestSplitters.java | 32 +-
.../lib/input/BaseTestLineRecordReaderBZip2.java | 14 +-
.../lib/input/TestCombineFileRecordReader.java | 6 +-
.../mapreduce/lib/input/TestFileInputFormat.java | 179 +-
.../mapreduce/lib/input/TestLineRecordReader.java | 133 +-
.../mapreduce/lib/jobcontrol/TestJobControl.java | 4 +-
.../lib/output/TestFileOutputCommitter.java | 75 +-
.../mapreduce/lib/output/TestFileOutputFormat.java | 5 +-
.../lib/output/TestMapFileOutputFormat.java | 8 +-
.../lib/output/TestPathOutputCommitter.java | 6 +-
.../lib/output/TestPathOutputCommitterFactory.java | 30 +-
.../output/TestPreemptableFileOutputCommitter.java | 12 +-
.../lib/partition/TestRehashPartitioner.java | 6 +-
.../hadoop/mapreduce/security/TestTokenCache.java | 19 +-
.../hadoop/mapreduce/split/TestJobSplitWriter.java | 16 +-
.../mapreduce/split/TestJobSplitWriterWithEC.java | 17 +-
.../mapreduce/task/reduce/TestEventFetcher.java | 2 +-
.../hadoop/mapreduce/task/reduce/TestFetcher.java | 86 +-
.../mapreduce/task/reduce/TestMergeManager.java | 106 +-
.../hadoop/mapreduce/task/reduce/TestMerger.java | 73 +-
.../task/reduce/TestShuffleClientMetrics.java | 4 +-
.../task/reduce/TestShuffleScheduler.java | 36 +-
.../org/apache/hadoop/mapreduce/tools/TestCLI.java | 40 +-
.../hadoop-mapreduce-client-hs/pom.xml | 5 +
.../hadoop/mapreduce/v2/hs/HistoryFileManager.java | 4 +-
.../apache/hadoop/mapreduce/v2/hs/JobHistory.java | 2 +-
.../hadoop/mapreduce/v2/hs/TestCompletedTask.java | 39 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 140 +-
...estHistoryFileManagerInitWithNonRunningDFS.java | 15 +-
...stHistoryServerFileSystemStateStoreService.java | 87 +-
.../TestHistoryServerLeveldbStateStoreService.java | 84 +-
.../v2/hs/TestJHSDelegationTokenSecretManager.java | 52 +-
.../hadoop/mapreduce/v2/hs/TestJobHistory.java | 16 +-
.../mapreduce/v2/hs/TestJobHistoryEntities.java | 67 +-
.../mapreduce/v2/hs/TestJobHistoryEvents.java | 74 +-
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 279 ++-
.../mapreduce/v2/hs/TestJobHistoryServer.java | 31 +-
.../v2/hs/TestJobIdHistoryFileInfoMap.java | 30 +-
.../hadoop/mapreduce/v2/hs/TestJobListCache.java | 19 +-
.../TestUnnecessaryBlockingOnHistoryFileInfo.java | 22 +-
.../mapreduce/v2/hs/server/TestHSAdminServer.java | 89 +-
.../hadoop/mapreduce/v2/hs/webapp/TestBlocks.java | 27 +-
.../mapreduce/v2/hs/webapp/TestHSWebApp.java | 18 +-
.../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 32 +-
.../mapreduce/v2/hs/webapp/TestHsWebServices.java | 37 +-
.../v2/hs/webapp/TestHsWebServicesAcls.java | 10 +-
.../v2/hs/webapp/TestHsWebServicesAttempts.java | 85 +-
.../v2/hs/webapp/TestHsWebServicesJobConf.java | 34 +-
.../v2/hs/webapp/TestHsWebServicesJobs.java | 104 +-
.../v2/hs/webapp/TestHsWebServicesJobsQuery.java | 88 +-
.../v2/hs/webapp/TestHsWebServicesLogs.java | 12 +-
.../v2/hs/webapp/TestHsWebServicesLogsExtend.java | 16 +-
.../v2/hs/webapp/TestHsWebServicesTasks.java | 91 +-
.../mapreduce/v2/hs/webapp/VerifyJobsUtils.java | 50 +-
.../mapreduce/v2/hs/webapp/dao/TestJobInfo.java | 26 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 5 +
.../apache/hadoop/conf/TestNoDefaultsJobConf.java | 12 +-
.../test/java/org/apache/hadoop/fs/DFSCIOTest.java | 6 +-
.../test/java/org/apache/hadoop/fs/TestDFSIO.java | 29 +-
.../java/org/apache/hadoop/fs/TestFileSystem.java | 12 +-
.../test/java/org/apache/hadoop/fs/TestJHLA.java | 10 +-
.../java/org/apache/hadoop/fs/slive/TestSlive.java | 10 +-
.../java/org/apache/hadoop/hdfs/TestNNBench.java | 44 +-
.../hadoop/io/TestSequenceFileMergeProgress.java | 4 +-
.../apache/hadoop/ipc/TestMRCJCSocketFactory.java | 20 +-
.../hadoop/mapred/ClusterMapReduceTestCase.java | 8 +-
.../org/apache/hadoop/mapred/HadoopTestCase.java | 8 +-
.../java/org/apache/hadoop/mapred/MRCaching.java | 7 +-
.../org/apache/hadoop/mapred/MiniMRCluster.java | 4 +-
.../apache/hadoop/mapred/NotificationTestCase.java | 16 +-
.../org/apache/hadoop/mapred/TestBadRecords.java | 17 +-
.../apache/hadoop/mapred/TestClientRedirect.java | 16 +-
.../hadoop/mapred/TestClientServiceDelegate.java | 200 +-
.../mapred/TestClusterMapReduceTestCase.java | 14 +-
.../java/org/apache/hadoop/mapred/TestCollect.java | 2 +-
.../hadoop/mapred/TestCombineFileInputFormat.java | 6 +-
.../hadoop/mapred/TestCombineOutputCollector.java | 2 +-
.../mapred/TestCombineSequenceFileInputFormat.java | 20 +-
.../hadoop/mapred/TestCombineTextInputFormat.java | 33 +-
.../mapred/TestCommandLineJobSubmission.java | 16 +-
.../org/apache/hadoop/mapred/TestComparators.java | 14 +-
.../mapred/TestConcatenatedCompressedInput.java | 143 +-
.../apache/hadoop/mapred/TestFieldSelection.java | 4 +-
.../mapred/TestFileInputFormatPathFilter.java | 12 +-
.../apache/hadoop/mapred/TestFileOutputFormat.java | 4 +-
.../hadoop/mapred/TestFixedLengthInputFormat.java | 78 +-
.../apache/hadoop/mapred/TestGetSplitHosts.java | 4 +-
.../java/org/apache/hadoop/mapred/TestIFile.java | 5 +-
.../org/apache/hadoop/mapred/TestIFileStreams.java | 10 +-
.../org/apache/hadoop/mapred/TestInputPath.java | 4 +-
.../hadoop/mapred/TestJavaSerialization.java | 29 +-
.../org/apache/hadoop/mapred/TestJobCleanup.java | 43 +-
.../org/apache/hadoop/mapred/TestJobClients.java | 12 +-
.../org/apache/hadoop/mapred/TestJobCounters.java | 38 +-
.../java/org/apache/hadoop/mapred/TestJobName.java | 10 +-
.../apache/hadoop/mapred/TestJobSysDirWithDFS.java | 8 +-
.../hadoop/mapred/TestKeyValueTextInputFormat.java | 52 +-
.../org/apache/hadoop/mapred/TestLazyOutput.java | 4 +-
.../hadoop/mapred/TestLineRecordReaderJobs.java | 4 +-
.../hadoop/mapred/TestLocalJobSubmission.java | 36 +-
.../hadoop/mapred/TestMRCJCFileInputFormat.java | 22 +-
.../mapred/TestMRCJCFileOutputCommitter.java | 26 +-
.../apache/hadoop/mapred/TestMRCJCJobClient.java | 9 +-
.../org/apache/hadoop/mapred/TestMRCJCJobConf.java | 10 +-
.../hadoop/mapred/TestMROpportunisticMaps.java | 4 +-
.../hadoop/mapred/TestMRTimelineEventHandling.java | 137 +-
.../apache/hadoop/mapred/TestMapOutputType.java | 12 +-
.../org/apache/hadoop/mapred/TestMapProgress.java | 8 +-
.../java/org/apache/hadoop/mapred/TestMapRed.java | 36 +-
.../java/org/apache/hadoop/mapred/TestMerge.java | 4 +-
.../apache/hadoop/mapred/TestMiniMRBringup.java | 7 +-
.../apache/hadoop/mapred/TestMiniMRChildTask.java | 43 +-
.../apache/hadoop/mapred/TestMiniMRClasspath.java | 9 +-
.../hadoop/mapred/TestMiniMRClientCluster.java | 71 +-
.../apache/hadoop/mapred/TestMiniMRDFSCaching.java | 12 +-
.../mapred/TestMiniMRWithDFSWithDistinctUsers.java | 15 +-
.../hadoop/mapred/TestMultiFileInputFormat.java | 6 +-
.../apache/hadoop/mapred/TestMultiFileSplit.java | 6 +-
.../hadoop/mapred/TestMultipleLevelCaching.java | 22 +-
.../mapred/TestMultipleTextOutputFormat.java | 4 +-
.../org/apache/hadoop/mapred/TestNetworkedJob.java | 39 +-
.../hadoop/mapred/TestOldCombinerGrouping.java | 27 +-
.../mapred/TestQueueConfigurationParser.java | 9 +-
.../org/apache/hadoop/mapred/TestReduceFetch.java | 17 +-
.../mapred/TestReduceFetchFromPartialMem.java | 27 +-
.../org/apache/hadoop/mapred/TestReduceTask.java | 6 +-
.../org/apache/hadoop/mapred/TestReporter.java | 38 +-
.../hadoop/mapred/TestResourceMgrDelegate.java | 72 +-
.../TestSequenceFileAsBinaryInputFormat.java | 20 +-
.../TestSequenceFileAsBinaryOutputFormat.java | 43 +-
.../mapred/TestSequenceFileAsTextInputFormat.java | 13 +-
.../hadoop/mapred/TestSequenceFileInputFilter.java | 4 +-
.../hadoop/mapred/TestSequenceFileInputFormat.java | 10 +-
.../org/apache/hadoop/mapred/TestSortedRanges.java | 4 +-
.../mapred/TestSpecialCharactersInOutputPath.java | 9 +-
.../hadoop/mapred/TestStatisticsCollector.java | 8 +-
.../org/apache/hadoop/mapred/TestTaskCommit.java | 32 +-
.../hadoop/mapred/TestTaskPerformanceSplits.java | 36 +-
.../org/apache/hadoop/mapred/TestTaskStatus.java | 89 +-
.../apache/hadoop/mapred/TestTextInputFormat.java | 127 +-
.../apache/hadoop/mapred/TestTextOutputFormat.java | 6 +-
.../hadoop/mapred/TestUserDefinedCounters.java | 6 +-
.../java/org/apache/hadoop/mapred/TestUtils.java | 5 +-
.../apache/hadoop/mapred/TestWritableJobConf.java | 4 +-
.../org/apache/hadoop/mapred/TestYARNRunner.java | 106 +-
.../org/apache/hadoop/mapred/UtilsForTests.java | 5 +-
.../hadoop/mapred/jobcontrol/TestJobControl.java | 37 +-
.../mapred/jobcontrol/TestLocalJobControl.java | 6 +-
.../apache/hadoop/mapred/join/TestDatamerge.java | 39 +-
.../hadoop/mapred/join/TestTupleWritable.java | 46 +-
.../join/TestWrappedRecordReaderClassloader.java | 10 +-
.../org/apache/hadoop/mapred/lib/TestChain.java | 2 +-
.../hadoop/mapred/lib/TestChainMapReduce.java | 12 +-
.../mapred/lib/TestDelegatingInputFormat.java | 7 +-
.../mapred/lib/TestKeyFieldBasedComparator.java | 10 +-
.../mapred/lib/TestKeyFieldBasedPartitioner.java | 8 +-
.../hadoop/mapred/lib/TestLineInputFormat.java | 22 +-
.../hadoop/mapred/lib/TestMultipleInputs.java | 4 +-
.../hadoop/mapred/lib/TestMultipleOutputs.java | 43 +-
.../mapred/lib/TestMultithreadedMapRunner.java | 6 +-
.../mapred/lib/aggregate/TestAggregates.java | 4 +-
.../hadoop/mapred/lib/db/TestConstructQuery.java | 8 +-
.../hadoop/mapred/pipes/TestPipeApplication.java | 14 +-
.../org/apache/hadoop/mapred/pipes/TestPipes.java | 27 +-
.../mapred/pipes/TestPipesNonJavaInputFormat.java | 16 +-
.../org/apache/hadoop/mapreduce/TestChild.java | 66 +-
.../mapreduce/TestClientProtocolProviderImpls.java | 12 +-
.../org/apache/hadoop/mapreduce/TestCounters.java | 21 +-
.../org/apache/hadoop/mapreduce/TestLargeSort.java | 16 +-
.../apache/hadoop/mapreduce/TestLocalRunner.java | 30 +-
.../TestMRIntermediateDataEncryption.java | 96 +-
.../apache/hadoop/mapreduce/TestMRJobClient.java | 100 +-
.../hadoop/mapreduce/TestMROutputFormat.java | 4 +-
.../apache/hadoop/mapreduce/TestMapCollection.java | 18 +-
.../org/apache/hadoop/mapreduce/TestMapReduce.java | 10 +-
.../hadoop/mapreduce/TestMapReduceLazyOutput.java | 4 +-
.../hadoop/mapreduce/TestMapperReducerCleanup.java | 33 +-
.../hadoop/mapreduce/TestNewCombinerGrouping.java | 27 +-
.../hadoop/mapreduce/TestNoJobSetupCleanup.java | 15 +-
.../apache/hadoop/mapreduce/TestTaskContext.java | 54 +-
.../hadoop/mapreduce/TestValueIterReset.java | 4 +-
.../mapreduce/TestYarnClientProtocolProvider.java | 14 +-
.../mapreduce/filecache/TestURIFragments.java | 5 +-
.../lib/aggregate/TestMapReduceAggregates.java | 6 +-
.../mapreduce/lib/chain/TestChainErrors.java | 22 +-
.../mapreduce/lib/chain/TestMapReduceChain.java | 58 +-
.../lib/chain/TestSingleElementChain.java | 12 +-
.../mapreduce/lib/db/TestDBOutputFormat.java | 8 +-
.../lib/db/TestDataDrivenDBInputFormat.java | 20 +-
.../mapreduce/lib/db/TestIntegerSplitter.java | 12 +-
.../hadoop/mapreduce/lib/db/TestTextSplitter.java | 12 +-
.../lib/fieldsel/TestMRFieldSelection.java | 10 +-
.../lib/input/TestCombineFileInputFormat.java | 30 +-
.../input/TestCombineSequenceFileInputFormat.java | 28 +-
.../lib/input/TestCombineTextInputFormat.java | 41 +-
.../lib/input/TestDelegatingInputFormat.java | 13 +-
.../lib/input/TestFixedLengthInputFormat.java | 78 +-
.../lib/input/TestLineRecordReaderJobs.java | 4 +-
.../lib/input/TestMRCJCFileInputFormat.java | 12 +-
.../lib/input/TestMRKeyValueTextInputFormat.java | 81 +-
.../TestMRSequenceFileAsBinaryInputFormat.java | 22 +-
.../input/TestMRSequenceFileAsTextInputFormat.java | 14 +-
.../lib/input/TestMRSequenceFileInputFilter.java | 4 +-
.../mapreduce/lib/input/TestMultipleInputs.java | 8 +-
.../mapreduce/lib/input/TestNLineInputFormat.java | 20 +-
.../lib/jobcontrol/TestControlledJob.java | 4 +-
.../lib/jobcontrol/TestMapReduceJobControl.java | 20 +-
.../TestMapReduceJobControlWithMocks.java | 16 +-
.../mapreduce/lib/join/TestJoinDatamerge.java | 56 +-
.../mapreduce/lib/join/TestJoinProperties.java | 66 +-
.../mapreduce/lib/join/TestJoinTupleWritable.java | 35 +-
.../lib/join/TestWrappedRRClassloader.java | 10 +-
.../mapreduce/lib/map/TestMultithreadedMapper.java | 6 +-
.../lib/output/TestJobOutputCommitter.java | 43 +-
.../lib/output/TestMRCJCFileOutputCommitter.java | 24 +-
.../lib/output/TestMRMultipleOutputs.java | 39 +-
.../TestMRSequenceFileAsBinaryOutputFormat.java | 41 +-
.../lib/partition/TestBinaryPartitioner.java | 6 +-
.../mapreduce/lib/partition/TestInputSampler.java | 11 +-
.../lib/partition/TestKeyFieldHelper.java | 72 +-
.../partition/TestMRKeyFieldBasedComparator.java | 6 +-
.../partition/TestMRKeyFieldBasedPartitioner.java | 37 +-
.../lib/partition/TestTotalOrderPartitioner.java | 21 +-
.../mapreduce/security/TestBinaryTokenFile.java | 21 +-
.../hadoop/mapreduce/security/TestJHSSecurity.java | 14 +-
.../mapreduce/security/TestMRCredentials.java | 12 +-
.../TestUmbilicalProtocolWithJobToken.java | 2 +-
.../security/ssl/TestEncryptedShuffle.java | 21 +-
.../token/delegation/TestDelegationToken.java | 26 +-
.../mapreduce/util/TestMRAsyncDiskService.java | 28 +-
.../v2/TestMRAMWithNonNormalizedCapabilities.java | 21 +-
.../hadoop/mapreduce/v2/TestMRAppWithCombiner.java | 15 +-
.../org/apache/hadoop/mapreduce/v2/TestMRJobs.java | 278 ++-
.../mapreduce/v2/TestMRJobsWithHistoryService.java | 35 +-
.../mapreduce/v2/TestMRJobsWithProfiler.java | 49 +-
.../hadoop/mapreduce/v2/TestMROldApiJobs.java | 42 +-
.../hadoop/mapreduce/v2/TestMiniMRProxyUser.java | 14 +-
.../hadoop/mapreduce/v2/TestNonExistentJob.java | 12 +-
.../apache/hadoop/mapreduce/v2/TestRMNMInfo.java | 76 +-
.../mapreduce/v2/TestSpeculativeExecOnCluster.java | 175 +-
.../mapreduce/v2/TestSpeculativeExecution.java | 72 +-
.../v2/TestSpeculativeExecutionWithMRApp.java | 47 +-
.../org/apache/hadoop/mapreduce/v2/TestUberAM.java | 31 +-
.../hadoop/util/TestMRCJCReflectionUtils.java | 9 +-
.../org/apache/hadoop/util/TestMRCJCRunJar.java | 6 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 5 +
.../hadoop/mapred/nativetask/TestTaskContext.java | 13 +-
.../nativetask/buffer/TestBufferPushPull.java | 17 +-
.../nativetask/buffer/TestByteBufferReadWrite.java | 2 +-
.../mapred/nativetask/buffer/TestInputBuffer.java | 2 +-
.../mapred/nativetask/buffer/TestOutputBuffer.java | 2 +-
.../nativetask/combinertest/CombinerTest.java | 18 +-
.../combinertest/LargeKVCombinerTest.java | 17 +-
.../combinertest/OldAPICombinerTest.java | 16 +-
.../nativetask/compresstest/CompressTest.java | 16 +-
.../nativetask/handlers/TestCombineHandler.java | 6 +-
.../handlers/TestNativeCollectorOnlyHandler.java | 23 +-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 38 +-
.../mapred/nativetask/kvtest/LargeKVTest.java | 25 +-
.../mapred/nativetask/nonsorttest/NonSortTest.java | 16 +-
.../mapred/nativetask/serde/TestKVSerializer.java | 23 +-
.../nativetask/serde/TestNativeSerialization.java | 12 +-
.../mapred/nativetask/testutil/ResultVerifier.java | 20 +-
.../mapred/nativetask/utils/TestBytesUtil.java | 17 +-
.../nativetask/utils/TestReadWriteBuffer.java | 2 +-
.../mapred/nativetask/utils/TestSizedWritable.java | 10 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 5 +
.../hadoop/mapred/TestFadvisedChunkedFile.java | 12 +-
.../hadoop/mapred/TestFadvisedFileRegion.java | 28 +-
.../hadoop/mapred/TestShuffleChannelHandler.java | 73 +-
.../apache/hadoop/mapred/TestShuffleHandler.java | 69 +-
.../hadoop/mapred/TestShuffleHandlerBase.java | 21 +-
hadoop-project/pom.xml | 46 +-
hadoop-project/src/site/markdown/index.md.vm | 162 +-
hadoop-tools/hadoop-aliyun/pom.xml | 26 +
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 16 +
.../org/apache/hadoop/fs/aliyun/oss/Constants.java | 15 +
.../fs/aliyun/oss/ITAliyunOSSSignatureV4.java | 98 +
.../src/test/resources/log4j.properties | 3 +
hadoop-tools/hadoop-archive-logs/pom.xml | 5 +
.../org/apache/hadoop/tools/HadoopArchiveLogs.java | 1 +
hadoop-tools/hadoop-archives/pom.xml | 5 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 2 +-
hadoop-tools/hadoop-aws/pom.xml | 46 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 56 +
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 3 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 84 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 360 ++-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 402 +--
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 19 +-
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 44 +-
.../java/org/apache/hadoop/fs/s3a/S3AStore.java | 66 +-
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 24 +
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 4 +
.../apache/hadoop/fs/s3a/VectoredIOContext.java | 104 +-
.../hadoop/fs/s3a/audit/AuditIntegration.java | 11 +
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 7 +
.../apache/hadoop/fs/s3a/audit/AuditorFlags.java | 17 +-
.../hadoop/fs/s3a/audit/OperationAuditor.java | 7 +
.../s3a/audit/impl/AbstractOperationAuditor.java | 79 +
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 7 +
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 14 +-
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 9 +-
.../hadoop/fs/s3a/audit/impl/NoopAuditor.java | 16 +-
.../apache/hadoop/fs/s3a/impl/ClientManager.java | 9 +-
.../hadoop/fs/s3a/impl/ClientManagerImpl.java | 60 +-
.../fs/s3a/impl/InputStreamCallbacksImpl.java | 123 +
.../apache/hadoop/fs/s3a/impl/S3AStoreImpl.java | 213 +-
.../streams/AbstractObjectInputStreamFactory.java | 98 +
.../fs/s3a/impl/streams/AnalyticsStream.java | 238 ++
.../s3a/impl/streams/AnalyticsStreamFactory.java | 109 +
.../streams/ClassicObjectInputStreamFactory.java | 73 +
.../impl/streams/FactoryBindingParameters.java} | 36 +-
.../fs/s3a/impl/streams/InputStreamType.java | 123 +
.../fs/s3a/impl/streams/ObjectInputStream.java | 421 ++++
.../impl/streams/ObjectInputStreamCallbacks.java | 64 +
.../s3a/impl/streams/ObjectInputStreamFactory.java | 90 +
.../fs/s3a/impl/streams/ObjectReadParameters.java | 190 ++
.../impl/streams/StreamFactoryRequirements.java | 148 ++
.../fs/s3a/impl/streams/StreamIntegration.java | 212 ++
.../hadoop/fs/s3a/impl/streams/package-info.java | 19 +-
.../hadoop/fs/s3a/prefetch/PrefetchOptions.java | 67 +
.../prefetch/PrefetchingInputStreamFactory.java | 109 +
.../fs/s3a/prefetch/S3ACachingInputStream.java | 16 +-
.../fs/s3a/prefetch/S3AInMemoryInputStream.java | 8 +-
.../fs/s3a/prefetch/S3APrefetchingInputStream.java | 67 +-
.../fs/s3a/prefetch/S3ARemoteInputStream.java | 14 +-
.../hadoop/fs/s3a/prefetch/S3ARemoteObject.java | 5 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 8 +-
.../s3a/statistics/S3AInputStreamStatistics.java | 8 +
.../statistics/impl/EmptyS3AStatisticsContext.java | 6 +
.../site/markdown/tools/hadoop-aws/prefetching.md | 14 +-
.../src/site/markdown/tools/hadoop-aws/reading.md | 228 ++
...TestS3AContractAnalyticsStreamVectoredRead.java | 55 +
.../fs/contract/s3a/ITestS3AContractCreate.java | 12 +
.../fs/contract/s3a/ITestS3AContractDistCp.java | 15 +
.../s3a/ITestS3AContractMultipartUploader.java | 7 +
.../fs/contract/s3a/ITestS3AContractSeek.java | 2 +-
.../contract/s3a/ITestS3AContractVectoredRead.java | 12 +
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 3 +-
.../hadoop/fs/s3a/ITestEMRFSCompatibility.java | 68 +
.../ITestS3AAnalyticsAcceleratorStreamReading.java | 193 ++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 18 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 4 +
.../hadoop/fs/s3a/ITestS3AFSMainOperations.java | 23 +
.../hadoop/fs/s3a/ITestS3AFileSystemContract.java | 13 +
.../hadoop/fs/s3a/ITestS3AIOStatisticsContext.java | 6 +
.../hadoop/fs/s3a/ITestS3AInputStreamLeakage.java | 50 +-
.../org/apache/hadoop/fs/s3a/ITestS3AMetrics.java | 6 +
.../fs/s3a/ITestS3APrefetchingCacheFiles.java | 5 +-
.../fs/s3a/ITestS3APrefetchingInputStream.java | 6 +-
.../fs/s3a/ITestS3APrefetchingLruEviction.java | 11 +-
.../hadoop/fs/s3a/ITestS3ARequesterPays.java | 19 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 80 +-
.../hadoop/fs/s3a/TestS3AInputStreamRetry.java | 27 +-
.../org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java | 4 +
.../hadoop/fs/s3a/audit/AccessCheckingAuditor.java | 10 +
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 25 +
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 9 +
.../fs/s3a/commit/ITestCommitOperationCost.java | 12 +-
.../fileContext/ITestS3AFileContextStatistics.java | 6 +
.../fs/s3a/impl/ITestConnectionTimeouts.java | 7 +-
.../fs/s3a/impl/streams/TestStreamFactories.java | 339 +++
.../fs/s3a/performance/ITestS3AOpenCost.java | 13 +-
.../fs/s3a/performance/ITestUnbufferDraining.java | 6 +-
.../fs/s3a/prefetch/MockS3ARemoteObject.java | 8 +-
.../hadoop/fs/s3a/prefetch/S3APrefetchFakes.java | 43 +-
.../fs/s3a/prefetch/TestS3ARemoteInputStream.java | 19 +-
.../fs/s3a/prefetch/TestS3ARemoteObject.java | 6 +-
.../s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java | 3 +
.../s3a/scale/ITestS3AInputStreamPerformance.java | 47 +-
.../ITestS3AContractStreamIOStatistics.java | 9 +
.../statistics/ITestS3AFileSystemStatistic.java | 6 +
.../hadoop-aws/src/test/resources/log4j.properties | 4 +
.../src/test/resources/malformed_footer.parquet | Bin 0 -> 451 bytes
.../src/test/resources/multi_row_group.parquet | Bin 0 -> 2080 bytes
hadoop-tools/hadoop-azure-datalake/pom.xml | 6 +-
.../dev-support/testrun-scripts/runtests.sh | 123 +-
.../dev-support/testrun-scripts/testsupport.sh | 19 +
hadoop-tools/hadoop-azure/pom.xml | 5 +
.../src/config/checkstyle-suppressions.xml | 2 +
.../hadoop/fs/azurebfs/AbfsBackoffMetrics.java | 312 ---
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 111 +-
.../hadoop/fs/azurebfs/AbfsCountersImpl.java | 12 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 155 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 461 ++--
.../fs/azurebfs/constants/AbfsHttpConstants.java | 40 +-
.../fs/azurebfs/constants/ConfigurationKeys.java | 27 +
.../constants/FileSystemConfigurations.java | 23 +
.../constants/HttpHeaderConfigurations.java | 34 +
.../fs/azurebfs/constants/MetricsConstants.java | 121 +
.../contracts/exceptions/AbfsDriverException.java | 8 +
.../ConcurrentWriteOperationDetectedException.java | 17 +
.../exceptions/InvalidIngressServiceException.java | 51 +
.../services/AppendRequestParameters.java | 54 +-
.../contracts/services/AzureServiceErrorCode.java | 8 +
.../services/BlobAppendRequestParameters.java | 8 +
.../fs/azurebfs/enums/AbfsBackoffMetricsEnum.java | 110 +
.../azurebfs/enums/AbfsReadFooterMetricsEnum.java | 97 +
.../hadoop/fs/azurebfs/enums/BlobCopyProgress.java | 27 +-
.../apache/hadoop/fs/azurebfs/enums/FileType.java | 30 +-
.../hadoop/fs/azurebfs/enums/RetryValue.java | 83 +
.../fs/azurebfs/enums/StatisticTypeEnum.java | 23 +-
.../fs/azurebfs/extensions/SASTokenProvider.java | 2 +
.../fs/azurebfs/services/AbfsAHCHttpOperation.java | 28 +-
.../fs/azurebfs/services/AbfsBackoffMetrics.java | 323 +++
.../hadoop/fs/azurebfs/services/AbfsBlobBlock.java | 72 +
.../fs/azurebfs/services/AbfsBlobClient.java | 1078 ++++++++-
.../hadoop/fs/azurebfs/services/AbfsBlock.java | 160 ++
.../fs/azurebfs/services/AbfsBlockStatus.java | 45 +-
.../hadoop/fs/azurebfs/services/AbfsClient.java | 277 ++-
.../fs/azurebfs/services/AbfsClientHandler.java | 29 +
.../hadoop/fs/azurebfs/services/AbfsCounters.java | 1 -
.../hadoop/fs/azurebfs/services/AbfsDfsClient.java | 271 ++-
.../hadoop/fs/azurebfs/services/AbfsErrors.java | 33 +-
.../fs/azurebfs/services/AbfsHttpOperation.java | 36 +-
.../fs/azurebfs/services/AbfsInputStream.java | 2 +-
.../hadoop/fs/azurebfs/services/AbfsLease.java | 165 +-
.../fs/azurebfs/services/AbfsOutputStream.java | 618 ++++-
.../azurebfs/services/AbfsOutputStreamContext.java | 67 +-
.../fs/azurebfs/services/AbfsPermission.java | 5 +-
.../azurebfs/services/AbfsReadFooterMetrics.java | 984 ++++----
.../fs/azurebfs/services/AbfsRestOperation.java | 83 +-
.../azurebfs/services/AbfsRestOperationType.java | 3 +-
.../services/AbstractAbfsStatisticsSource.java | 153 ++
.../azurebfs/services/AzureBlobBlockManager.java | 209 ++
.../azurebfs/services/AzureBlobIngressHandler.java | 349 +++
.../fs/azurebfs/services/AzureBlockManager.java | 178 ++
.../fs/azurebfs/services/AzureDFSBlockManager.java | 95 +
.../azurebfs/services/AzureDFSIngressHandler.java | 280 +++
.../AzureDfsToBlobIngressFallbackHandler.java | 265 ++
.../fs/azurebfs/services/AzureIngressHandler.java | 209 ++
.../fs/azurebfs/services/BlobDeleteHandler.java | 201 ++
.../fs/azurebfs/services/BlobRenameHandler.java | 614 +++++
.../hadoop/fs/azurebfs/services/BlockEntry.java | 86 +
.../fs/azurebfs/services/ListActionTaker.java | 278 +++
.../hadoop/fs/azurebfs/services/ListBlobQueue.java | 197 ++
.../fs/azurebfs/services/ListResponseData.java | 102 +
.../fs/azurebfs/services/PathInformation.java | 108 +
.../fs/azurebfs/services/RenameAtomicity.java | 332 +++
.../RenamePendingJsonFormat.java} | 48 +-
.../fs/azurebfs/services/VersionedFileStatus.java | 139 ++
.../apache/hadoop/fs/azurebfs/utils/Listener.java | 2 +
.../hadoop/fs/azurebfs/utils/TracingContext.java | 65 +
.../apache/hadoop/fs/azurebfs/utils/UriUtils.java | 43 +
.../hadoop-azure/src/site/markdown/blobEndpoint.md | 7 +-
.../hadoop-azure/src/site/markdown/fns_blob.md | 84 +-
.../hadoop/fs/azure/AbstractWasbTestBase.java | 11 +-
.../fs/azure/AbstractWasbTestWithTimeout.java | 37 +-
.../fs/azure/AzureBlobStorageTestAccount.java | 8 +-
.../fs/azure/ITestAzureConcurrentOutOfBandIo.java | 2 +-
.../azure/ITestAzureFileSystemErrorConditions.java | 5 +-
.../hadoop/fs/azure/ITestBlobDataValidation.java | 18 +-
.../fs/azure/ITestBlobTypeSpeedDifference.java | 2 +-
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 71 +-
.../hadoop/fs/azure/ITestContainerChecks.java | 35 +-
.../ITestFileSystemOperationExceptionHandling.java | 150 +-
.../ITestFileSystemOperationExceptionMessage.java | 2 +-
...emOperationsExceptionHandlingMultiThreaded.java | 294 +--
.../ITestFileSystemOperationsWithThreads.java | 19 +-
.../hadoop/fs/azure/ITestListPerformance.java | 34 +-
.../ITestNativeAzureFSAuthorizationCaching.java | 4 +-
.../fs/azure/ITestNativeAzureFileSystemAppend.java | 36 +-
...stNativeAzureFileSystemAtomicRenameDirList.java | 2 +-
.../ITestNativeAzureFileSystemClientLogging.java | 12 +-
.../ITestNativeAzureFileSystemConcurrencyLive.java | 31 +-
.../fs/azure/ITestNativeAzureFileSystemLive.java | 4 +-
.../fs/azure/ITestNativeFileSystemStatistics.java | 9 +-
.../ITestOutOfBandAzureBlobOperationsLive.java | 2 +-
.../fs/azure/ITestOutputStreamSemantics.java | 59 +-
.../hadoop/fs/azure/ITestPageBlobInputStream.java | 32 +-
.../hadoop/fs/azure/ITestPageBlobOutputStream.java | 2 +-
.../azure/ITestReadAndSeekPageBlobAfterWrite.java | 15 +-
.../hadoop/fs/azure/ITestWasbRemoteCallHelper.java | 253 +-
.../fs/azure/ITestWasbUriAndConfiguration.java | 49 +-
.../fs/azure/NativeAzureFileSystemBaseTest.java | 34 +-
.../apache/hadoop/fs/azure/TestBlobMetadata.java | 12 +-
.../fs/azure/TestBlobOperationDescriptor.java | 2 +-
.../fs/azure/TestClientThrottlingAnalyzer.java | 35 +-
.../fs/azure/TestKeyPageBlobDirectories.java | 7 +-
.../TestNativeAzureFileSystemAuthorization.java | 519 ++--
.../TestNativeAzureFileSystemBlockCompaction.java | 23 +-
.../TestNativeAzureFileSystemConcurrency.java | 14 +-
.../TestNativeAzureFileSystemFileNameCheck.java | 4 +-
.../fs/azure/TestNativeAzureFileSystemMocked.java | 14 +-
.../TestNativeAzureFileSystemUploadLogic.java | 13 +-
.../fs/azure/TestOutOfBandAzureBlobOperations.java | 12 +-
.../fs/azure/TestShellDecryptionKeyProvider.java | 6 +-
.../fs/azure/TestSyncableDataOutputStream.java | 2 +-
.../org/apache/hadoop/fs/azure/TestWasbFsck.java | 15 +-
.../azure/integration/AbstractAzureScaleTest.java | 5 +-
.../fs/azure/integration/AzureTestUtils.java | 21 +-
.../azure/integration/CleanupTestContainers.java | 2 +-
.../fs/azure/integration/ITestAzureHugeFiles.java | 33 +-
.../ITestAzureFileSystemInstrumentation.java | 119 +-
.../azure/metrics/TestBandwidthGaugeUpdater.java | 11 +-
.../TestNativeAzureFileSystemMetricsSystem.java | 9 +-
.../fs/azure/metrics/TestRollingWindowAverage.java | 4 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 193 +-
.../apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 8 +-
.../fs/azurebfs/ITestAbfsCustomEncryption.java | 56 +-
.../ITestAbfsListStatusRemoteIterator.java | 1 +
.../fs/azurebfs/ITestAbfsNetworkStatistics.java | 59 +-
.../fs/azurebfs/ITestAbfsReadFooterMetrics.java | 39 +-
.../azurebfs/ITestAbfsRestOperationException.java | 133 +-
.../hadoop/fs/azurebfs/ITestAbfsStatistics.java | 2 -
.../azurebfs/ITestAzureBlobFileSystemAppend.java | 1172 ++++++++-
.../ITestAzureBlobFileSystemAttributes.java | 184 +-
.../azurebfs/ITestAzureBlobFileSystemChecksum.java | 48 +-
.../ITestAzureBlobFileSystemChooseSAS.java | 2 +
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 2207 +++++++++++++++--
.../ITestAzureBlobFileSystemDelegationSAS.java | 22 +-
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 470 +++-
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 19 +-
.../ITestAzureBlobFileSystemFileStatus.java | 126 +-
.../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 5 +
.../ITestAzureBlobFileSystemInitAndCreate.java | 24 +-
.../fs/azurebfs/ITestAzureBlobFileSystemLease.java | 75 +-
.../ITestAzureBlobFileSystemListStatus.java | 256 +-
.../fs/azurebfs/ITestAzureBlobFileSystemMkDir.java | 29 +-
.../ITestAzureBlobFileSystemPermission.java | 3 +
.../ITestAzureBlobFileSystemRandomRead.java | 1 +
.../azurebfs/ITestAzureBlobFileSystemRename.java | 2551 +++++++++++++++++++-
.../ITestAzureBlobFileSystemRenameUnicode.java | 6 +-
.../fs/azurebfs/ITestAzureBlobFilesystemAcl.java | 5 +
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 22 +-
.../hadoop/fs/azurebfs/ITestSharedKeyAuth.java | 15 +-
.../fs/azurebfs/ITestSmallWriteOptimization.java | 21 +-
.../fs/azurebfs/ITestWasbAbfsCompatibility.java | 12 +-
.../apache/hadoop/fs/azurebfs/MockIntercept.java | 43 +
.../hadoop/fs/azurebfs/TestTracingContext.java | 2 +-
.../azurebfs/constants/TestConfigurationKeys.java | 1 +
.../azurebfs/contract/TestBlobListXmlParser.java | 154 ++
...tSchemaTest.java => TestDfsListJsonParser.java} | 12 +-
.../fs/azurebfs/services/AbfsClientTestUtil.java | 185 +-
.../fs/azurebfs/services/ITestAbfsClient.java | 9 +-
.../azurebfs/services/ITestAbfsClientHandler.java | 12 +-
.../ITestAbfsHttpClientRequestExecutor.java | 59 +-
.../azurebfs/services/ITestAbfsOutputStream.java | 187 +-
.../services/ITestAbfsPaginatedDelete.java | 12 +-
.../azurebfs/services/ITestAbfsRestOperation.java | 5 +-
.../services/ITestApacheClientConnectionPool.java | 3 +-
.../services/RenameAtomicityTestUtils.java | 79 +
.../azurebfs/services/TestAbfsBackoffMetrics.java | 115 +
.../fs/azurebfs/services/TestAbfsClient.java | 39 +
.../fs/azurebfs/services/TestAbfsOutputStream.java | 56 +-
.../services/TestAbfsReadFooterMetrics.java | 98 +
.../services/TestAbfsRenameRetryRecovery.java | 86 +-
.../azurebfs/services/TestAbfsRestOperation.java | 5 +-
.../TestAbfsRestOperationMockFailures.java | 9 +-
.../fs/azurebfs/services/TestListActionTaker.java | 145 ++
.../azurebfs/utils/AzcopyExecutionException.java} | 21 +-
.../hadoop/fs/azurebfs/utils/AzcopyToolHelper.java | 319 +++
.../fs/azurebfs/utils/DirectoryStateHelper.java | 118 +
.../fs/azurebfs/utils/TestAzcopyToolHelper.java | 78 +
.../fs/azurebfs/utils/TracingHeaderValidator.java | 45 +-
.../accountName_settings.xml.template | 24 +
hadoop-tools/hadoop-datajoin/pom.xml | 5 +
hadoop-tools/hadoop-distcp/pom.xml | 5 +
.../org/apache/hadoop/tools/SimpleCopyListing.java | 15 +-
.../org/apache/hadoop/tools/TestCopyFilter.java | 16 +-
.../org/apache/hadoop/tools/TestCopyListing.java | 175 +-
.../hadoop/tools/TestCopyListingFileStatus.java | 4 +-
.../org/apache/hadoop/tools/TestDistCpOptions.java | 214 +-
.../org/apache/hadoop/tools/TestDistCpSync.java | 92 +-
.../hadoop/tools/TestDistCpSyncReverseBase.java | 60 +-
.../org/apache/hadoop/tools/TestDistCpSystem.java | 52 +-
.../org/apache/hadoop/tools/TestDistCpViewFs.java | 16 +-
.../apache/hadoop/tools/TestDistCpWithAcls.java | 15 +-
.../hadoop/tools/TestDistCpWithRawXAttrs.java | 63 +-
.../apache/hadoop/tools/TestDistCpWithXAttrs.java | 10 +-
.../org/apache/hadoop/tools/TestExternalCall.java | 67 +-
.../hadoop/tools/TestFileBasedCopyListing.java | 48 +-
.../hadoop/tools/TestGlobbedCopyListing.java | 17 +-
.../org/apache/hadoop/tools/TestIntegration.java | 199 +-
.../org/apache/hadoop/tools/TestOptionsParser.java | 312 +--
.../apache/hadoop/tools/TestRegexCopyFilter.java | 15 +-
.../tools/TestRegexpInConfigurationFilter.java | 22 +-
.../apache/hadoop/tools/TestTrueCopyFilter.java | 9 +-
.../apache/hadoop/tools/mapred/TestCopyMapper.java | 197 +-
.../tools/mapred/TestCopyMapperCompositeCrc.java | 4 +-
.../hadoop/tools/mapred/TestCopyOutputFormat.java | 43 +-
.../hadoop/tools/mapred/TestDeletedDirTracker.java | 43 +-
.../tools/mapred/TestRetriableFileCopyCommand.java | 31 +-
.../tools/mapred/TestUniformSizeInputFormat.java | 22 +-
.../tools/mapred/lib/TestDynamicInputFormat.java | 59 +-
.../apache/hadoop/tools/util/DistCpTestUtils.java | 13 +-
.../apache/hadoop/tools/util/TestDistCpUtils.java | 472 ++--
.../tools/util/TestDistCpUtilsWithCombineMode.java | 37 +-
.../hadoop/tools/util/TestProducerConsumer.java | 30 +-
.../hadoop/tools/util/TestRetriableCommand.java | 17 +-
.../tools/util/TestThrottledInputStream.java | 23 +-
.../hadoop-dynamometer-blockgen/pom.xml | 5 +
.../hadoop-dynamometer-infra/pom.xml | 5 +
.../hadoop-dynamometer-workload/pom.xml | 5 +
.../audit/AuditCommandParser.java | 5 +-
.../audit/AuditLogDirectParser.java | 5 +-
.../audit/AuditLogHiveTableParser.java | 4 +-
.../audit/AuditReplayCommand.java | 30 +-
.../workloadgenerator/audit/AuditReplayMapper.java | 4 +-
.../audit/TestAuditLogDirectParser.java | 39 +-
hadoop-tools/hadoop-extras/pom.xml | 5 +
hadoop-tools/hadoop-federation-balance/pom.xml | 5 +
hadoop-tools/hadoop-fs2img/pom.xml | 5 +
hadoop-tools/hadoop-gridmix/pom.xml | 5 +
.../org/apache/hadoop/metrics2/sink/KafkaSink.java | 2 +
hadoop-tools/hadoop-resourceestimator/pom.xml | 5 +
hadoop-tools/hadoop-rumen/pom.xml | 5 +
hadoop-tools/hadoop-sls/pom.xml | 5 +
.../java/org/apache/hadoop/yarn/sls/AMRunner.java | 5 +-
.../apache/hadoop/yarn/sls/BaseSLSRunnerTest.java | 10 +
hadoop-tools/hadoop-streaming/pom.xml | 5 +
.../yarn/api/records/timeline/TimelineEntity.java | 3 +-
.../yarn/api/records/timeline/TimelineEvent.java | 2 +-
.../records/timelineservice/ApplicationEntity.java | 7 +-
.../api/records/timelineservice/FlowRunEntity.java | 4 +-
.../records/timelineservice/TimelineEntity.java | 3 +-
.../api/records/timelineservice/TimelineEvent.java | 3 +-
.../records/timelineservice/TimelineMetric.java | 2 +-
.../writer/TimelineEntitySetWriter.java | 4 +-
.../pom.xml | 5 +
.../distributedshell/ApplicationMaster.java | 2 +-
.../distributedshell/DistributedShellBaseTest.java | 110 +-
.../applications/distributedshell/TestClient.java | 4 +-
.../distributedshell/TestDSAppMaster.java | 68 +-
.../distributedshell/TestDSTimelineV10.java | 218 +-
.../distributedshell/TestDSTimelineV15.java | 18 +-
.../distributedshell/TestDSTimelineV20.java | 89 +-
.../TestDSWithMultipleNodeManager.java | 106 +-
.../hadoop-yarn-services-core/pom.xml | 6 +-
.../hadoop/yarn/service/ServiceScheduler.java | 4 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 33 +-
.../TestDefaultUpgradeComponentsFinder.java | 23 +-
.../apache/hadoop/yarn/service/TestServiceAM.java | 86 +-
.../hadoop/yarn/service/TestServiceManager.java | 108 +-
.../yarn/service/TestYarnNativeServices.java | 235 +-
.../client/TestBuildExternalComponents.java | 17 +-
.../hadoop/yarn/service/client/TestServiceCLI.java | 85 +-
.../yarn/service/client/TestServiceClient.java | 46 +-
.../yarn/service/component/TestComponent.java | 114 +-
.../TestComponentDecommissionInstances.java | 35 +-
.../component/TestComponentRestartPolicy.java | 4 +-
.../component/instance/TestComponentInstance.java | 99 +-
.../yarn/service/conf/TestAppJsonResolve.java | 39 +-
.../yarn/service/conf/TestLoadExampleAppJson.java | 25 +-
.../service/conf/TestValidateServiceNames.java | 7 +-
.../containerlaunch/TestAbstractLauncher.java | 10 +-
.../yarn/service/monitor/TestServiceMonitor.java | 19 +-
.../service/monitor/probe/TestDefaultProbe.java | 60 +-
.../provider/TestAbstractProviderService.java | 30 +-
.../yarn/service/provider/TestProviderUtils.java | 12 +-
.../providers/TestAbstractClientProvider.java | 24 +-
.../providers/TestDefaultClientProvider.java | 11 +-
.../service/providers/TestProviderFactory.java | 4 +-
.../TestServiceTimelinePublisher.java | 12 +-
.../yarn/service/utils/TestCoreFileSystem.java | 15 +-
.../hadoop/yarn/service/utils/TestFilterUtils.java | 41 +-
.../yarn/service/utils/TestServiceApiUtil.java | 115 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 5 +
.../ApplicationMasterServiceProtoTestBase.java | 4 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 71 +-
.../client/TestApplicationClientProtocolOnHA.java | 83 +-
...licationMasterServiceProtocolForTimelineV2.java | 24 +-
.../TestApplicationMasterServiceProtocolOnHA.java | 21 +-
.../TestFederationRMFailoverProxyProvider.java | 28 +-
.../TestHedgingRequestRMFailoverProxyProvider.java | 14 +-
.../hadoop/yarn/client/TestMemoryPageUtils.java | 6 +-
.../client/TestNoHaRMFailoverProxyProvider.java | 67 +-
.../apache/hadoop/yarn/client/TestRMFailover.java | 10 +-
.../yarn/client/TestRMFailoverProxyProvider.java | 78 +-
...eManagerAdministrationProtocolPBClientImpl.java | 16 +-
.../yarn/client/TestResourceTrackerOnHA.java | 23 +-
.../hadoop/yarn/client/TestYarnApiClasses.java | 8 +-
.../client/api/async/impl/TestAMRMClientAsync.java | 56 +-
.../client/api/async/impl/TestNMClientAsync.java | 45 +-
.../yarn/client/api/impl/BaseAMRMClientTest.java | 24 +-
.../hadoop/yarn/client/api/impl/TestAHSClient.java | 58 +-
.../yarn/client/api/impl/TestAHSv2ClientImpl.java | 10 +-
.../yarn/client/api/impl/TestAMRMClient.java | 295 ++-
.../api/impl/TestAMRMClientContainerRequest.java | 79 +-
.../client/api/impl/TestAMRMClientOnRMRestart.java | 83 +-
.../impl/TestAMRMClientPlacementConstraints.java | 56 +-
.../hadoop/yarn/client/api/impl/TestAMRMProxy.java | 55 +-
.../hadoop/yarn/client/api/impl/TestNMClient.java | 42 +-
.../TestOpportunisticContainerAllocationE2E.java | 58 +-
.../client/api/impl/TestSharedCacheClientImpl.java | 59 +-
.../yarn/client/api/impl/TestYarnClient.java | 316 ++-
.../yarn/client/api/impl/TestYarnClientImpl.java | 73 +-
.../api/impl/TestYarnClientWithReservation.java | 131 +-
.../hadoop/yarn/client/cli/TestClusterCLI.java | 8 +-
.../apache/hadoop/yarn/client/cli/TestGpgCLI.java | 8 +-
.../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 169 +-
.../yarn/client/cli/TestNodeAttributesCLI.java | 99 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 73 +-
.../hadoop/yarn/client/cli/TestRouterCLI.java | 10 +-
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 113 +-
.../apache/hadoop/yarn/client/cli/TestTopCLI.java | 40 +-
.../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 204 +-
.../yarn/client/util/TestFormattingCLIUtils.java | 4 +-
.../yarn/client/util/TestYarnClientUtils.java | 91 +-
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 4 +-
.../apache/hadoop/yarn/event/EventDispatcher.java | 4 +-
.../ifile/LogAggregationIndexedFileController.java | 4 +-
.../client/RMDelegationTokenIdentifier.java | 6 +-
.../yarn/util/AbstractLivelinessMonitor.java | 2 +
.../java/org/apache/hadoop/yarn/util/Clock.java | 4 +
.../apache/hadoop/yarn/util/MonotonicClock.java | 4 +
.../hadoop/yarn/util/ProcfsBasedProcessTree.java | 2 +
.../org/apache/hadoop/yarn/util/SystemClock.java | 6 +-
.../java/org/apache/hadoop/yarn/util/UTCClock.java | 5 +-
.../hadoop/yarn/util/WindowsBasedProcessTree.java | 2 +
.../org/apache/hadoop/yarn/webapp/WebApps.java | 12 +
.../TestLogAggregationIndexedFileController.java | 2 +-
.../apache/hadoop/yarn/util/ControlledClock.java | 2 +
.../yarn/util/TestProcfsBasedProcessTree.java | 2 +
.../yarn/util/TestWindowsBasedProcessTree.java | 1 +
.../hadoop-yarn-server-common/pom.xml | 5 +
.../store/impl/MemoryFederationStateStore.java | 2 +-
.../store/impl/SQLFederationStateStore.java | 4 +-
.../impl/ZKFederationStateStoreOpDurations.java | 6 +
.../store/impl/ZookeeperFederationStateStore.java | 9 +-
.../test/java/org/apache/hadoop/yarn/TestRPC.java | 56 +-
.../yarn/TestResourceTrackerPBClientImpl.java | 14 +-
.../apache/hadoop/yarn/TestYSCRPCFactories.java | 12 +-
.../apache/hadoop/yarn/TestYSCRecordFactory.java | 11 +-
.../hadoop/yarn/TestYarnServerApiClasses.java | 31 +-
.../org/apache/hadoop/yarn/lib/TestZKClient.java | 28 +-
.../yarn/server/MockResourceManagerFacade.java | 27 +-
.../hadoop/yarn/server/TestAMRMClientRelayer.java | 40 +-
.../hadoop/yarn/server/api/TestServerRMProxy.java | 9 +-
.../api/protocolrecords/TestProtocolRecords.java | 63 +-
.../TestRegisterNodeManagerRequest.java | 19 +-
.../TestRegisterNodeManagerResponse.java | 6 +-
.../federation/cache/TestFederationCache.java | 41 +-
.../TestFederationRMFailoverProxyProvider.java | 4 +-
.../policies/BaseFederationPoliciesTest.java | 86 +-
...rationPolicyInitializationContextValidator.java | 60 +-
.../policies/TestFederationPolicyUtils.java | 10 +-
.../policies/TestRouterPolicyFacade.java | 40 +-
.../TestBroadcastAMRMProxyFederationPolicy.java | 17 +-
.../amrmproxy/TestHomeAMRMProxyPolicy.java | 12 +-
.../TestLocalityMulticastAMRMProxyPolicy.java | 119 +-
.../amrmproxy/TestRejectAMRMProxyPolicy.java | 26 +-
.../policies/manager/BasePolicyManagerTest.java | 28 +-
.../TestHashBasedBroadcastPolicyManager.java | 4 +-
.../policies/manager/TestHomePolicyManager.java | 4 +-
.../TestPriorityBroadcastPolicyManager.java | 11 +-
.../manager/TestRejectAllPolicyManager.java | 4 +-
.../manager/TestUniformBroadcastPolicyManager.java | 4 +-
.../manager/TestWeightedHomePolicyManager.java | 11 +-
.../manager/TestWeightedLocalityPolicyManager.java | 13 +-
.../policies/router/BaseRouterPoliciesTest.java | 28 +-
.../policies/router/TestHashBasedRouterPolicy.java | 11 +-
.../policies/router/TestLoadBasedRouterPolicy.java | 12 +-
.../policies/router/TestLocalityRouterPolicy.java | 30 +-
.../policies/router/TestPriorityRouterPolicy.java | 11 +-
.../policies/router/TestRejectRouterPolicy.java | 48 +-
.../router/TestUniformRandomRouterPolicy.java | 11 +-
.../router/TestWeightedRandomRouterPolicy.java | 23 +-
.../resolver/TestDefaultSubClusterResolver.java | 47 +-
.../store/impl/FederationStateStoreBaseTest.java | 244 +-
.../store/impl/TestMemoryFederationStateStore.java | 6 +-
.../store/impl/TestSQLFederationStateStore.java | 6 +-
.../impl/TestZookeeperFederationStateStore.java | 19 +-
.../TestFederationStateStoreClientMetrics.java | 37 +-
.../records/TestFederationProtocolRecords.java | 10 +-
.../store/sql/FederationSQLAccuracyTest.java | 8 +-
.../sql/TestFederationMySQLScriptAccuracy.java | 2 +-
.../sql/TestFederationSQLServerScriptAccuracy.java | 2 +-
.../TestFederationStateStoreInputValidator.java | 278 +--
.../utils/FederationStateStoreTestUtil.java | 2 +-
.../utils/TestFederationRegistryClient.java | 33 +-
.../utils/TestFederationStateStoreFacade.java | 225 +-
.../utils/TestFederationStateStoreFacadeRetry.java | 27 +-
.../metrics/TestAMRMClientRelayerMetrics.java | 86 +-
...DistributedOpportunisticContainerAllocator.java | 96 +-
...estTimelineAuthenticationFilterInitializer.java | 15 +-
.../uam/TestUnmanagedApplicationManager.java | 93 +-
.../yarn/server/utils/TestLeveldbIterator.java | 20 +-
.../yarn/server/webapp/ContainerBlockTest.java | 6 +-
.../hadoop/yarn/server/webapp/TestAppsBlock.java | 44 +-
.../hadoop/yarn/server/webapp/TestLogServlet.java | 2 +-
.../yarn/server/webapp/TestLogWebService.java | 15 +-
.../pom.xml | 5 +
.../GlobalPolicyGenerator.java | 3 +-
.../globalpolicygenerator/webapp/GPGWebApp.java | 2 +
.../globalpolicygenerator/TestGPGPolicyFacade.java | 58 +-
.../TestGlobalPolicyGenerator.java | 8 +-
.../TestDefaultApplicationCleaner.java | 28 +-
.../policygenerator/TestLoadBasedGlobalPolicy.java | 10 +-
.../policygenerator/TestPolicyGenerator.java | 70 +-
.../secure/AbstractGlobalPolicyGeneratorTest.java | 18 +-
.../secure/TestGpgSecureLogins.java | 8 +-
.../subclustercleaner/TestSubClusterCleaner.java | 16 +-
.../webapp/TestGPGWebApp.java | 2 +-
.../webapp/TestGPGWebServices.java | 4 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 15 +
.../nodemanager/amrmproxy/AMRMProxyService.java | 4 +-
.../amrmproxy/FederationInterceptor.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 2 +-
.../containermanager/container/ContainerImpl.java | 4 +-
.../container/SlidingWindowRetryPolicy.java | 2 +-
.../linux/resources/AbstractCGroupsHandler.java | 4 +-
.../AbstractCGroupsResourceCalculator.java | 4 +-
.../resources/CGroupElasticMemoryController.java | 4 +-
.../localizer/ResourceLocalizationService.java | 42 +-
.../util/CgroupsLCEResourcesHandler.java | 4 +-
.../server/nodemanager/DummyContainerManager.java | 2 +-
.../server/nodemanager/NodeManagerTestBase.java | 9 +-
.../server/nodemanager/TestContainerExecutor.java | 76 +-
.../nodemanager/TestContainerManagerWithLCE.java | 13 +-
.../nodemanager/TestDefaultContainerExecutor.java | 77 +-
.../server/nodemanager/TestDeletionService.java | 18 +-
.../nodemanager/TestDirectoryCollection.java | 377 +--
.../yarn/server/nodemanager/TestEventFlow.java | 2 +-
.../nodemanager/TestLinuxContainerExecutor.java | 77 +-
.../TestLinuxContainerExecutorWithMocks.java | 96 +-
.../nodemanager/TestLocalDirsHandlerService.java | 72 +-
.../yarn/server/nodemanager/TestNMAuditLogger.java | 14 +-
.../TestNetworkTagMappingJsonManager.java | 23 +-
.../yarn/server/nodemanager/TestNodeManager.java | 69 +-
.../server/nodemanager/TestNodeManagerMXBean.java | 7 +-
.../server/nodemanager/TestNodeManagerReboot.java | 61 +-
.../server/nodemanager/TestNodeManagerResync.java | 72 +-
.../nodemanager/TestNodeManagerShutdown.java | 36 +-
.../nodemanager/TestNodeResourceMonitor.java | 6 +-
.../server/nodemanager/TestNodeStatusUpdater.java | 255 +-
.../TestNodeStatusUpdaterForAttributes.java | 67 +-
.../TestNodeStatusUpdaterForLabels.java | 64 +-
.../yarn/server/nodemanager/TestRPCFactories.java | 15 +-
.../yarn/server/nodemanager/TestRecordFactory.java | 10 +-
.../nodemanager/amrmproxy/BaseAMRMProxyTest.java | 39 +-
.../amrmproxy/TestAMRMProxyMetrics.java | 104 +-
.../amrmproxy/TestAMRMProxyService.java | 183 +-
.../amrmproxy/TestAMRMProxyTokenSecretManager.java | 17 +-
.../amrmproxy/TestFederationInterceptor.java | 266 +-
.../amrmproxy/TestFederationInterceptorSecure.java | 71 +-
.../nodemanager/api/impl/pb/TestNMProtoUtils.java | 4 +-
.../impl/pb/TestPBLocalizerRPC.java | 6 +-
.../protocolrecords/impl/pb/TestPBRecordImpl.java | 38 +-
.../containermanager/BaseContainerManagerTest.java | 23 +-
.../containermanager/TestAuxServices.java | 229 +-
.../containermanager/TestContainerManager.java | 397 ++-
.../TestContainerManagerRecovery.java | 32 +-
.../nodemanager/containermanager/TestNMProxy.java | 42 +-
.../application/TestApplication.java | 22 +-
.../containermanager/container/TestContainer.java | 47 +-
.../container/TestResourceMappings.java | 18 +-
.../container/TestSlidingWindowRetryPolicy.java | 59 +-
.../task/TestDockerContainerDeletionTask.java | 8 +-
.../deletion/task/TestFileDeletionTask.java | 12 +-
.../launcher/TestContainerCleanup.java | 16 +-
.../launcher/TestContainerLaunch.java | 492 ++--
.../launcher/TestContainerLaunchParameterized.java | 11 +-
.../launcher/TestContainerRelaunch.java | 48 +-
.../launcher/TestContainersLauncher.java | 21 +-
.../TestPrivilegedOperationExecutor.java | 50 +-
.../TestCGroupElasticMemoryController.java | 177 +-
.../TestCGroupsBlkioResourceHandlerImpl.java | 32 +-
.../TestCGroupsCpuResourceHandlerImpl.java | 55 +-
.../linux/resources/TestCGroupsHandlerBase.java | 12 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 143 +-
.../TestCGroupsMemoryResourceHandlerImpl.java | 62 +-
.../resources/TestCGroupsResourceCalculator.java | 12 +-
.../TestCGroupsV2CpuResourceHandlerImpl.java | 32 +-
.../linux/resources/TestCGroupsV2HandlerImpl.java | 127 +-
.../resources/TestCGroupsV2ResourceCalculator.java | 12 +-
.../TestCgroupsV2MemoryResourceHandlerImpl.java | 44 +-
.../linux/resources/TestDefaultOOMHandler.java | 288 +--
.../TestNetworkPacketTaggingHandlerImpl.java | 28 +-
.../linux/resources/TestResourceHandlerModule.java | 29 +-
.../TestTrafficControlBandwidthHandlerImpl.java | 43 +-
.../linux/resources/TestTrafficController.java | 40 +-
.../fpga/TestFpgaResourceHandlerImpl.java | 122 +-
.../resources/gpu/TestGpuResourceAllocator.java | 131 +-
.../resources/gpu/TestGpuResourceHandlerImpl.java | 120 +-
.../resources/numa/TestNumaResourceAllocator.java | 63 +-
.../numa/TestNumaResourceHandlerImpl.java | 10 +-
.../TestDelegatingLinuxContainerRuntime.java | 10 +-
.../linux/runtime/TestDockerContainerRuntime.java | 1363 ++++++-----
.../runtime/TestHdfsManifestToResourcesPlugin.java | 18 +-
.../runtime/TestImageTagToManifestPlugin.java | 26 +-
.../TestJavaSandboxLinuxContainerRuntime.java | 73 +-
.../linux/runtime/TestRuncContainerRuntime.java | 233 +-
.../linux/runtime/docker/TestDockerClient.java | 24 +-
.../runtime/docker/TestDockerCommandExecutor.java | 15 +-
.../runtime/docker/TestDockerImagesCommand.java | 13 +-
.../runtime/docker/TestDockerInspectCommand.java | 8 +-
.../runtime/docker/TestDockerKillCommand.java | 8 +-
.../runtime/docker/TestDockerLoadCommand.java | 8 +-
.../runtime/docker/TestDockerPullCommand.java | 8 +-
.../linux/runtime/docker/TestDockerRmCommand.java | 8 +-
.../linux/runtime/docker/TestDockerRunCommand.java | 8 +-
.../runtime/docker/TestDockerStartCommand.java | 8 +-
.../runtime/docker/TestDockerStopCommand.java | 8 +-
.../runtime/docker/TestDockerVolumeCommand.java | 12 +-
.../localizer/TestContainerLocalizer.java | 117 +-
.../localizer/TestLocalCacheCleanup.java | 6 +-
.../localizer/TestLocalCacheDirectoryManager.java | 56 +-
.../localizer/TestLocalResource.java | 6 +-
.../localizer/TestLocalResourcesTrackerImpl.java | 174 +-
.../localizer/TestLocalizedResource.java | 4 +-
.../localizer/TestResourceLocalizationService.java | 221 +-
.../localizer/TestResourceSet.java | 32 +-
.../sharedcache/TestSharedCacheUploadService.java | 4 +-
.../sharedcache/TestSharedCacheUploader.java | 8 +-
.../logaggregation/TestAppLogAggregatorImpl.java | 12 +-
.../logaggregation/TestLogAggregationService.java | 248 +-
.../loghandler/TestNonAggregatingLogHandler.java | 14 +-
.../monitor/TestContainerMetrics.java | 28 +-
.../monitor/TestContainersMonitor.java | 78 +-
.../TestContainersMonitorResourceChange.java | 33 +-
.../resourceplugin/TestResourcePluginManager.java | 50 +-
.../resourceplugin/com/nec/TestNECVEPlugin.java | 154 +-
.../com/nec/TestVEDeviceDiscoverer.java | 135 +-
.../nvidia/TestNvidiaGPUPluginForRuntimeV2.java | 127 +-
.../deviceframework/TestDeviceMappingManager.java | 32 +-
.../deviceframework/TestDevicePluginAdapter.java | 137 +-
.../resourceplugin/fpga/TestAoclOutputParser.java | 4 +-
.../resourceplugin/fpga/TestFpgaDiscoverer.java | 186 +-
.../fpga/TestIntelFpgaOpenclPlugin.java | 18 +-
.../resourceplugin/gpu/TestGpuDiscoverer.java | 219 +-
.../resourceplugin/gpu/TestGpuResourcePlugin.java | 42 +-
.../gpu/TestNvidiaDockerV1CommandPlugin.java | 29 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 15 +-
...tAllocationBasedResourceUtilizationTracker.java | 14 +-
...estContainerSchedulerBehaviorCompatibility.java | 11 +-
...ContainerSchedulerOppContainersByResources.java | 17 +-
.../scheduler/TestContainerSchedulerQueuing.java | 165 +-
.../scheduler/TestContainerSchedulerRecovery.java | 12 +-
.../executor/TestContainerReapContext.java | 8 +-
.../nodemanager/health/TestExceptionReporter.java | 2 +-
.../health/TestNodeHealthCheckerService.java | 59 +-
.../health/TestNodeHealthScriptRunner.java | 51 +-
.../tracker/TestNMLogAggregationStatusTracker.java | 51 +-
.../metrics/TestNodeManagerMetrics.java | 36 +-
.../TestConfigurationNodeAttributesProvider.java | 82 +-
.../TestConfigurationNodeLabelsProvider.java | 30 +-
.../TestScriptBasedNodeAttributesProvider.java | 46 +-
.../TestScriptBasedNodeLabelsProvider.java | 45 +-
.../recovery/TestNMLeveldbStateStoreService.java | 60 +-
.../scheduler/TestDistributedScheduler.java | 62 +-
.../TestNMContainerTokenSecretManager.java | 12 +-
.../security/TestNMTokenSecretManagerInNM.java | 12 +-
.../timelineservice/TestNMTimelinePublisher.java | 97 +-
.../util/TestCgroupsLCEResourcesHandler.java | 132 +-
.../util/TestNodeManagerHardwareUtils.java | 50 +-
.../nodemanager/util/TestProcessIdFileReader.java | 21 +-
.../nodemanager/webapp/TestContainerLogsPage.java | 39 +-
.../server/nodemanager/webapp/TestNMAppsPage.java | 15 +-
.../webapp/TestNMContainerWebSocket.java | 20 +-
.../server/nodemanager/webapp/TestNMWebFilter.java | 10 +-
.../server/nodemanager/webapp/TestNMWebServer.java | 19 +-
.../nodemanager/webapp/TestNMWebServices.java | 93 +-
.../nodemanager/webapp/TestNMWebServicesApps.java | 56 +-
.../webapp/TestNMWebServicesAuxServices.java | 31 +-
.../webapp/TestNMWebServicesContainers.java | 64 +-
.../nodemanager/webapp/TestNMWebTerminal.java | 12 +-
.../dao/gpu/TestGpuDeviceInformationParser.java | 26 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 11 +-
.../server/resourcemanager/ClientRMService.java | 4 +-
.../DecommissioningNodesWatcher.java | 2 +-
.../server/resourcemanager/NodesListManager.java | 4 +-
.../resourcemanager/RMActiveServiceContext.java | 4 +-
.../yarn/server/resourcemanager/RMContextImpl.java | 2 +-
.../yarn/server/resourcemanager/RMServerUtils.java | 4 +-
.../federation/FederationClientMethod.java | 2 +-
.../federation/FederationStateStoreService.java | 4 +-
.../ProportionalCapacityPreemptionPolicy.java | 4 +-
.../invariants/ReservationInvariantsChecker.java | 2 +-
.../resourcemanager/recovery/ZKRMStateStore.java | 4 +-
.../reservation/AbstractReservationSystem.java | 4 +-
.../reservation/AbstractSchedulerPlanFollower.java | 2 +-
.../reservation/CapacitySchedulerPlanFollower.java | 2 +-
.../reservation/FairSchedulerPlanFollower.java | 2 +-
.../resourcemanager/reservation/InMemoryPlan.java | 4 +-
.../resourcemanager/reservation/PlanFollower.java | 2 +-
.../reservation/ReservationInputValidator.java | 2 +-
.../planning/SimpleCapacityReplanner.java | 4 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 6 +-
.../rmapp/attempt/AMLivelinessMonitor.java | 2 +-
.../rmapp/monitor/RMAppLifetimeMonitor.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 4 +-
.../resourcemanager/scheduler/QueueMetrics.java | 3 +-
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../scheduler/activities/ActivitiesManager.java | 2 +-
.../scheduler/capacity/AbstractLeafQueue.java | 2 +-
.../capacity/AutoCreatedQueueDeletionPolicy.java | 2 +-
.../capacity/CapacitySchedulerContext.java | 2 +-
.../QueueConfigurationAutoRefreshPolicy.java | 4 +-
.../capacity/QueueManagementDynamicEditPolicy.java | 4 +-
.../allocator/RegularContainerAllocator.java | 90 +-
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 4 +-
.../fair/AllocationFileLoaderService.java | 4 +-
.../security/DelegationTokenRenewer.java | 9 +-
.../resourcemanager/webapp/RMWebAppFilter.java | 1 +
.../webapp/dao/AppActivitiesInfo.java | 2 +-
.../yarn/server/resourcemanager/ACLsTestBase.java | 4 +-
.../ApplicationMasterServiceTestBase.java | 90 +-
.../yarn/server/resourcemanager/HATestUtil.java | 2 +-
.../hadoop/yarn/server/resourcemanager/MockRM.java | 45 +-
.../yarn/server/resourcemanager/NodeManager.java | 7 +-
.../ParameterizedSchedulerTestBase.java | 14 +-
.../server/resourcemanager/QueueACLsTestBase.java | 35 +-
.../resourcemanager/ReservationACLsTestBase.java | 58 +-
.../resourcemanager/TestAMAuthorization.java | 65 +-
.../server/resourcemanager/TestAppManager.java | 283 +--
.../TestAppManagerWithFairScheduler.java | 31 +-
.../resourcemanager/TestApplicationACLs.java | 111 +-
.../resourcemanager/TestApplicationCleanup.java | 53 +-
.../TestApplicationMasterLauncher.java | 67 +-
.../TestApplicationMasterServiceCapacity.java | 41 +-
.../TestApplicationMasterServiceFair.java | 4 +-
.../TestApplicationMasterServiceInterceptor.java | 25 +-
.../TestCapacitySchedulerMetrics.java | 24 +-
.../resourcemanager/TestClientRMService.java | 701 +++---
.../server/resourcemanager/TestClientRMTokens.java | 21 +-
.../server/resourcemanager/TestClusterMetrics.java | 27 +-
.../TestContainerResourceUsage.java | 99 +-
.../TestDecommissioningNodesWatcher.java | 30 +-
.../resourcemanager/TestLeaderElectorService.java | 14 +-
.../resourcemanager/TestMoveApplication.java | 23 +-
.../TestNodeBlacklistingOnAMFailures.java | 46 +-
...stOpportunisticContainerAllocatorAMService.java | 236 +-
.../hadoop/yarn/server/resourcemanager/TestRM.java | 175 +-
.../server/resourcemanager/TestRMAdminService.java | 210 +-
.../server/resourcemanager/TestRMAuditLogger.java | 17 +-
.../server/resourcemanager/TestRMDispatcher.java | 10 +-
.../yarn/server/resourcemanager/TestRMHA.java | 180 +-
.../resourcemanager/TestRMHAForNodeLabels.java | 20 +-
.../server/resourcemanager/TestRMHAMetrics.java | 23 +-
.../TestRMHATimelineCollectors.java | 8 +-
.../resourcemanager/TestRMNodeTransitions.java | 598 +++--
.../resourcemanager/TestRMProxyUsersConf.java | 22 +-
.../yarn/server/resourcemanager/TestRMRestart.java | 715 +++---
.../server/resourcemanager/TestRMServerUtils.java | 111 +-
.../resourcemanager/TestRMStoreCommands.java | 44 +-
.../resourcemanager/TestRMTimelineService.java | 16 +-
.../TestReservationSystemWithRMHA.java | 4 +-
.../resourcemanager/TestResourceManager.java | 64 +-
.../resourcemanager/TestResourceManagerMXBean.java | 7 +-
.../TestResourceTrackerService.java | 751 +++---
.../resourcemanager/TestSignalContainer.java | 9 +-
.../resourcemanager/TestTokenClientRMService.java | 31 +-
.../TestWorkPreservingRMRestart.java | 275 ++-
.../TestWorkPreservingUnmanagedAM.java | 67 +-
.../ahs/TestRMApplicationHistoryWriter.java | 84 +-
.../applicationsmanager/TestAMLaunchFailure.java | 234 --
.../TestAMRMRPCNodeUpdates.java | 86 +-
.../applicationsmanager/TestAMRMRPCResponseId.java | 27 +-
.../applicationsmanager/TestAMRestart.java | 236 +-
.../applicationsmanager/TestASMStateMachine.java | 217 --
.../TestApplicationMasterExpiry.java | 174 --
.../TestSchedulerNegotiator.java | 164 --
.../blacklist/TestBlacklistManager.java | 53 +-
.../TestFederationRMStateStoreService.java | 112 +-
.../TestFederationStateStoreServiceMetrics.java | 4 +-
.../TestRMAppLogAggregationStatus.java | 129 +-
.../TestCombinedSystemMetricsPublisher.java | 77 +-
.../metrics/TestSystemMetricsPublisher.java | 224 +-
.../metrics/TestSystemMetricsPublisherForV2.java | 74 +-
.../monitor/TestSchedulingMonitor.java | 16 +-
.../TestPreemptionForQueueWithPriorities.java | 6 +-
.../TestProportionalCapacityPreemptionPolicy.java | 43 +-
...lCapacityPreemptionPolicyForNodePartitions.java | 6 +-
...acityPreemptionPolicyForReservedContainers.java | 6 +-
...lCapacityPreemptionPolicyInterQueueWithDRF.java | 6 +-
...ortionalCapacityPreemptionPolicyIntraQueue.java | 6 +-
...cityPreemptionPolicyIntraQueueFairOrdering.java | 6 +-
...apacityPreemptionPolicyIntraQueueUserLimit.java | 6 +-
...lCapacityPreemptionPolicyIntraQueueWithDRF.java | 6 +-
...ionalCapacityPreemptionPolicyMockFramework.java | 51 +-
...alCapacityPreemptionPolicyPreemptToBalance.java | 4 +-
...ionalCapacityPreemptionPolicyMockFramework.java | 26 +-
.../invariants/TestMetricsInvariantChecker.java | 22 +-
.../TestFileSystemNodeAttributeStore.java | 65 +-
.../nodelabels/TestNodeAttributesManager.java | 82 +-
.../nodelabels/TestNodeLabelFileReplication.java | 10 +-
.../TestRMDelegatedNodeLabelsUpdater.java | 15 +-
.../nodelabels/TestRMNodeLabelsManager.java | 205 +-
.../placement/TestAppNameMappingPlacementRule.java | 10 +-
.../placement/TestFairQueuePlacementUtils.java | 110 +-
.../placement/TestPlacementFactory.java | 25 +-
.../placement/TestPlacementManager.java | 23 +-
.../placement/TestPlacementRuleFS.java | 52 +-
.../TestUserGroupMappingPlacementRule.java | 16 +-
.../csmappingrule/TestCSMappingPlacementRule.java | 76 +-
.../placement/csmappingrule/TestMappingRule.java | 6 +-
.../csmappingrule/TestMappingRuleActions.java | 7 +-
.../csmappingrule/TestMappingRuleMatchers.java | 66 +-
.../TestMappingRuleValidationContextImpl.java | 7 +-
.../csmappingrule/TestVariableContext.java | 70 +-
.../preprocessor/TestContextProcessor.java | 6 +-
.../preprocessor/TestNodeLabelProcessor.java | 2 +-
.../preprocessor/TestQueueProcessor.java | 7 +-
.../preprocessor/TestTagAddProcessor.java | 8 +-
.../recovery/RMStateStoreTestBase.java | 139 +-
.../TestCheckRemoveZKNodeRMStateStore.java | 89 +-
.../recovery/TestFSRMStateStore.java | 46 +-
.../recovery/TestLeveldbRMStateStore.java | 47 +-
.../recovery/TestMemoryRMStateStore.java | 12 +-
.../resourcemanager/recovery/TestProtos.java | 8 +-
.../recovery/TestRMStateStoreUtils.java | 8 +-
.../recovery/TestZKRMStateStore.java | 346 ++-
.../recovery/TestZKRMStateStorePerf.java | 10 +-
.../TestZKRMStateStoreZKClientConnections.java | 28 +-
.../reservation/BaseSharingPolicyTest.java | 18 +-
.../reservation/ReservationSystemTestUtil.java | 25 +-
.../reservation/TestCapacityOverTimePolicy.java | 28 +-
.../TestCapacitySchedulerPlanFollower.java | 33 +-
.../reservation/TestFairSchedulerPlanFollower.java | 29 +-
.../reservation/TestInMemoryPlan.java | 206 +-
.../TestInMemoryReservationAllocation.java | 47 +-
.../reservation/TestNoOverCommitPolicy.java | 28 +-
.../TestPeriodicRLESparseResourceAllocation.java | 73 +-
.../TestRLESparseResourceAllocation.java | 127 +-
.../reservation/TestReservationInputValidator.java | 269 +--
.../reservation/TestReservationSystem.java | 49 +-
.../reservation/TestReservationSystemUtil.java | 20 +-
.../reservation/TestSchedulerPlanFollowerBase.java | 50 +-
.../reservation/planning/TestAlignedPlanner.java | 314 +--
.../planning/TestGreedyReservationAgent.java | 287 ++-
.../planning/TestReservationAgents.java | 31 +-
.../planning/TestSimpleCapacityReplanner.java | 65 +-
.../resource/TestResourceProfiles.java | 55 +-
.../resourcetracker/TestNMExpiry.java | 12 +-
.../resourcetracker/TestNMReconnect.java | 70 +-
.../resourcetracker/TestRMNMRPCResponseId.java | 24 +-
.../rmapp/TestApplicationLifetimeMonitor.java | 170 +-
.../rmapp/TestNodesListManager.java | 47 +-
.../rmapp/TestRMAppTransitions.java | 339 +--
.../rmapp/attempt/TestAMLivelinessMonitor.java | 13 +-
.../attempt/TestRMAppAttemptImplDiagnostics.java | 31 +-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 477 ++--
.../rmcontainer/TestRMContainerImpl.java | 56 +-
.../scheduler/TestAbstractYarnScheduler.java | 255 +-
.../scheduler/TestAppSchedulingInfo.java | 69 +-
.../scheduler/TestClusterNodeTracker.java | 93 +-
.../TestConfigurationMutationACLPolicies.java | 10 +-
.../scheduler/TestPartitionQueueMetrics.java | 98 +-
.../scheduler/TestQueueMetrics.java | 83 +-
.../TestQueueMetricsForCustomResources.java | 24 +-
.../scheduler/TestResourceUsage.java | 24 +-
.../scheduler/TestSchedulerApplicationAttempt.java | 80 +-
.../scheduler/TestSchedulerHealth.java | 124 +-
.../scheduler/TestSchedulerOvercommit.java | 52 +-
.../scheduler/TestSchedulerUtils.java | 206 +-
.../TestSchedulingWithAllocationRequestId.java | 54 +-
.../activities/TestActivitiesManager.java | 111 +-
.../CapacitySchedulerPreemptionTestBase.java | 15 +-
.../CapacitySchedulerQueueCalculationTestBase.java | 4 +-
.../capacity/CapacitySchedulerQueueHelpers.java | 17 +-
.../capacity/CapacitySchedulerTestUtilities.java | 11 +-
.../scheduler/capacity/QueueAssertionBuilder.java | 15 +-
.../TestAMAllocatedToNonExclusivePartition.java | 11 +-
.../TestAbsoluteResourceConfiguration.java | 628 ++---
.../TestAbsoluteResourceWithAutoQueue.java | 45 +-
.../scheduler/capacity/TestApplicationLimits.java | 35 +-
.../capacity/TestApplicationLimitsByPartition.java | 171 +-
.../capacity/TestApplicationPriority.java | 142 +-
.../TestApplicationPriorityACLConfiguration.java | 11 +-
.../capacity/TestApplicationPriorityACLs.java | 14 +-
.../TestAutoCreatedQueueDeletionPolicy.java | 79 +-
.../capacity/TestAutoCreatedQueueTemplate.java | 90 +-
.../capacity/TestCSAllocateCustomResource.java | 50 +-
.../capacity/TestCSMaxRunningAppsEnforcer.java | 8 +-
.../scheduler/capacity/TestCSQueueStore.java | 15 +-
.../scheduler/capacity/TestCapacityScheduler.java | 433 ++--
.../TestCapacitySchedulerAmbiguousLeafs.java | 2 +-
.../capacity/TestCapacitySchedulerApps.java | 283 +--
.../TestCapacitySchedulerAsyncScheduling.java | 188 +-
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 19 +-
...apacitySchedulerAutoCreatedQueuePreemption.java | 13 +-
.../TestCapacitySchedulerAutoQueueCreation.java | 97 +-
.../TestCapacitySchedulerConfigValidator.java | 224 +-
.../TestCapacitySchedulerConfiguration.java | 8 +-
.../TestCapacitySchedulerDynamicBehavior.java | 22 +-
.../TestCapacitySchedulerLazyPreemption.java | 125 +-
.../TestCapacitySchedulerMaxParallelApps.java | 49 +-
.../capacity/TestCapacitySchedulerMultiNodes.java | 220 +-
...tCapacitySchedulerMultiNodesWithPreemption.java | 37 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 508 ++--
...lerNewQueueAutoCreationWithCapacityVectors.java | 101 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 131 +-
.../capacity/TestCapacitySchedulerNodes.java | 84 +-
.../capacity/TestCapacitySchedulerPerf.java | 28 +-
.../capacity/TestCapacitySchedulerQueueACLs.java | 8 +-
.../TestCapacitySchedulerQueueMappingFactory.java | 36 +-
.../capacity/TestCapacitySchedulerQueues.java | 440 ++--
...stCapacitySchedulerSchedulingRequestUpdate.java | 2 +-
.../TestCapacitySchedulerSurgicalPreemption.java | 139 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 69 +-
...estCapacitySchedulerWithMultiResourceTypes.java | 74 +-
...stCapacitySchedulerWorkflowPriorityMapping.java | 8 +-
.../scheduler/capacity/TestChildQueueOrder.java | 12 +-
.../capacity/TestConfigurationProperties.java | 88 +-
.../capacity/TestContainerAllocation.java | 268 +-
.../scheduler/capacity/TestContainerResizing.java | 219 +-
.../capacity/TestIncreaseAllocationExpirer.java | 83 +-
.../scheduler/capacity/TestLeafQueue.java | 157 +-
.../TestMixedQueueResourceCalculation.java | 28 +-
.../capacity/TestNodeLabelContainerAllocation.java | 363 +--
.../scheduler/capacity/TestParentQueue.java | 140 +-
.../scheduler/capacity/TestQueueCapacities.java | 38 +-
.../capacity/TestQueueCapacityVector.java | 44 +-
.../TestQueueConfigurationAutoRefreshPolicy.java | 36 +-
.../TestQueueManagementDynamicEditPolicy.java | 8 +-
.../scheduler/capacity/TestQueueMappings.java | 42 +-
.../scheduler/capacity/TestQueueParsing.java | 388 +--
.../scheduler/capacity/TestQueuePath.java | 106 +-
.../scheduler/capacity/TestQueueState.java | 78 +-
.../scheduler/capacity/TestQueueStateManager.java | 52 +-
.../scheduler/capacity/TestReservationQueue.java | 16 +-
.../scheduler/capacity/TestReservations.java | 45 +-
.../scheduler/capacity/TestResourceVector.java | 55 +-
.../TestSchedulingRequestContainerAllocation.java | 232 +-
...tSchedulingRequestContainerAllocationAsync.java | 67 +-
.../TestUniformQueueResourceCalculation.java | 2 +-
.../scheduler/capacity/TestUsersManager.java | 21 +-
.../TestWorkPreservingRMRestartForNodeLabel.java | 25 +-
.../capacity/conf/ConfigurationStoreBaseTest.java | 14 +-
.../conf/PersistentConfigurationStoreBaseTest.java | 12 +-
.../conf/TestConfigurationUpdateAssembler.java | 14 +-
.../conf/TestFSSchedulerConfigurationStore.java | 36 +-
.../conf/TestInMemoryConfigurationStore.java | 4 +-
.../conf/TestLeveldbConfigurationStore.java | 41 +-
.../conf/TestMutableCSConfigurationProvider.java | 16 +-
.../conf/TestQueueCapacityConfigParser.java | 81 +-
.../capacity/conf/TestZKConfigurationStore.java | 187 +-
.../capacity/placement/TestMappingRuleCreator.java | 119 +-
.../converter/TestLegacyMappingRuleToJson.java | 18 +-
...TestPriorityUtilizationQueueOrderingPolicy.java | 6 +-
.../TestDeactivatedLeafQueuesByLabel.java | 5 +-
.../constraint/TestAllocationTagsManager.java | 158 +-
.../constraint/TestAllocationTagsNamespace.java | 75 +-
.../constraint/TestBatchedRequestsIterators.java | 11 +-
.../TestPlacementConstraintManagerService.java | 116 +-
.../constraint/TestPlacementConstraintsUtil.java | 194 +-
.../constraint/TestPlacementProcessor.java | 118 +-
.../constraint/algorithm/TestCircularIterator.java | 16 +-
.../algorithm/TestLocalAllocationTagsManager.java | 30 +-
...CentralizedOpportunisticContainerAllocator.java | 35 +-
.../distributed/TestNodeQueueLoadMonitor.java | 228 +-
.../scheduler/fair/FairSchedulerTestBase.java | 7 +-
.../fair/TestAllocationFileLoaderService.java | 231 +-
.../scheduler/fair/TestAppRunnability.java | 18 +-
.../fair/TestApplicationMasterServiceWithFS.java | 37 +-
.../scheduler/fair/TestComputeFairShares.java | 26 +-
.../scheduler/fair/TestConfigurableResource.java | 17 +-
.../scheduler/fair/TestContinuousScheduling.java | 52 +-
.../scheduler/fair/TestFSAppAttempt.java | 91 +-
.../scheduler/fair/TestFSAppStarvation.java | 56 +-
.../scheduler/fair/TestFSLeafQueue.java | 55 +-
.../scheduler/fair/TestFSParentQueue.java | 10 +-
.../scheduler/fair/TestFSQueueMetrics.java | 214 +-
.../scheduler/fair/TestFSSchedulerNode.java | 137 +-
.../scheduler/fair/TestFairScheduler.java | 1245 +++++-----
.../fair/TestFairSchedulerConfiguration.java | 363 +--
.../scheduler/fair/TestFairSchedulerFairShare.java | 12 +-
.../fair/TestFairSchedulerPreemption.java | 135 +-
.../scheduler/fair/TestFairSchedulerUtilities.java | 8 +-
.../TestFairSchedulerWithMultiResourceTypes.java | 12 +-
.../scheduler/fair/TestMaxRunningAppsEnforcer.java | 8 +-
.../scheduler/fair/TestQueueManager.java | 295 +--
.../fair/TestQueueManagerRealScheduler.java | 12 +-
.../scheduler/fair/TestQueuePlacementPolicy.java | 34 +-
.../scheduler/fair/TestSchedulingPolicy.java | 104 +-
.../scheduler/fair/TestSchedulingUpdate.java | 53 +-
.../fair/TestVisitedResourceRequestTracker.java | 31 +-
.../converter/FSConfigConverterTestCommons.java | 4 +-
.../converter/TestConvertedConfigValidator.java | 24 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 158 +-
.../converter/TestFSConfigToCSConfigConverter.java | 389 ++-
.../TestFSConfigToCSConfigConverterMain.java | 60 +-
.../TestFSConfigToCSConfigRuleHandler.java | 45 +-
.../fair/converter/TestFSQueueConverter.java | 314 ++-
.../fair/converter/TestFSYarnSiteConverter.java | 212 +-
.../converter/TestQueuePlacementConverter.java | 82 +-
.../TestWeightToPercentageConverter.java | 101 +-
.../TestWeightToWeightConverter.java | 49 +-
.../TestDominantResourceFairnessPolicy.java | 183 +-
.../scheduler/fair/policies/TestEmptyQueues.java | 16 +-
.../scheduler/fifo/TestFifoScheduler.java | 202 +-
.../TestSingleConstraintAppPlacementAllocator.java | 63 +-
.../scheduler/policy/MockSchedulableEntity.java | 2 +-
.../scheduler/policy/TestFairOrderingPolicy.java | 61 +-
.../scheduler/policy/TestFifoOrderingPolicy.java | 59 +-
.../TestFifoOrderingPolicyForPendingApps.java | 6 +-
...tFifoOrderingPolicyWithExclusivePartitions.java | 46 +-
.../resourcemanager/security/TestAMRMTokens.java | 90 +-
.../security/TestClientToAMTokens.java | 70 +-
.../security/TestDelegationTokenRenewer.java | 194 +-
.../TestDelegationTokenRenewerLifecycle.java | 2 +-
.../security/TestProxyCAManager.java | 30 +-
.../security/TestRMAuthenticationFilter.java | 16 +-
.../security/TestRMDelegationTokens.java | 35 +-
.../volume/csi/TestVolumeCapabilityRange.java | 51 +-
.../volume/csi/TestVolumeLifecycle.java | 62 +-
.../volume/csi/TestVolumeMetaData.java | 147 +-
.../volume/csi/TestVolumeProcessor.java | 57 +-
.../webapp/ActivitiesTestUtils.java | 33 +-
.../server/resourcemanager/webapp/TestAppPage.java | 2 +-
.../webapp/TestApplicationsRequestBuilder.java | 186 +-
.../resourcemanager/webapp/TestNodesPage.java | 41 +-
.../resourcemanager/webapp/TestRMWebApp.java | 24 +-
.../webapp/TestRMWebAppFairScheduler.java | 6 +-
.../webapp/TestRMWebServiceAppsNodelabel.java | 36 +-
.../resourcemanager/webapp/TestRMWebServices.java | 225 +-
.../webapp/TestRMWebServicesAppAttempts.java | 55 +-
.../TestRMWebServicesAppCustomResourceTypes.java | 8 +-
.../webapp/TestRMWebServicesApps.java | 513 ++--
.../TestRMWebServicesAppsCustomResourceTypes.java | 14 +-
.../webapp/TestRMWebServicesAppsModification.java | 155 +-
.../webapp/TestRMWebServicesCapacitySched.java | 47 +-
...TestRMWebServicesCapacitySchedDefaultLabel.java | 27 +-
...estRMWebServicesCapacitySchedDynamicConfig.java | 30 +-
...icesCapacitySchedDynamicConfigAbsoluteMode.java | 30 +-
...rvicesCapacitySchedDynamicConfigWeightMode.java | 30 +-
...cesCapacitySchedDynamicConfigWeightModeDQC.java | 30 +-
...ebServicesCapacitySchedLegacyQueueCreation.java | 30 +-
...pacitySchedLegacyQueueCreationAbsoluteMode.java | 32 +-
...WebServicesCapacitySchedulerConfigMutation.java | 40 +-
...estRMWebServicesCapacitySchedulerMixedMode.java | 6 +-
...itySchedulerMixedModeAbsoluteAndPercentage.java | 6 +-
...lerMixedModeAbsoluteAndPercentageAndWeight.java | 6 +-
...edModeAbsoluteAndPercentageAndWeightVector.java | 6 +-
...edulerMixedModeAbsoluteAndPercentageVector.java | 6 +-
...apacitySchedulerMixedModeAbsoluteAndWeight.java | 6 +-
...ySchedulerMixedModeAbsoluteAndWeightVector.java | 6 +-
...acitySchedulerMixedModePercentageAndWeight.java | 6 +-
...chedulerMixedModePercentageAndWeightVector.java | 6 +-
.../TestRMWebServicesConfigurationMutation.java | 50 +-
.../webapp/TestRMWebServicesContainers.java | 10 +-
...estRMWebServicesCustomResourceTypesCommons.java | 6 +-
...RMWebServicesDelegationTokenAuthentication.java | 65 +-
.../webapp/TestRMWebServicesDelegationTokens.java | 59 +-
.../TestRMWebServicesForCSWithPartitions.java | 238 +-
...TestRMWebServicesHttpStaticUserPermissions.java | 14 +-
.../webapp/TestRMWebServicesNodeLabels.java | 44 +-
.../webapp/TestRMWebServicesNodes.java | 146 +-
.../webapp/TestRMWebServicesReservation.java | 160 +-
.../TestRMWebServicesSchedulerActivities.java | 51 +-
...esSchedulerActivitiesWithMultiNodesEnabled.java | 44 +-
.../webapp/TestRMWebappAuthentication.java | 38 +-
.../webapp/TestRedirectionErrorPage.java | 2 +-
.../resourcemanager/webapp/TestWebServiceUtil.java | 6 +-
.../webapp/dao/TestFairSchedulerQueueInfo.java | 11 +-
.../FairSchedulerJsonVerifications.java | 59 +-
.../FairSchedulerXmlVerifications.java | 41 +-
.../TestRMWebServicesFairScheduler.java | 14 +-
...ebServicesFairSchedulerCustomResourceTypes.java | 14 +-
.../webapp/helper/AppInfoJsonVerifications.java | 106 +-
.../webapp/helper/AppInfoXmlVerifications.java | 110 +-
.../helper/JsonCustomResourceTypeTestcase.java | 4 +-
.../helper/ResourceRequestsJsonVerifications.java | 105 +-
.../helper/ResourceRequestsXmlVerifications.java | 79 +-
.../helper/XmlCustomResourceTypeTestCase.java | 8 +-
.../hadoop/yarn/webapp/TestRMWithCSRFFilter.java | 40 +-
.../hadoop/yarn/webapp/TestRMWithXFSFilter.java | 21 +-
.../clientrm/FederationClientInterceptor.java | 20 +-
.../rmadmin/FederationRMAdminInterceptor.java | 4 +-
.../router/webapp/FederationInterceptorREST.java | 4 +-
.../router/clientrm/BaseRouterClientRMTest.java | 4 +-
.../webapp/MockDefaultRequestInterceptorREST.java | 2 +-
.../webapp/TestFederationInterceptorREST.java | 2 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 8 +-
.../pom.xml | 11 +-
.../TestDocumentStoreCollectionCreator.java | 14 +-
.../TestDocumentStoreTimelineReaderImpl.java | 107 +-
.../TestDocumentStoreTimelineWriterImpl.java | 32 +-
.../collection/TestDocumentOperations.java | 108 +-
.../cosmosdb/TestCosmosDBDocumentStoreReader.java | 40 +-
.../cosmosdb/TestCosmosDBDocumentStoreWriter.java | 40 +-
.../pom.xml | 37 +-
.../AbstractTimelineReaderHBaseTestBase.java | 95 +-
.../reader/FlowActivityEntityListReader.java | 123 +
.../reader/FlowActivityEntityReader.java} | 44 +-
.../reader/FlowActivityEntitySetReader.java | 124 +
.../reader/FlowRunEntityReader.java} | 43 +-
.../reader/FlowRunEntitySetReader.java} | 48 +-
.../TestTimelineReaderWebServicesHBaseStorage.java | 537 ++--
.../reader/TimelineEntityListReader.java | 122 +
.../reader/TimelineEntitySetReader.java | 125 +
.../storage/TestHBaseTimelineStorageApps.java | 94 +-
.../storage/TestHBaseTimelineStorageDomain.java | 16 +-
.../storage/TestHBaseTimelineStorageEntities.java | 99 +-
.../storage/TestHBaseTimelineStorageSchema.java | 14 +-
.../storage/TestTimelineReaderHBaseDown.java | 25 +-
.../storage/TestTimelineWriterHBaseDown.java | 16 +-
.../storage/flow/TestHBaseStorageFlowActivity.java | 18 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 20 +-
.../flow/TestHBaseStorageFlowRunCompaction.java | 32 +-
.../storage/HBaseTimelineWriterImpl.java | 8 +-
.../storage/common/ColumnRWHelper.java | 2 +-
.../server/webproxy/amfilter/TestAmFilter.java | 2 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 10 +-
.../src/main/webapp/WEB-INF/wro.properties | 16 +
.../app/components/queue-usage-donut-chart.js | 39 +-
.../yarn-queue-partition-capacity-labels.js | 34 +-
.../src/main/webapp/app/helpers/get-from-map.js | 72 +
.../webapp/app/models/yarn-queue/capacity-queue.js | 12 +-
.../app/serializers/yarn-queue/capacity-queue.js | 30 +-
.../main/webapp/app/templates/cluster-overview.hbs | 70 +-
.../app/templates/components/partition-usage.hbs | 66 +
.../yarn-queue-partition-capacity-labels.hbs | 295 ++-
.../components/yarn-queue/capacity-queue.hbs | 4 +-
.../hadoop-yarn-ui/src/main/webapp/bower.json | 4 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 1 -
.../webapp/tests/unit/helpers/get-from-map-test.js | 16 +-
.../LICENSE-uriparser2.txt | 0
2265 files changed, 85313 insertions(+), 48636 deletions(-)
delete mode 100644 hadoop-cloud-storage-project/hadoop-tos/src/main/assembly/hadoop-tos.xml
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/package-info.java (72%)
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/package-info.java (73%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/ExcludePrivateAnnotationsJDiffDoclet.java (100%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/ExcludePrivateAnnotationsStandardDoclet.java (100%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/IncludePublicAnnotationsJDiffDoclet.java (100%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/IncludePublicAnnotationsStandardDoclet.java (100%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/RootDocProcessor.java (100%)
rename hadoop-common-project/hadoop-annotations/src/main/{java => java8}/org/apache/hadoop/classification/tools/StabilityOptions.java (100%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/VectorIOBufferPool.java
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop}/util/Clock.java (96%)
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop}/util/MonotonicClock.java (95%)
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop}/util/SystemClock.java (94%)
copy {hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop}/util/UTCClock.java (97%)
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestName.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/AsyncRpcProtocolPBUtil.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterClientNamenodeProtocolServerSideTranslatorPB.java
copy hadoop-hdfs-project/{hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java => hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterClientProtocolTranslatorPB.java} (56%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterGetUserMappingsProtocolServerSideTranslatorPB.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterGetUserMappingsProtocolTranslatorPB.java
copy hadoop-hdfs-project/{hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java => hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterNamenodeProtocolServerSideTranslatorPB.java} (54%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterNamenodeProtocolTranslatorPB.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterRefreshUserMappingsProtocolServerSideTranslatorPB.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterRefreshUserMappingsProtocolTranslatorPB.java
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java (67%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterAsyncRpcFairnessPolicyController.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ThreadLocalContext.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/AsyncErasureCoding.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/AsyncQuota.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncCacheAdmin.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncClientProtocol.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncNamenodeProtocol.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncRpcClient.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncSnapshot.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncStoragePolicy.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncUserProtocol.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java => hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/package-info.java (56%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/ApplyFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/Async.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncApplyFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncBiFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncCatchFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncForEachRun.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncRun.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncUtil.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/CatchFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/FinallyFunction.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/package-info.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestAsyncRpcProtocolPBUtil.java
copy hadoop-hdfs-project/{hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/HFSTestCase.java => hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestClientProtocol.java} (68%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestClientProtocolServerSideTranslatorPB.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestClientProtocolTranslatorPB.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestRouterClientSideTranslatorPB.java
copy hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/{TestRouterRpcFairnessPolicyController.java => TestRouterAsyncRpcFairnessPolicyController.java} (53%)
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/RouterAsyncProtocolTestBase.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestAsyncRouterAdmin.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncCacheAdmin.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncClientProtocol.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncErasureCoding.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncMountTable.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncNamenodeProtocol.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncQuota.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRPCMultipleDestinationMountTableResolver.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpc.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcClient.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcMultiDestination.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcServer.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncSnapshot.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncStoragePolicy.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncUserProtocol.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncWebHdfsMethods.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/AsyncClass.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/BaseClass.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/SyncClass.java
create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/utils/TestAsyncUtil.java
copy hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestTrueCopyFilter.java => hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java (63%)
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java => hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java (62%)
rename hadoop-hdfs-project/hadoop-hdfs/src/test/{java => java8}/org/apache/hadoop/hdfs/TestDFSClientFailover.java (100%)
create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditorFlags.java (78%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InputStreamCallbacksImpl.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AbstractObjectInputStreamFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/AnalyticsStreamFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ClassicObjectInputStreamFactory.java
copy hadoop-tools/{hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/BlobAppendRequestParameters.java => hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/FactoryBindingParameters.java} (53%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/InputStreamType.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ObjectInputStream.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ObjectInputStreamCallbacks.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ObjectInputStreamFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/ObjectReadParameters.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/StreamFactoryRequirements.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/StreamIntegration.java
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/streams/package-info.java (73%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/PrefetchOptions.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/PrefetchingInputStreamFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/reading.md
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractAnalyticsStreamVectoredRead.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestEMRFSCompatibility.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAnalyticsAcceleratorStreamReading.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/streams/TestStreamFactories.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/resources/malformed_footer.parquet
create mode 100644 hadoop-tools/hadoop-aws/src/test/resources/multi_row_group.parquet
delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/MetricsConstants.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidIngressServiceException.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/AbfsBackoffMetricsEnum.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/AbfsReadFooterMetricsEnum.java
copy hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/BlobCopyProgress.java (73%)
rename hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/FileType.java (70%)
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/RetryValue.java
rename hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Clock.java => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/enums/StatisticTypeEnum.java (76%)
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBackoffMetrics.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobBlock.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlock.java
copy hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestClient.java => hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlockStatus.java (53%)
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbstractAbfsStatisticsSource.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobBlockManager.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlobIngressHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureBlockManager.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSBlockManager.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDFSIngressHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureDfsToBlobIngressFallbackHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AzureIngressHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlobDeleteHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlobRenameHandler.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlockEntry.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListActionTaker.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListBlobQueue.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ListResponseData.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/PathInformation.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java
copy hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{contracts/services/BlobAppendRequestParameters.java => services/RenamePendingJsonFormat.java} (56%)
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/VersionedFileStatus.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/MockIntercept.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java
rename hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/{ListResultSchemaTest.java => TestDfsListJsonParser.java} (93%)
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicityTestUtils.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsBackoffMetrics.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsReadFooterMetrics.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestListActionTaker.java
copy hadoop-tools/hadoop-azure/src/{main/java/org/apache/hadoop/fs/azurebfs/utils/Listener.java => test/java/org/apache/hadoop/fs/azurebfs/utils/AzcopyExecutionException.java} (54%)
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AzcopyToolHelper.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DirectoryStateHelper.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestAzcopyToolHelper.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterExpiry.java
delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/FlowActivityEntityListReader.java
copy hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/writer/TimelineEntitySetWriter.java => hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/FlowActivityEntityReader.java} (52%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/FlowActivityEntitySetReader.java
copy hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/writer/TimelineEntitySetWriter.java => hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/FlowRunEntityReader.java} (52%)
copy hadoop-yarn-project/hadoop-yarn/{hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/writer/TimelineEntitySetWriter.java => hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/FlowRunEntitySetReader.java} (51%)
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityListReader.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntitySetReader.java
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/wro.properties
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/get-from-map.js
create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/partition-usage.hbs
copy hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/HFSTestCase.java => hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tests/unit/helpers/get-from-map-test.js (75%)
copy hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/LICENSE => licenses-binary/LICENSE-uriparser2.txt (100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop) 01/01: HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation (Part 2: Add unit tests.).
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a commit to branch HADOOP-19236-02
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e392fad87b25ae31a7509d9d3d9ec5262ec067e7
Author: Jinglun <ji...@apache.org>
AuthorDate: Sun Jan 26 09:27:19 2025 +0800
HADOOP-19236. Incorporate VolcanoEngine Cloud TOS File System Implementation (Part 2: Add unit tests.).
Contributed by: ZhengHu, SunXin, XianyinXin, Rascal Wu, FangBo, Yuanzhihuan.
---
.../java/org/apache/hadoop/fs/tosfs/TestEnv.java | 36 +
.../org/apache/hadoop/fs/tosfs/TestRawFSUtils.java | 42 +
.../apache/hadoop/fs/tosfs/TestRawFileSystem.java | 55 +
.../apache/hadoop/fs/tosfs/TestTosChecksum.java | 126 ++
.../apache/hadoop/fs/tosfs/TestTosFileSystem.java | 53 +
.../hadoop/fs/tosfs/commit/BaseJobSuite.java | 268 ++++
.../hadoop/fs/tosfs/commit/CommitterTestBase.java | 433 ++++++
.../apache/hadoop/fs/tosfs/commit/JobSuite.java | 228 +++
.../hadoop/fs/tosfs/commit/MRJobTestBase.java | 241 ++++
.../hadoop/fs/tosfs/commit/TestCommitter.java | 29 +
.../apache/hadoop/fs/tosfs/commit/TestMRJob.java | 49 +
.../fs/tosfs/commit/TestMagicOutputStream.java | 203 +++
.../fs/tosfs/commit/mapred/CommitterTestBase.java | 381 +++++
.../hadoop/fs/tosfs/commit/mapred/JobSuite.java | 227 +++
.../fs/tosfs/commit/mapred/TestCommitter.java | 29 +
.../hadoop/fs/tosfs/contract/TestChecksum.java | 137 ++
.../hadoop/fs/tosfs/contract/TestCreate.java | 40 +
.../hadoop/fs/tosfs/contract/TestDelete.java | 69 +
.../hadoop/fs/tosfs/contract/TestDistCp.java | 46 +
.../fs/tosfs/contract/TestGetFileStatus.java | 169 +++
.../apache/hadoop/fs/tosfs/contract/TestMkdir.java | 39 +
.../apache/hadoop/fs/tosfs/contract/TestOpen.java | 75 +
.../hadoop/fs/tosfs/contract/TestRename.java | 281 ++++
.../hadoop/fs/tosfs/contract/TestRootDir.java | 37 +
.../apache/hadoop/fs/tosfs/contract/TestSeek.java | 37 +
.../hadoop/fs/tosfs/contract/TestUnbuffer.java | 37 +
.../apache/hadoop/fs/tosfs/contract/TestXAttr.java | 177 +++
.../hadoop/fs/tosfs/contract/TosContract.java | 61 +
.../fs/tosfs/object/ObjectStorageTestBase.java | 108 ++
.../hadoop/fs/tosfs/object/ObjectTestUtils.java | 127 ++
.../fs/tosfs/object/TestDirectoryStorage.java | 224 +++
.../object/TestObjectMultiRangeInputStream.java | 451 ++++++
.../fs/tosfs/object/TestObjectOutputStream.java | 423 ++++++
.../tosfs/object/TestObjectRangeInputStream.java | 144 ++
.../hadoop/fs/tosfs/object/TestObjectStorage.java | 1495 ++++++++++++++++++++
.../tosfs/object/tos/TestChainTOSInputStream.java | 245 ++++
.../object/tos/TestDelegationClientBuilder.java | 462 ++++++
.../fs/tosfs/object/tos/TestTOSInputStream.java | 153 ++
.../fs/tosfs/object/tos/TestTOSObjectStorage.java | 302 ++++
.../fs/tosfs/object/tos/TestTOSRetryPolicy.java | 210 +++
.../tos/auth/TestAbstractCredentialsProvider.java | 60 +
.../auth/TestDefaultCredentialsProviderChain.java | 209 +++
.../auth/TestEnvironmentCredentialsProvider.java | 67 +
.../tos/auth/TestSimpleCredentialsProvider.java | 76 +
.../apache/hadoop/fs/tosfs/ops/TestBaseFsOps.java | 306 ++++
.../apache/hadoop/fs/tosfs/ops/TestBaseOps.java | 78 +
.../hadoop/fs/tosfs/ops/TestDefaultFsOps.java | 104 ++
.../hadoop/fs/tosfs/ops/TestDirectoryFsOps.java | 83 ++
.../apache/hadoop/fs/tosfs/ops/TestRenameOp.java | 198 +++
.../org/apache/hadoop/fs/tosfs/util/TempFiles.java | 98 ++
.../apache/hadoop/fs/tosfs/util/TestFSUtils.java | 66 +
.../apache/hadoop/fs/tosfs/util/TestIterables.java | 165 +++
.../hadoop/fs/tosfs/util/TestLazyReload.java | 95 ++
.../org/apache/hadoop/fs/tosfs/util/TestRange.java | 87 ++
.../hadoop-tos/src/test/resources/contract/tos.xml | 113 ++
.../hadoop-tos/src/test/resources/core-site.xml | 59 +
.../hadoop-tos/src/test/resources/log4j.properties | 23 +
57 files changed, 9836 insertions(+)
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestEnv.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestEnv.java
new file mode 100644
index 00000000000..c95967bec50
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestEnv.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+
+public final class TestEnv {
+ public static final String ENV_TOS_UNIT_TEST_ENABLED = "TOS_UNIT_TEST_ENABLED";
+ private static final boolean TOS_TEST_ENABLED;
+
+ static {
+ TOS_TEST_ENABLED = ParseUtils.envAsBoolean(ENV_TOS_UNIT_TEST_ENABLED, true);
+ }
+
+ private TestEnv() {}
+
+ public static boolean checkTestEnabled() {
+ return TOS_TEST_ENABLED;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFSUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFSUtils.java
new file mode 100644
index 00000000000..511a13b8421
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFSUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestRawFSUtils {
+
+ @Test
+ public void testIsAncestor() {
+ assertTrue(RawFSUtils.inSubtree("/", "/"));
+ assertTrue(RawFSUtils.inSubtree("/", "/a"));
+ assertTrue(RawFSUtils.inSubtree("/a", "/a"));
+ assertFalse(RawFSUtils.inSubtree("/a", "/"));
+ assertTrue(RawFSUtils.inSubtree("/", "/a/b/c"));
+ assertFalse(RawFSUtils.inSubtree("/a/b/c", "/"));
+ assertTrue(RawFSUtils.inSubtree("/", "/a/b/c.txt"));
+ assertFalse(RawFSUtils.inSubtree("/a/b/c.txt", "/"));
+ assertTrue(RawFSUtils.inSubtree("/a/b/", "/a/b"));
+ assertTrue(RawFSUtils.inSubtree("/a/b/", "/a/b/c"));
+ assertFalse(RawFSUtils.inSubtree("/a/b/c", "/a/b"));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFileSystem.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFileSystem.java
new file mode 100644
index 00000000000..05c7bcb7e46
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestRawFileSystem.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TestRawFileSystem {
+ @Test
+ public void testInitializeFileSystem() throws URISyntaxException, IOException {
+ Configuration conf = new Configuration();
+ try (RawFileSystem fs = new RawFileSystem()) {
+ fs.initialize(new URI("filestore://bucket_a/a/b/c"), conf);
+ assertEquals("bucket_a", fs.bucket());
+
+ fs.initialize(new URI("filestore://bucket-/a/b/c"), conf);
+ assertEquals("bucket-", fs.bucket());
+
+ fs.initialize(new URI("filestore://-bucket/a/b/c"), conf);
+ assertEquals("-bucket", fs.bucket());
+ }
+ }
+
+ @Test
+ public void testBucketNotExist() {
+ Configuration conf = new Configuration();
+ RawFileSystem fs = new RawFileSystem();
+ assertThrows(FileNotFoundException.class,
+ () -> fs.initialize(new URI("tos://not-exist-bucket/"), conf), "Bucket doesn't exist.");
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosChecksum.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosChecksum.java
new file mode 100644
index 00000000000..87b80ae7737
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosChecksum.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.FileStoreKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.ChecksumType;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.util.TempFiles;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestTosChecksum {
+ private static final String FILE_STORE_ROOT = TempFiles.newTempDir("TestTosChecksum");
+ private static final String ALGORITHM_NAME = "mock-algorithm";
+ private static final String PREFIX = UUIDUtils.random();
+
+ private ObjectStorage objectStorage;
+
+ public void setObjectStorage(ObjectStorage objectStorage) {
+ this.objectStorage = objectStorage;
+ }
+
+
+ static Stream<Arguments> provideArguments() throws URISyntaxException {
+ List<Arguments> values = new ArrayList<>();
+
+ // Case 1: file store.
+ Configuration fileStoreConf = new Configuration();
+ fileStoreConf.set(FileStoreKeys.FS_FILESTORE_CHECKSUM_ALGORITHM, ALGORITHM_NAME);
+ fileStoreConf.set(FileStoreKeys.FS_FILESTORE_CHECKSUM_TYPE, ChecksumType.MD5.name());
+ fileStoreConf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key("filestore"), FILE_STORE_ROOT);
+ URI uri0 = new URI("filestore://" + TestUtility.bucket() + "/");
+
+ values.add(Arguments.of(
+ ChecksumType.MD5,
+ fileStoreConf,
+ uri0,
+ ObjectStorageFactory.create(uri0.getScheme(), uri0.getAuthority(), fileStoreConf)
+ ));
+
+ // Case 2: tos.
+ Configuration tosConf = new Configuration();
+ tosConf.set(TosKeys.FS_TOS_CHECKSUM_ALGORITHM, ALGORITHM_NAME);
+ tosConf.set(TosKeys.FS_TOS_CHECKSUM_TYPE, ChecksumType.CRC32C.name());
+ URI uri1 = new URI(TOS_SCHEME + "://" + TestUtility.bucket() + "/");
+
+ values.add(Arguments.of(
+ ChecksumType.CRC32C,
+ tosConf,
+ uri1,
+ ObjectStorageFactory.create(uri1.getScheme(), uri1.getAuthority(), tosConf)
+ ));
+
+ return values.stream();
+ }
+
+ @AfterEach
+ public void tearDown() {
+ objectStorage.deleteAll(PREFIX);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testChecksumInfo(ChecksumType type, Configuration conf, URI uri,
+ ObjectStorage objectStore) {
+ setObjectStorage(objectStore);
+
+ assertEquals(ALGORITHM_NAME, objectStore.checksumInfo().algorithm());
+ assertEquals(type, objectStore.checksumInfo().checksumType());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testFileChecksum(ChecksumType type, Configuration conf, URI uri,
+ ObjectStorage objectStore) throws Exception {
+ setObjectStorage(objectStore);
+
+ try (RawFileSystem fs = new RawFileSystem()) {
+ fs.initialize(uri, conf);
+ Path file = new Path("/" + PREFIX, "testFileChecksum");
+ fs.create(file).close();
+ FileChecksum checksum = fs.getFileChecksum(file, Long.MAX_VALUE);
+ assertEquals(ALGORITHM_NAME, checksum.getAlgorithmName());
+
+ String key = file.toString().substring(1);
+ byte[] checksumData = objectStore.head(key).checksum();
+ assertArrayEquals(checksumData, checksum.getBytes());
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosFileSystem.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosFileSystem.java
new file mode 100644
index 00000000000..8b70f261416
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosFileSystem.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestTosFileSystem {
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Test
+ public void testUriVerification() throws URISyntaxException, IOException {
+ Configuration conf = new Configuration(false);
+ conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://cluster-0/");
+
+ TosFileSystem tfs = new TosFileSystem();
+ assertThrows(IllegalArgumentException.class,
+ () -> tfs.initialize(new URI("hdfs://cluster/"), conf), "Expect invalid uri error.");
+ assertThrows(IllegalArgumentException.class, () -> tfs.initialize(new URI("/path"), conf),
+ "Expect invalid uri error.");
+ tfs.initialize(new URI(String.format("tos://%s/", TestUtility.bucket())), conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/BaseJobSuite.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/BaseJobSuite.java
new file mode 100644
index 00000000000..2ce55281c85
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/BaseJobSuite.java
@@ -0,0 +1,268 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.object.MultipartUpload;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public abstract class BaseJobSuite {
+ private static final Logger LOG = LoggerFactory.getLogger(BaseJobSuite.class);
+ public static final int DEFAULT_APP_ATTEMPT_ID = 1;
+ protected static final Text KEY_1 = new Text("key1");
+ protected static final Text KEY_2 = new Text("key2");
+ protected static final Text VAL_1 = new Text("val1");
+ protected static final Text VAL_2 = new Text("val2");
+
+ private Job job;
+ private String jobId;
+ private FileSystem fs;
+ private Path outputPath;
+ private ObjectStorage storage;
+
+ private final boolean dumpObjectStorage = ParseUtils.envAsBoolean("DUMP_OBJECT_STORAGE", false);
+
+ protected abstract Path magicPartPath();
+
+ protected abstract Path magicPendingSetPath();
+
+ protected abstract void assertSuccessMarker() throws IOException;
+
+ protected abstract void assertSummaryReport(Path reportDir) throws IOException;
+
+ protected abstract void assertNoTaskAttemptPath() throws IOException;
+
+ protected void assertMagicPathExist(Path output) throws IOException {
+ Path magicPath = CommitUtils.magicPath(output);
+ assertTrue(fs.exists(magicPath), String.format("Magic path: %s should exist", magicPath));
+ }
+
+ protected void assertMagicPathNotExist(Path output) throws IOException {
+ Path magicPath = CommitUtils.magicPath(output);
+ assertFalse(fs.exists(magicPath), String.format("Magic path: %s should not exist", magicPath));
+ }
+
+ protected abstract boolean skipTests();
+
+ public Path magicPendingPath() {
+ Path magicPart = magicPartPath();
+ return new Path(magicPart.getParent(), magicPart.getName() + ".pending");
+ }
+
+ public Path magicJobPath() {
+ return CommitUtils.magicPath(outputPath);
+ }
+
+ public String magicPartKey() {
+ return ObjectUtils.pathToKey(magicPartPath());
+ }
+
+ public String destPartKey() {
+ return MagicOutputStream.toDestKey(magicPartPath());
+ }
+
+ public Job job() {
+ return job;
+ }
+
+ public String jobId() {
+ return jobId;
+ }
+
+ public FileSystem fs() {
+ return fs;
+ }
+
+ public Path outputPath() {
+ return outputPath;
+ }
+
+ public ObjectStorage storage() {
+ return storage;
+ }
+
+ public void setJob(Job value) {
+ this.job = value;
+ }
+
+ public void setJobId(String value) {
+ this.jobId = value;
+ }
+
+ public void setFs(FileSystem value) {
+ this.fs = value;
+ }
+
+ public void setOutputPath(Path value) {
+ this.outputPath = value;
+ }
+
+ public void setObjectStorage(ObjectStorage value) {
+ this.storage = value;
+ }
+
+ public void assertHasMagicKeys() {
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(magicJobPath(), true), "");
+ assertTrue(
+ Iterables.any(objects, o -> o.key().contains(CommitUtils.MAGIC) && o.key().contains(jobId)),
+ "Should have some __magic object keys");
+ }
+
+ public void assertHasBaseKeys() {
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(magicJobPath(), true), "");
+ assertTrue(
+ Iterables.any(objects, o -> o.key().contains(CommitUtils.BASE) && o.key().contains(jobId)),
+ "Should have some __base object keys");
+ }
+
+ public void assertNoMagicPendingFile() {
+ String magicPendingKey = String.format("%s.pending", magicPartKey());
+ assertNull(storage.head(magicPendingKey), "Magic pending key should exist");
+ }
+
+ public void assertHasMagicPendingFile() {
+ String magicPendingKey = String.format("%s.pending", magicPartKey());
+ assertNotNull(storage.head(magicPendingKey), "Magic pending key should exist");
+ }
+
+ public void assertNoMagicMultipartUpload() {
+ Iterable<MultipartUpload> uploads =
+ storage.listUploads(ObjectUtils.pathToKey(magicJobPath(), true));
+ boolean anyMagicUploads = Iterables.any(uploads, u -> u.key().contains(CommitUtils.MAGIC));
+ assertFalse(anyMagicUploads, "Should have no magic multipart uploads");
+ }
+
+ public void assertNoMagicObjectKeys() {
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(magicJobPath(), true), "");
+ boolean anyMagicUploads =
+ Iterables.any(objects, o -> o.key().contains(CommitUtils.MAGIC) && o.key().contains(jobId));
+ assertFalse(anyMagicUploads, "Should not have any magic keys");
+ }
+
+ public void assertHasPendingSet() {
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(magicJobPath(), true), "");
+ boolean anyPendingSet = Iterables.any(objects,
+ o -> o.key().contains(CommitUtils.PENDINGSET_SUFFIX) && o.key().contains(jobId));
+ assertTrue(anyPendingSet, "Should have the expected .pendingset file");
+ }
+
+ public void assertPendingSetAtRightLocation() {
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(magicJobPath(), true), "");
+ Path magicJobAttemptPath =
+ CommitUtils.magicJobAttemptPath(job().getJobID().toString(), DEFAULT_APP_ATTEMPT_ID,
+ outputPath);
+ String inQualifiedPath = magicJobAttemptPath.toUri().getPath().substring(1);
+ Iterable<ObjectInfo> filtered = Iterables.filter(objects,
+ o -> o.key().contains(CommitUtils.PENDINGSET_SUFFIX) && o.key().contains(jobId));
+ boolean pendingSetAtRightLocation = Iterables.any(filtered,
+ o -> o.key().startsWith(inQualifiedPath) && o.key().contains(jobId));
+ assertTrue(pendingSetAtRightLocation,
+ "The .pendingset file should locate at the job's magic output path.");
+ }
+
+ public void assertMultipartUpload(int expectedUploads) {
+ // Note: should be care in concurrent case: they need to check the same output path.
+ Iterable<MultipartUpload> uploads =
+ storage.listUploads(ObjectUtils.pathToKey(outputPath, true));
+ long actualUploads = StreamSupport.stream(uploads.spliterator(), false).count();
+ assertEquals(expectedUploads, actualUploads);
+ }
+
+ public void assertPartFiles(int num) throws IOException {
+ FileStatus[] files = fs.listStatus(outputPath,
+ f -> !MagicOutputStream.isMagic(new Path(f.toUri())) && f.toUri().toString()
+ .contains("part-"));
+ assertEquals(num, files.length);
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(outputPath, true), "");
+ List<ObjectInfo> infos = Arrays.stream(Iterables.toArray(objects, ObjectInfo.class))
+ .filter(o -> o.key().contains("part-")).collect(Collectors.toList());
+ assertEquals(num, infos.size(),
+ String.format("Part files number should be %d, but got %d", num, infos.size()));
+ }
+
+ public void assertNoPartFiles() throws IOException {
+ FileStatus[] files = fs.listStatus(outputPath,
+ f -> !MagicOutputStream.isMagic(new Path(f.toUri())) && f.toUri().toString()
+ .contains("part-"));
+ assertEquals(0, files.length);
+ Iterable<ObjectInfo> objects = storage.listAll(ObjectUtils.pathToKey(outputPath, true), "");
+ boolean anyPartFile = Iterables.any(objects, o -> o.key().contains("part-"));
+ assertFalse(anyPartFile, "Should have no part files");
+ }
+
+ public void dumpObjectStorage() {
+ if (dumpObjectStorage) {
+ LOG.info("===> Dump object storage - Start <===");
+ dumpObjectKeys();
+ dumpMultipartUploads();
+ LOG.info("===> Dump object storage - End <===");
+ }
+ }
+
+ public void dumpObjectKeys() {
+ String prefix = ObjectUtils.pathToKey(magicJobPath());
+ LOG.info("Dump object keys with prefix {}", prefix);
+ storage.listAll("", "").forEach(o -> LOG.info("Dump object keys - {}", o));
+ }
+
+ public void dumpMultipartUploads() {
+ String prefix = ObjectUtils.pathToKey(magicJobPath());
+ LOG.info("Dump multi part uploads with prefix {}", prefix);
+ storage.listUploads("")
+ .forEach(u -> LOG.info("Dump multipart uploads - {}", u));
+ }
+
+ public void verifyPartContent() throws IOException {
+ String partKey = destPartKey();
+ LOG.info("Part key to verify is: {}", partKey);
+ try (InputStream in = storage.get(partKey).stream()) {
+ byte[] data = IOUtils.toByteArray(in);
+ String expected = String.format("%s\t%s\n%s\t%s\n", KEY_1, VAL_1, KEY_2, VAL_2);
+ assertEquals(expected, new String(data, StandardCharsets.UTF_8));
+ }
+ }
+
+ public void assertSuccessMarkerNotExist() throws IOException {
+ Path succPath = CommitUtils.successMarker(outputPath);
+ assertFalse(fs.exists(succPath), String.format("%s should not exists", succPath));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/CommitterTestBase.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/CommitterTestBase.java
new file mode 100644
index 00000000000..93f6ca434fb
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/CommitterTestBase.java
@@ -0,0 +1,433 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public abstract class CommitterTestBase {
+ private Configuration conf;
+ private FileSystem fs;
+ private Path outputPath;
+ private TaskAttemptID job1Task0Attempt0;
+ private TaskAttemptID job2Task1Attempt0;
+ private Path reportDir;
+
+ @BeforeEach
+ public void setup() throws IOException {
+ conf = newConf();
+ fs = FileSystem.get(conf);
+ String uuid = UUIDUtils.random();
+ outputPath = fs.makeQualified(new Path("/test/" + uuid));
+ job1Task0Attempt0 = JobSuite.createTaskAttemptId(randomTrimmedJobId(), 0, 0);
+ job2Task1Attempt0 = JobSuite.createTaskAttemptId(randomTrimmedJobId(), 1, 0);
+
+ reportDir = fs.makeQualified(new Path("/report/" + uuid));
+ fs.mkdirs(reportDir);
+ conf.set(Committer.COMMITTER_SUMMARY_REPORT_DIR, reportDir.toUri().toString());
+ }
+
+ protected abstract Configuration newConf();
+
+ @AfterEach
+ public void teardown() {
+ CommonUtils.runQuietly(() -> fs.delete(outputPath, true));
+ IOUtils.closeStream(fs);
+ }
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ List<String> committerThreads = Thread.getAllStackTraces().keySet()
+ .stream()
+ .map(Thread::getName)
+ .filter(n -> n.startsWith(Committer.THREADS_PREFIX))
+ .collect(Collectors.toList());
+ assertTrue(committerThreads.isEmpty(), "Outstanding committer threads");
+ }
+
+ private static String randomTrimmedJobId() {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+ return String.format("%s%04d_%04d", formatter.format(new Date()),
+ (long) (Math.random() * 1000),
+ (long) (Math.random() * 1000));
+ }
+
+ private static String randomFormedJobId() {
+ return String.format("job_%s", randomTrimmedJobId());
+ }
+
+ @Test
+ public void testSetupJob() throws IOException {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ // Setup job.
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+ }
+
+ @Test
+ public void testSetupJobWithOrphanPaths() throws IOException, InterruptedException {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ // Orphan success marker.
+ Path successPath = CommitUtils.successMarker(outputPath);
+ CommitUtils.save(fs, successPath, new byte[]{});
+ assertTrue(fs.exists(successPath));
+
+ // Orphan job path.
+ Path jobPath = CommitUtils.magicJobPath(suite.committer().jobId(), outputPath);
+ fs.mkdirs(jobPath);
+ assertTrue(fs.exists(jobPath), "The job path should be existing");
+ Path subPath = new Path(jobPath, "tmp.pending");
+ CommitUtils.save(fs, subPath, new byte[]{});
+ assertTrue(fs.exists(subPath), "The sub path under job path should be existing.");
+ FileStatus jobPathStatus = fs.getFileStatus(jobPath);
+
+ Thread.sleep(1000L);
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+
+ assertFalse(fs.exists(successPath), "Should have deleted the success path");
+ assertTrue(fs.exists(jobPath), "Should have re-created the job path");
+ assertFalse(fs.exists(subPath), "Should have deleted the sub path under the job path");
+ }
+
+ @Test
+ public void testSetupTask() throws IOException {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ // Remaining attempt task path.
+ Path taskAttemptBasePath =
+ CommitUtils.magicTaskAttemptBasePath(suite.taskAttemptContext(), outputPath);
+ Path subTaskAttemptPath = new Path(taskAttemptBasePath, "tmp.pending");
+ CommitUtils.save(fs, subTaskAttemptPath, new byte[]{});
+ assertTrue(fs.exists(taskAttemptBasePath));
+ assertTrue(fs.exists(subTaskAttemptPath));
+
+ // Setup job.
+ suite.setupJob();
+ suite.assertHasMagicKeys();
+ // It will clear all the job path once we've set up the job.
+ assertFalse(fs.exists(taskAttemptBasePath));
+ assertFalse(fs.exists(subTaskAttemptPath));
+
+ // Left some the task paths.
+ CommitUtils.save(fs, subTaskAttemptPath, new byte[]{});
+ assertTrue(fs.exists(taskAttemptBasePath));
+ assertTrue(fs.exists(subTaskAttemptPath));
+
+ // Setup task.
+ suite.setupTask();
+ assertFalse(fs.exists(subTaskAttemptPath));
+ }
+
+ @Test
+ public void testCommitTask() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+
+ // Setup job
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+
+ // Setup task
+ suite.setupTask();
+
+ // Write records.
+ suite.assertNoMagicPendingFile();
+ suite.assertMultipartUpload(0);
+ suite.writeOutput();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(1);
+ // Assert the pending file content.
+ Path pendingPath = suite.magicPendingPath();
+ byte[] pendingData = CommitUtils.load(suite.fs(), pendingPath);
+ Pending pending = Pending.deserialize(pendingData);
+ assertEquals(suite.destPartKey(), pending.destKey());
+ assertEquals(20, pending.length());
+ assertEquals(1, pending.parts().size());
+
+ // Commit the task.
+ suite.commitTask();
+
+ // Verify the pending set file.
+ suite.assertHasPendingSet();
+ // Assert the pending set file content.
+ Path pendingSetPath = suite.magicPendingSetPath();
+ byte[] pendingSetData = CommitUtils.load(suite.fs(), pendingSetPath);
+ PendingSet pendingSet = PendingSet.deserialize(pendingSetData);
+ assertEquals(suite.job().getJobID().toString(), pendingSet.jobId());
+ assertEquals(1, pendingSet.commits().size());
+ assertEquals(pending, pendingSet.commits().get(0));
+ assertEquals(pendingSet.extraData(), ImmutableMap.of(CommitUtils.TASK_ATTEMPT_ID,
+ suite.taskAttemptContext().getTaskAttemptID().toString()));
+
+ // Complete the multipart upload and verify the results.
+ ObjectStorage storage = suite.storage();
+ storage.completeUpload(pending.destKey(), pending.uploadId(), pending.parts());
+ suite.verifyPartContent();
+ }
+
+ @Test
+ public void testAbortTask() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+
+ // Pre-check before the output write.
+ suite.assertNoMagicPendingFile();
+ suite.assertMultipartUpload(0);
+
+ // Execute the output write.
+ suite.writeOutput();
+
+ // Post-check after the output write.
+ suite.assertHasMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(1);
+ // Assert the pending file content.
+ Path pendingPath = suite.magicPendingPath();
+ byte[] pendingData = CommitUtils.load(suite.fs(), pendingPath);
+ Pending pending = Pending.deserialize(pendingData);
+ assertEquals(suite.destPartKey(), pending.destKey());
+ assertEquals(20, pending.length());
+ assertEquals(1, pending.parts().size());
+
+ // Abort the task.
+ suite.abortTask();
+
+ // Verify the state after aborting task.
+ suite.assertNoMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(0);
+ suite.assertNoTaskAttemptPath();
+ }
+
+ @Test
+ public void testCommitJob() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job.
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.assertSummaryReport(reportDir);
+ suite.verifyPartContent();
+ }
+
+
+ @Test
+ public void testCommitJobFailed() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job.
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ }
+
+ @Test
+ public void testCommitJobSuccessMarkerFailed() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ CommitUtils.injectError("marker");
+ // Commit the job.
+ suite.assertNoPartFiles();
+ assertThrows(IOException.class, suite::commitJob, "Expect commit job error.");
+ CommitUtils.removeError("marker");
+
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarkerNotExist();
+ assertEquals(0, suite.fs().listStatus(suite.outputPath()).length);
+ }
+
+ @Test
+ public void testTaskCommitAfterJobCommit() throws Exception {
+ JobSuite suite = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.verifyPartContent();
+
+ // Commit the task again.
+ assertThrows(FileNotFoundException.class, suite::commitTask);
+ }
+
+ @Test
+ public void testTaskCommitWithConsistentJobId() throws Exception {
+ Configuration config = newConf();
+ String consistentJobId = randomFormedJobId();
+ config.set(CommitUtils.SPARK_WRITE_UUID, consistentJobId);
+ JobSuite suite = JobSuite.create(config, job1Task0Attempt0, outputPath);
+ assumeFalse(suite.skipTests());
+
+ // By now, we have two "jobId"s, one is spark uuid, and the other is the jobId in taskAttempt.
+ // The job committer will adopt the former.
+ suite.setupJob();
+
+ // Next, we clear spark uuid, and set the jobId of taskAttempt to another value. In this case,
+ // the committer will take the jobId of taskAttempt as the final jobId, which is not consistent
+ // with the one that committer holds.
+ config.unset(CommitUtils.SPARK_WRITE_UUID);
+ String anotherJobId = randomTrimmedJobId();
+ TaskAttemptID taskAttemptId1 =
+ JobSuite.createTaskAttemptId(anotherJobId, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+ final TaskAttemptContext attemptContext1 =
+ JobSuite.createTaskAttemptContext(config, taskAttemptId1, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+
+ assertThrows(IllegalArgumentException.class, () -> suite.setupTask(attemptContext1),
+ "JobId set in the context");
+
+ // Even though we use another taskAttempt, as long as we ensure the spark uuid is consistent,
+ // the jobId in committer is consistent.
+ config.set(CommitUtils.SPARK_WRITE_UUID, consistentJobId);
+ config.set(FileOutputFormat.OUTDIR, outputPath.toString());
+ anotherJobId = randomTrimmedJobId();
+ TaskAttemptID taskAttemptId2 =
+ JobSuite.createTaskAttemptId(anotherJobId, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+ TaskAttemptContext attemptContext2 =
+ JobSuite.createTaskAttemptContext(config, taskAttemptId2, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+
+ suite.setupTask(attemptContext2);
+ // Write output must use the same task context with setup task.
+ suite.writeOutput(attemptContext2);
+ // Commit task must use the same task context with setup task.
+ suite.commitTask(attemptContext2);
+ suite.assertPendingSetAtRightLocation();
+
+ // Commit the job
+ suite.assertNoPartFiles();
+ suite.commitJob();
+
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.verifyPartContent();
+ }
+
+ @Test
+ public void testConcurrentJobs() throws Exception {
+ JobSuite suite1 = JobSuite.create(conf, job1Task0Attempt0, outputPath);
+ JobSuite suite2 = JobSuite.create(conf, job2Task1Attempt0, outputPath);
+ assumeFalse(suite1.skipTests());
+ assumeFalse(suite2.skipTests());
+ suite1.setupJob();
+ suite2.setupJob();
+ suite1.setupTask();
+ suite2.setupTask();
+ suite1.writeOutput();
+ suite2.writeOutput();
+ suite1.commitTask();
+ suite2.commitTask();
+
+ // Job2 commit the job.
+ suite2.assertNoPartFiles();
+ suite2.commitJob();
+ suite2.assertPartFiles(1);
+
+ suite2.assertNoMagicMultipartUpload();
+ suite2.assertNoMagicObjectKeys();
+ suite2.assertSuccessMarker();
+ suite2.assertSummaryReport(reportDir);
+ suite2.verifyPartContent();
+ suite2.assertMagicPathExist(outputPath);
+
+ // Job1 commit the job.
+ suite1.commitJob();
+ suite2.assertPartFiles(2);
+
+ // Verify the output.
+ suite1.assertNoMagicMultipartUpload();
+ suite1.assertNoMagicObjectKeys();
+ suite1.assertSuccessMarker();
+ suite1.assertSummaryReport(reportDir);
+ suite1.verifyPartContent();
+ suite1.assertMagicPathNotExist(outputPath);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/JobSuite.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/JobSuite.java
new file mode 100644
index 00000000000..95821796461
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/JobSuite.java
@@ -0,0 +1,228 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.net.NetUtils;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public final class JobSuite extends BaseJobSuite {
+ private static final CommitterFactory FACTORY = new CommitterFactory();
+ private final JobContext jobContext;
+ private final TaskAttemptContext taskAttemptContext;
+ private final Committer committer;
+
+ private JobSuite(FileSystem fs, Configuration conf, TaskAttemptID taskAttemptId, int appAttemptId,
+ Path outputPath) throws IOException {
+ setFs(fs);
+ // Initialize the job instance.
+ setJob(Job.getInstance(conf));
+ job().setJobID(JobID.forName(CommitUtils.buildJobId(conf, taskAttemptId.getJobID())));
+ this.jobContext = createJobContext(job().getConfiguration(), taskAttemptId);
+ setJobId(CommitUtils.buildJobId(jobContext));
+ this.taskAttemptContext =
+ createTaskAttemptContext(job().getConfiguration(), taskAttemptId, appAttemptId);
+
+ // Set job output directory.
+ FileOutputFormat.setOutputPath(job(), outputPath);
+ setOutputPath(outputPath);
+ setObjectStorage(ObjectStorageFactory.create(outputPath.toUri().getScheme(),
+ outputPath.toUri().getAuthority(), conf));
+
+ // Initialize committer.
+ this.committer = (Committer) FACTORY.createOutputCommitter(outputPath, taskAttemptContext);
+ }
+
+ public static JobSuite create(Configuration conf, TaskAttemptID taskAttemptId, Path outDir)
+ throws IOException {
+ FileSystem fs = outDir.getFileSystem(conf);
+ return new JobSuite(fs, conf, taskAttemptId, DEFAULT_APP_ATTEMPT_ID, outDir);
+ }
+
+ public static TaskAttemptID createTaskAttemptId(String trimmedJobId, int attemptId) {
+ String attempt = String.format("attempt_%s_m_000000_%d", trimmedJobId, attemptId);
+ return TaskAttemptID.forName(attempt);
+ }
+
+ public static TaskAttemptID createTaskAttemptId(String trimmedJobId, int taskId, int attemptId) {
+ String[] parts = trimmedJobId.split("_");
+ return new TaskAttemptID(parts[0], Integer.parseInt(parts[1]), TaskType.MAP, taskId, attemptId);
+ }
+
+ public static JobContext createJobContext(Configuration jobConf, TaskAttemptID taskAttemptId) {
+ return new JobContextImpl(jobConf, taskAttemptId.getJobID());
+ }
+
+ public static TaskAttemptContext createTaskAttemptContext(
+ Configuration jobConf, TaskAttemptID taskAttemptId, int appAttemptId) throws IOException {
+ // Set the key values for job configuration.
+ jobConf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptId.toString());
+ jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
+ jobConf.set(PathOutputCommitterFactory.COMMITTER_FACTORY_CLASS,
+ CommitterFactory.class.getName());
+ return new TaskAttemptContextImpl(jobConf, taskAttemptId);
+ }
+
+ public void setupJob() throws IOException {
+ committer.setupJob(jobContext);
+ }
+
+ public void setupTask() throws IOException {
+ committer.setupTask(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void setupTask(TaskAttemptContext taskAttemptCxt) throws IOException {
+ committer.setupTask(taskAttemptCxt);
+ }
+
+ public void writeOutput() throws Exception {
+ writeOutput(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void writeOutput(TaskAttemptContext taskAttemptCxt) throws Exception {
+ RecordWriter<Object, Object> writer = new TextOutputFormat<>().getRecordWriter(taskAttemptCxt);
+ NullWritable nullKey = NullWritable.get();
+ NullWritable nullVal = NullWritable.get();
+ Object[] keys = new Object[]{KEY_1, nullKey, null, nullKey, null, KEY_2};
+ Object[] vals = new Object[]{VAL_1, nullVal, null, null, nullVal, VAL_2};
+ try {
+ assertEquals(keys.length, vals.length);
+ for (int i = 0; i < keys.length; i++) {
+ writer.write(keys[i], vals[i]);
+ }
+ } finally {
+ writer.close(taskAttemptCxt);
+ }
+ }
+
+ public boolean needsTaskCommit() {
+ return committer.needsTaskCommit(taskAttemptContext);
+ }
+
+ public void commitTask() throws IOException {
+ committer.commitTask(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void commitTask(TaskAttemptContext taskAttemptCxt) throws IOException {
+ committer.commitTask(taskAttemptCxt);
+ }
+
+ public void abortTask() throws IOException {
+ committer.abortTask(taskAttemptContext);
+ }
+
+ public void commitJob() throws IOException {
+ committer.commitJob(jobContext);
+ }
+
+ @Override
+ public Path magicPartPath() {
+ return new Path(committer.getWorkPath(),
+ FileOutputFormat.getUniqueFile(taskAttemptContext, "part", ""));
+ }
+
+ @Override
+ public Path magicPendingSetPath() {
+ return CommitUtils.magicTaskPendingSetPath(taskAttemptContext, outputPath());
+ }
+
+ public TaskAttemptContext taskAttemptContext() {
+ return taskAttemptContext;
+ }
+
+ public Committer committer() {
+ return committer;
+ }
+
+ @Override
+ public void assertNoTaskAttemptPath() throws IOException {
+ Path path = CommitUtils.magicTaskAttemptBasePath(taskAttemptContext, outputPath());
+ assertFalse(fs().exists(path), "Task attempt path should be not existing");
+ String pathToKey = ObjectUtils.pathToKey(path);
+ assertNull(storage().head(pathToKey), "Should have no task attempt path key");
+ }
+
+ @Override
+ protected boolean skipTests() {
+ return storage().bucket().isDirectory();
+ }
+
+ @Override
+ public void assertSuccessMarker() throws IOException {
+ Path succPath = CommitUtils.successMarker(outputPath());
+ assertTrue(fs().exists(succPath), String.format("%s should be exists", succPath));
+ SuccessData successData = SuccessData.deserialize(CommitUtils.load(fs(), succPath));
+ assertEquals(SuccessData.class.getName(), successData.name());
+ assertTrue(successData.success());
+ assertEquals(NetUtils.getHostname(), successData.hostname());
+ assertEquals(CommitUtils.COMMITTER_NAME, successData.committer());
+ assertEquals(
+ String.format("Task committer %s", taskAttemptContext.getTaskAttemptID()),
+ successData.description());
+ assertEquals(job().getJobID().toString(), successData.jobId());
+ assertEquals(1, successData.filenames().size());
+ assertEquals(destPartKey(), successData.filenames().get(0));
+ }
+
+ @Override
+ public void assertSummaryReport(Path reportDir) throws IOException {
+ Path reportPath = CommitUtils.summaryReport(reportDir, job().getJobID().toString());
+ assertTrue(fs().exists(reportPath), String.format("%s should be exists", reportPath));
+ SuccessData reportData = SuccessData.deserialize(CommitUtils.load(fs(), reportPath));
+ assertEquals(SuccessData.class.getName(), reportData.name());
+ assertTrue(reportData.success());
+ assertEquals(NetUtils.getHostname(), reportData.hostname());
+ assertEquals(CommitUtils.COMMITTER_NAME, reportData.committer());
+ assertEquals(
+ String.format("Task committer %s", taskAttemptContext.getTaskAttemptID()),
+ reportData.description());
+ assertEquals(job().getJobID().toString(), reportData.jobId());
+ assertEquals(1, reportData.filenames().size());
+ assertEquals(destPartKey(), reportData.filenames().get(0));
+ assertEquals("clean", reportData.diagnostics().get("stage"));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/MRJobTestBase.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/MRJobTestBase.java
new file mode 100644
index 00000000000..ae887051ada
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/MRJobTestBase.java
@@ -0,0 +1,241 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.terasort.TeraGen;
+import org.apache.hadoop.examples.terasort.TeraSort;
+import org.apache.hadoop.examples.terasort.TeraSortConfigKeys;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.WordCount;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public abstract class MRJobTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(MRJobTestBase.class);
+
+ private static Configuration conf = new Configuration();
+ private static MiniMRYarnCluster yarnCluster;
+
+ private static FileSystem fs;
+
+ private static Path testDataPath;
+
+ public static void setConf(Configuration newConf) {
+ conf = newConf;
+ }
+
+ @BeforeAll
+ public static void beforeClass() throws IOException {
+ assumeTrue(TestEnv.checkTestEnabled());
+
+ conf.setBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, false);
+ conf.setBoolean(YarnConfiguration.NM_DISK_HEALTH_CHECK_ENABLE, false);
+ conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
+
+ conf.set("mapreduce.outputcommitter.factory.scheme.tos",
+ CommitterFactory.class.getName()); // 3x newApiCommitter=true.
+ conf.set("mapred.output.committer.class",
+ Committer.class.getName()); // 2x and 3x newApiCommitter=false.
+ conf.set("mapreduce.outputcommitter.class",
+ org.apache.hadoop.fs.tosfs.commit.Committer.class.getName()); // 2x newApiCommitter=true.
+
+ // Start the yarn cluster.
+ yarnCluster = new MiniMRYarnCluster("yarn-" + System.currentTimeMillis(), 2);
+ LOG.info("Default filesystem: {}", conf.get("fs.defaultFS"));
+ LOG.info("Default filesystem implementation: {}", conf.get("fs.AbstractFileSystem.tos.impl"));
+
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ fs = FileSystem.get(conf);
+ testDataPath = new Path("/mr-test-" + UUIDUtils.random())
+ .makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ }
+
+ @AfterAll
+ public static void afterClass() throws IOException {
+ if (!TestEnv.checkTestEnabled()) {
+ return;
+ }
+
+ fs.delete(testDataPath, true);
+ if (yarnCluster != null) {
+ yarnCluster.stop();
+ }
+ }
+
+ @AfterEach
+ public void after() throws IOException {
+ }
+
+ @Test
+ public void testTeraGen() throws Exception {
+ Path teraGenPath =
+ new Path(testDataPath, "teraGen").makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path output = new Path(teraGenPath, "output");
+ JobConf jobConf = new JobConf(yarnCluster.getConfig());
+ jobConf.addResource(conf);
+ jobConf.setInt(TeraSortConfigKeys.SAMPLE_SIZE.key(), 1000);
+ jobConf.setInt(TeraSortConfigKeys.NUM_PARTITIONS.key(), 10);
+ jobConf.setBoolean(TeraSortConfigKeys.USE_SIMPLE_PARTITIONER.key(), false);
+
+ String[] args = new String[]{Integer.toString(1000), output.toString()};
+ int result = ToolRunner.run(jobConf, new TeraGen(), args);
+ assertEquals(0, result, String.format("teragen %s", StringUtils.join(" ", args)));
+
+ // Verify the success data.
+ ObjectStorage storage = ObjectStorageFactory.create(
+ output.toUri().getScheme(), output.toUri().getAuthority(), conf);
+ int byteSizes = 0;
+
+ Path success = new Path(output, CommitUtils.SUCCESS);
+ byte[] serializedData = CommitUtils.load(fs, success);
+ SuccessData successData = SuccessData.deserialize(serializedData);
+ assertTrue(successData.success(), "Should execute successfully");
+ // Assert the destination paths.
+ assertEquals(2, successData.filenames().size());
+ successData.filenames().sort(String::compareTo);
+ assertEquals(ObjectUtils.pathToKey(new Path(output, "part-m-00000")),
+ successData.filenames().get(0));
+ assertEquals(ObjectUtils.pathToKey(new Path(output, "part-m-00001")),
+ successData.filenames().get(1));
+
+ for (String partFileKey : successData.filenames()) {
+ ObjectInfo objectInfo = storage.head(partFileKey);
+ assertNotNull(objectInfo, "Output file should be existing");
+ byteSizes += objectInfo.size();
+ }
+
+ assertEquals(byteSizes, 100 /* Each row 100 bytes */ * 1000 /* total 1000 rows */);
+ }
+
+ @Test
+ public void testTeraSort() throws Exception {
+ Path teraGenPath =
+ new Path(testDataPath, "teraGen").makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path inputPath = new Path(teraGenPath, "output");
+ Path outputPath = new Path(teraGenPath, "sortOutput");
+ JobConf jobConf = new JobConf(yarnCluster.getConfig());
+ jobConf.addResource(conf);
+ jobConf.setInt(TeraSortConfigKeys.SAMPLE_SIZE.key(), 1000);
+ jobConf.setInt(TeraSortConfigKeys.NUM_PARTITIONS.key(), 10);
+ jobConf.setBoolean(TeraSortConfigKeys.USE_SIMPLE_PARTITIONER.key(), false);
+ String[] args = new String[]{inputPath.toString(), outputPath.toString()};
+ int result = ToolRunner.run(jobConf, new TeraSort(), args);
+ assertEquals(0, result, String.format("terasort %s", StringUtils.join(" ", args)));
+
+ // Verify the success data.
+ ObjectStorage storage = ObjectStorageFactory
+ .create(outputPath.toUri().getScheme(), outputPath.toUri().getAuthority(), conf);
+ int byteSizes = 0;
+
+ Path success = new Path(outputPath, CommitUtils.SUCCESS);
+ byte[] serializedData = CommitUtils.load(fs, success);
+ SuccessData successData = SuccessData.deserialize(serializedData);
+ assertTrue(successData.success(), "Should execute successfully");
+ // Assert the destination paths.
+ assertEquals(1, successData.filenames().size());
+ successData.filenames().sort(String::compareTo);
+ assertEquals(ObjectUtils.pathToKey(new Path(outputPath, "part-r-00000")),
+ successData.filenames().get(0));
+
+ for (String partFileKey : successData.filenames()) {
+ ObjectInfo objectInfo = storage.head(partFileKey);
+ assertNotNull(objectInfo, "Output file should be existing");
+ byteSizes += objectInfo.size();
+ }
+
+ assertEquals(byteSizes, 100 /* Each row 100 bytes */ * 1000 /* total 1000 rows */);
+ }
+
+ @Disabled
+ @Test
+ public void testWordCount() throws Exception {
+ Path wordCountPath =
+ new Path(testDataPath, "wc").makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path output = new Path(wordCountPath, "output");
+ Path input = new Path(wordCountPath, "input");
+ JobConf jobConf = new JobConf(yarnCluster.getConfig());
+ jobConf.addResource(conf);
+
+ if (!fs.mkdirs(input)) {
+ throw new IOException("Mkdirs failed to create " + input.toString());
+ }
+
+ DataOutputStream file = fs.create(new Path(input, "part-0"));
+ file.writeBytes("a a b c");
+ file.close();
+
+ String[] args = new String[]{input.toString(), output.toString()};
+ int result = ToolRunner.run(jobConf, new WordCount(), args);
+ assertEquals(0, result, String.format("WordCount %s", StringUtils.join(" ", args)));
+
+ // Verify the success path.
+ assertTrue(fs.exists(new Path(output, CommitUtils.SUCCESS)));
+ assertTrue(fs.exists(new Path(output, "part-00000")));
+
+ Path success = new Path(output, CommitUtils.SUCCESS);
+ assertTrue(CommitUtils.load(fs, success).length != 0, "Success file must be not empty");
+
+ byte[] serializedData = CommitUtils.load(fs, new Path(output, "part-00000"));
+ String outputAsStr = new String(serializedData);
+ Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
+ assertEquals(2, (int) resAsMap.get("a"));
+ assertEquals(1, (int) resAsMap.get("b"));
+ assertEquals(1, (int) resAsMap.get("c"));
+ }
+
+ private Map<String, Integer> getResultAsMap(String outputAsStr) {
+ Map<String, Integer> result = new HashMap<>();
+ for (String line : outputAsStr.split("\n")) {
+ String[] tokens = line.split("\t");
+ assertTrue(tokens.length > 1,
+ String.format("Not enough tokens in in string %s from output %s", line, outputAsStr));
+ result.put(tokens[0], Integer.parseInt(tokens[1]));
+ }
+ return result;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestCommitter.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestCommitter.java
new file mode 100644
index 00000000000..017b98e9564
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestCommitter.java
@@ -0,0 +1,29 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+
+public class TestCommitter extends CommitterTestBase {
+ @Override
+ protected Configuration newConf() {
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", String.format("tos://%s", TestUtility.bucket()));
+ return conf;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMRJob.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMRJob.java
new file mode 100644
index 00000000000..a27cb71206e
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMRJob.java
@@ -0,0 +1,49 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.BeforeAll;
+
+import java.io.IOException;
+
+public class TestMRJob extends MRJobTestBase {
+
+ @BeforeAll
+ public static void beforeClass() throws IOException {
+ // Create the new configuration and set it to the IT Case.
+ Configuration newConf = new Configuration();
+ newConf.set("fs.defaultFS", String.format("tos://%s", TestUtility.bucket()));
+ // Application in yarn cluster cannot read the environment variables from user bash, so here we
+ // set it into the config manually.
+ newConf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key("tos"),
+ ParseUtils.envAsString(TOS.ENV_TOS_ENDPOINT, false));
+ newConf.set(TosKeys.FS_TOS_ACCESS_KEY_ID,
+ ParseUtils.envAsString(TOS.ENV_TOS_ACCESS_KEY_ID, false));
+ newConf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY,
+ ParseUtils.envAsString(TOS.ENV_TOS_SECRET_ACCESS_KEY, false));
+
+ MRJobTestBase.setConf(newConf);
+ // Continue to prepare the IT Case environments.
+ MRJobTestBase.beforeClass();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMagicOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMagicOutputStream.java
new file mode 100644
index 00000000000..0844b130beb
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestMagicOutputStream.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.object.MultipartUpload;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageTestBase;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.object.staging.StagingPart;
+import org.apache.hadoop.fs.tosfs.object.staging.State;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class TestMagicOutputStream extends ObjectStorageTestBase {
+
+ private static ExecutorService threadPool;
+
+ @BeforeAll
+ public static void beforeClass() {
+ threadPool = ThreadPools.newWorkerPool("TestMagicOutputStream-pool");
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ if (!threadPool.isShutdown()) {
+ threadPool.shutdown();
+ }
+ }
+
+ private static Path path(String p) {
+ return new Path(p);
+ }
+
+ private static Path path(Path parent, String child) {
+ return new Path(parent, child);
+ }
+
+ @Test
+ public void testCreateDestKey() {
+ Object[][] testCases = new Object[][]{
+ new Object[]{path("tos://bucket/__magic/a.txt"), "a.txt"},
+ new Object[] {path("tos://bucket/output/__magic/job-1/tasks/tasks-attempt-0/a.txt"),
+ "output/a.txt"},
+ new Object[]{path("tos://bucket/__magic/job0/task0/__base/a.txt"), "a.txt"},
+ new Object[] {path("tos://bucket/output/__magic/job0/task0/__base/part/part-m-1000"),
+ "output/part/part-m-1000"},
+ new Object[]{path("tos://bucket/a/b/c/__magic/__base/d/e/f"), "a/b/c/d/e/f"},
+ new Object[]{path("tos://bucket/a/b/c/__magic/d/e/f"), "a/b/c/f"},
+ };
+
+ for (Object[] input : testCases) {
+ String actualDestKey = MagicOutputStream.toDestKey((Path) input[0]);
+ assertEquals(actualDestKey, input[1], "Unexpected destination key.");
+ }
+ }
+
+ @Test
+ public void testNonMagicPath() {
+ try (MagicOutputStream ignored = new TestingMagicOutputStream(path(testDir(), "non-magic"))) {
+ fail("Cannot create magic output stream for non-magic path");
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void testWriteZeroByte() throws IOException {
+ Path magic = path(path(testDir(), CommitUtils.MAGIC), "zero-byte.txt");
+ MagicOutputStream out = new TestingMagicOutputStream(magic);
+ // write zero-byte and close.
+ out.close();
+ assertStagingFiles(0, out.stagingParts());
+
+ // Read and validate the .pending contents
+ try (InputStream in = getStorage().get(out.pendingKey()).stream()) {
+ byte[] data = IOUtils.toByteArray(in);
+ Pending commit = Pending.deserialize(data);
+ assertEquals(getStorage().bucket().name(), commit.bucket());
+ assertEquals(out.destKey(), commit.destKey());
+ assertTrue(StringUtils.isNoneEmpty(commit.uploadId()));
+ assertTrue(commit.createdTimestamp() > 0);
+ assertEquals(1, commit.parts().size());
+ assertEquals(0, commit.length());
+ assertEquals(out.upload().uploadId(), commit.uploadId());
+ }
+ }
+
+ public void testWrite(int len) throws IOException {
+ Path magic = path(path(testDir(), CommitUtils.MAGIC), len + ".txt");
+ int uploadPartSize = 8 << 20;
+ int partNum = (len - 1) / (8 << 20) + 1;
+
+ MagicOutputStream out = new TestingMagicOutputStream(magic);
+ byte[] data = TestUtility.rand(len);
+ out.write(data);
+ out.close();
+
+ assertStagingFiles(partNum, out.stagingParts());
+ assertEquals(ObjectUtils.pathToKey(magic) + CommitUtils.PENDING_SUFFIX, out.pendingKey());
+
+ Pending commit;
+ try (InputStream in = getStorage().get(out.pendingKey()).stream()) {
+ byte[] serializedData = IOUtils.toByteArray(in);
+ commit = Pending.deserialize(serializedData);
+ assertEquals(getStorage().bucket().name(), commit.bucket());
+ assertEquals(out.destKey(), commit.destKey());
+ assertTrue(commit.createdTimestamp() > 0);
+ assertEquals(len, commit.length());
+ assertEquals(out.upload().uploadId(), commit.uploadId());
+ // Verify the upload part list.
+ assertEquals(partNum, commit.parts().size());
+ if (!commit.parts().isEmpty()) {
+ for (int i = 0; i < partNum - 1; i += 1) {
+ assertEquals(uploadPartSize, commit.parts().get(i).size());
+ }
+ Part lastPart = commit.parts().get(partNum - 1);
+ assertTrue(lastPart.size() > 0 && lastPart.size() <= uploadPartSize);
+ }
+ }
+
+ // List multipart uploads
+ int uploadsNum = 0;
+ for (MultipartUpload upload : getStorage().listUploads(out.destKey())) {
+ uploadsNum += 1;
+ assertEquals(out.upload(), upload);
+ }
+ assertEquals(1L, uploadsNum);
+
+ // The target object is still not visible for object storage.
+ assertNull(getStorage().head(out.destKey()));
+
+ // Complete the upload and validate the content.
+ getStorage().completeUpload(out.destKey(), out.upload().uploadId(), commit.parts());
+ try (InputStream in = getStorage().get(out.destKey()).stream()) {
+ assertArrayEquals(data, IOUtils.toByteArray(in));
+ }
+ }
+
+ @Test
+ public void testWrite1MB() throws IOException {
+ testWrite(1 << 20);
+ }
+
+ @Test
+ public void testWrite24MB() throws IOException {
+ testWrite(24 << 20);
+ }
+
+ @Test
+ public void testWrite100MB() throws IOException {
+ testWrite(100 << 20);
+ }
+
+ private static void assertStagingFiles(int expectedNum, List<StagingPart> stagings) {
+ assertEquals(expectedNum, stagings.size());
+ for (StagingPart staging : stagings) {
+ assertEquals(State.CLEANED, staging.state());
+ }
+ }
+
+ private class TestingMagicOutputStream extends MagicOutputStream {
+
+ TestingMagicOutputStream(Path magic) {
+ super(fs(), getStorage(), threadPool, tosConf(), magic);
+ }
+
+ protected void persist(Path p, byte[] data) {
+ getStorage().put(ObjectUtils.pathToKey(p), data);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/CommitterTestBase.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/CommitterTestBase.java
new file mode 100644
index 00000000000..e34088c23d2
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/CommitterTestBase.java
@@ -0,0 +1,381 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.commit.CommitUtils;
+import org.apache.hadoop.fs.tosfs.commit.Pending;
+import org.apache.hadoop.fs.tosfs.commit.PendingSet;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public abstract class CommitterTestBase {
+ private Configuration conf;
+ private FileSystem fs;
+ private Path outputPath;
+ private TaskAttemptID taskAttempt0;
+ private Path reportDir;
+
+ @BeforeEach
+ public void setup() throws IOException {
+ conf = newConf();
+ fs = FileSystem.get(conf);
+ String uuid = UUIDUtils.random();
+ outputPath = fs.makeQualified(new Path("/test/" + uuid));
+ taskAttempt0 = JobSuite.createTaskAttemptId(randomTrimmedJobId(), 0);
+
+ reportDir = fs.makeQualified(new Path("/report/" + uuid));
+ fs.mkdirs(reportDir);
+ conf.set(org.apache.hadoop.fs.tosfs.commit.Committer.COMMITTER_SUMMARY_REPORT_DIR,
+ reportDir.toUri().toString());
+ }
+
+ protected abstract Configuration newConf();
+
+ @AfterEach
+ public void teardown() {
+ CommonUtils.runQuietly(() -> fs.delete(outputPath, true));
+ IOUtils.closeStream(fs);
+ }
+
+ @BeforeClass
+ public static void beforeClass() {
+ Assumptions.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ if (!TestEnv.checkTestEnabled()) {
+ return;
+ }
+
+ List<String> committerThreads = Thread.getAllStackTraces().keySet()
+ .stream()
+ .map(Thread::getName)
+ .filter(n -> n.startsWith(org.apache.hadoop.fs.tosfs.commit.Committer.THREADS_PREFIX))
+ .collect(Collectors.toList());
+ assertTrue(committerThreads.isEmpty(), "Outstanding committer threads");
+ }
+
+ private static String randomTrimmedJobId() {
+ SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+ return String.format("%s%04d_%04d", formatter.format(new Date()),
+ (long) (Math.random() * 1000),
+ (long) (Math.random() * 1000));
+ }
+
+ private static String randomFormedJobId() {
+ return String.format("job_%s", randomTrimmedJobId());
+ }
+
+ @Test
+ public void testSetupJob() throws IOException {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+
+ // Setup job.
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+ }
+
+ @Test
+ public void testSetupJobWithOrphanPaths() throws IOException, InterruptedException {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+
+ // Orphan success marker.
+ Path successPath = CommitUtils.successMarker(outputPath);
+ CommitUtils.save(fs, successPath, new byte[]{});
+ assertTrue(fs.exists(successPath), "The success file should exist.");
+
+ // Orphan job path.
+ Path jobPath = CommitUtils.magicJobPath(suite.committer().jobId(), outputPath);
+ fs.mkdirs(jobPath);
+ assertTrue(fs.exists(jobPath), "The job path should exist.");
+ Path subPath = new Path(jobPath, "tmp.pending");
+ CommitUtils.save(fs, subPath, new byte[]{});
+ assertTrue(fs.exists(subPath), "The sub path under job path should be existing.");
+ FileStatus jobPathStatus = fs.getFileStatus(jobPath);
+
+ Thread.sleep(1000L);
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+
+ assertFalse(fs.exists(successPath), "Should have deleted the success path");
+ assertTrue(fs.exists(jobPath), "Should have re-created the job path");
+ assertFalse(fs.exists(subPath), "Should have deleted the sub path under the job path");
+ }
+
+ @Test
+ public void testSetupTask() throws IOException {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+
+ // Remaining attempt task path.
+ Path taskAttemptBasePath =
+ CommitUtils.magicTaskAttemptBasePath(suite.taskAttemptContext(), outputPath);
+ Path subTaskAttemptPath = new Path(taskAttemptBasePath, "tmp.pending");
+ CommitUtils.save(fs, subTaskAttemptPath, new byte[]{});
+ assertTrue(fs.exists(taskAttemptBasePath));
+ assertTrue(fs.exists(subTaskAttemptPath));
+
+ // Setup job.
+ suite.setupJob();
+ suite.assertHasMagicKeys();
+ // It will clear all the job path once we've set up the job.
+ assertFalse(fs.exists(taskAttemptBasePath));
+ assertFalse(fs.exists(subTaskAttemptPath));
+
+ // Left some the task paths.
+ CommitUtils.save(fs, subTaskAttemptPath, new byte[]{});
+ assertTrue(fs.exists(taskAttemptBasePath));
+ assertTrue(fs.exists(subTaskAttemptPath));
+
+ // Setup task.
+ suite.setupTask();
+ assertFalse(fs.exists(subTaskAttemptPath));
+ }
+
+ @Test
+ public void testCommitTask() throws Exception {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+ // Setup job
+ suite.setupJob();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicKeys();
+
+ // Setup task
+ suite.setupTask();
+
+ // Write records.
+ suite.assertNoMagicPendingFile();
+ suite.assertMultipartUpload(0);
+ suite.writeOutput();
+ suite.dumpObjectStorage();
+ suite.assertHasMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(1);
+ // Assert the pending file content.
+ Path pendingPath = suite.magicPendingPath();
+ byte[] pendingData = CommitUtils.load(suite.fs(), pendingPath);
+ Pending pending = Pending.deserialize(pendingData);
+ assertEquals(suite.destPartKey(), pending.destKey());
+ assertEquals(20, pending.length());
+ assertEquals(1, pending.parts().size());
+
+ // Commit the task.
+ suite.commitTask();
+
+ // Verify the pending set file.
+ suite.assertHasPendingSet();
+ // Assert the pending set file content.
+ Path pendingSetPath = suite.magicPendingSetPath();
+ byte[] pendingSetData = CommitUtils.load(suite.fs(), pendingSetPath);
+ PendingSet pendingSet = PendingSet.deserialize(pendingSetData);
+ assertEquals(suite.job().getJobID().toString(), pendingSet.jobId());
+ assertEquals(1, pendingSet.commits().size());
+ assertEquals(pending, pendingSet.commits().get(0));
+ assertEquals(pendingSet.extraData(), ImmutableMap.of(CommitUtils.TASK_ATTEMPT_ID,
+ suite.taskAttemptContext().getTaskAttemptID().toString()));
+
+ // Complete the multipart upload and verify the results.
+ ObjectStorage storage = suite.storage();
+ storage.completeUpload(pending.destKey(), pending.uploadId(), pending.parts());
+ suite.verifyPartContent();
+ }
+
+ @Test
+ public void testAbortTask() throws Exception {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+
+ // Pre-check before the output write.
+ suite.assertNoMagicPendingFile();
+ suite.assertMultipartUpload(0);
+
+ // Execute the output write.
+ suite.writeOutput();
+
+ // Post-check after the output write.
+ suite.assertHasMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(1);
+ // Assert the pending file content.
+ Path pendingPath = suite.magicPendingPath();
+ byte[] pendingData = CommitUtils.load(suite.fs(), pendingPath);
+ Pending pending = Pending.deserialize(pendingData);
+ assertEquals(suite.destPartKey(), pending.destKey());
+ assertEquals(20, pending.length());
+ assertEquals(1, pending.parts().size());
+
+ // Abort the task.
+ suite.abortTask();
+
+ // Verify the state after aborting task.
+ suite.assertNoMagicPendingFile();
+ suite.assertNoMagicMultipartUpload();
+ suite.assertMultipartUpload(0);
+ suite.assertNoTaskAttemptPath();
+ }
+
+ @Test
+ public void testCommitJob() throws Exception {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job.
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.assertSummaryReport(reportDir);
+ suite.verifyPartContent();
+ }
+
+
+ @Test
+ public void testCommitJobFailed() throws Exception {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job.
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ }
+
+ @Test
+ public void testTaskCommitAfterJobCommit() throws Exception {
+ JobSuite suite = JobSuite.create(conf, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+ suite.setupJob();
+ suite.setupTask();
+ suite.writeOutput();
+ suite.commitTask();
+
+ // Commit the job
+ suite.assertNoPartFiles();
+ suite.commitJob();
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.verifyPartContent();
+
+ // Commit the task again.
+ assertThrows(FileNotFoundException.class, suite::commitTask);
+ }
+
+ @Test
+ public void testTaskCommitWithConsistentJobId() throws Exception {
+ Configuration config = newConf();
+ String consistentJobId = randomFormedJobId();
+ config.set(CommitUtils.SPARK_WRITE_UUID, consistentJobId);
+ JobSuite suite = JobSuite.create(config, taskAttempt0, outputPath);
+ Assume.assumeFalse(suite.skipTests());
+
+ // By now, we have two "jobId"s, one is spark uuid, and the other is the jobId in taskAttempt.
+ // The job committer will adopt the former.
+ suite.setupJob();
+
+ // Next, we clear spark uuid, and set the jobId of taskAttempt to another value. In this case,
+ // the committer will take the jobId of taskAttempt as the final jobId, which is not consistent
+ // with the one that committer holds.
+ config.unset(CommitUtils.SPARK_WRITE_UUID);
+ JobConf jobConf = new JobConf(config);
+ String anotherJobId = randomTrimmedJobId();
+ TaskAttemptID taskAttemptId1 =
+ JobSuite.createTaskAttemptId(anotherJobId, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+ final TaskAttemptContext attemptContext1 =
+ JobSuite.createTaskAttemptContext(jobConf, taskAttemptId1, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+
+ assertThrows(IllegalArgumentException.class, () -> suite.setupTask(attemptContext1),
+ "JobId set in the context");
+
+ // Even though we use another taskAttempt, as long as we ensure the spark uuid is consistent,
+ // the jobId in committer is consistent.
+ config.set(CommitUtils.SPARK_WRITE_UUID, consistentJobId);
+ config.set(FileOutputFormat.OUTDIR, outputPath.toString());
+ jobConf = new JobConf(config);
+ anotherJobId = randomTrimmedJobId();
+ TaskAttemptID taskAttemptId2 =
+ JobSuite.createTaskAttemptId(anotherJobId, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+ TaskAttemptContext attemptContext2 =
+ JobSuite.createTaskAttemptContext(jobConf, taskAttemptId2, JobSuite.DEFAULT_APP_ATTEMPT_ID);
+
+ suite.setupTask(attemptContext2);
+ // Write output must use the same task context with setup task.
+ suite.writeOutput(attemptContext2);
+ // Commit task must use the same task context with setup task.
+ suite.commitTask(attemptContext2);
+ suite.assertPendingSetAtRightLocation();
+
+ // Commit the job
+ suite.assertNoPartFiles();
+ suite.commitJob();
+
+ // Verify the output.
+ suite.assertNoMagicMultipartUpload();
+ suite.assertNoMagicObjectKeys();
+ suite.assertSuccessMarker();
+ suite.verifyPartContent();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/JobSuite.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/JobSuite.java
new file mode 100644
index 00000000000..5376faec2cf
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/JobSuite.java
@@ -0,0 +1,227 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.commit.BaseJobSuite;
+import org.apache.hadoop.fs.tosfs.commit.CommitUtils;
+import org.apache.hadoop.fs.tosfs.commit.SuccessData;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobContextImpl;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskAttemptContextImpl;
+import org.apache.hadoop.mapred.TaskAttemptID;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.net.NetUtils;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public final class JobSuite extends BaseJobSuite {
+ private final JobContext jobContext;
+ private final TaskAttemptContext taskAttemptContext;
+ private final Committer committer;
+
+ private JobSuite(FileSystem fs, JobConf conf,
+ TaskAttemptID taskAttemptId, int appAttemptId, Path outputPath)
+ throws IOException {
+ setFs(fs);
+ // Initialize the job instance.
+ setJob(Job.getInstance(conf));
+ job().setJobID(JobID.forName(CommitUtils.buildJobId(conf, taskAttemptId.getJobID())));
+ this.jobContext = createJobContext(conf, taskAttemptId);
+ this.taskAttemptContext = createTaskAttemptContext(conf, taskAttemptId, appAttemptId);
+ setJobId(CommitUtils.buildJobId(jobContext));
+
+ // Set job output directory.
+ FileOutputFormat.setOutputPath(conf, outputPath);
+ setOutputPath(outputPath);
+ setObjectStorage(ObjectStorageFactory.create(outputPath.toUri().getScheme(),
+ outputPath.toUri().getAuthority(), conf));
+
+ // Initialize committer.
+ this.committer = new Committer();
+ this.committer.setupTask(taskAttemptContext);
+ }
+
+ public static JobSuite create(Configuration conf, TaskAttemptID taskAttemptId, Path outDir)
+ throws IOException {
+ FileSystem fs = outDir.getFileSystem(conf);
+ return new JobSuite(fs, new JobConf(conf), taskAttemptId, DEFAULT_APP_ATTEMPT_ID, outDir);
+ }
+
+ public static TaskAttemptID createTaskAttemptId(String trimmedJobId, int attemptId) {
+ String attempt = String.format("attempt_%s_m_000000_%d", trimmedJobId, attemptId);
+ return TaskAttemptID.forName(attempt);
+ }
+
+ public static JobContext createJobContext(JobConf jobConf, TaskAttemptID taskAttemptId) {
+ return new JobContextImpl(jobConf, taskAttemptId.getJobID());
+ }
+
+ public static TaskAttemptContext createTaskAttemptContext(
+ JobConf jobConf, TaskAttemptID taskAttemptId, int appAttemptId) throws IOException {
+ // Set the key values for job configuration.
+ jobConf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptId.toString());
+ jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, appAttemptId);
+ jobConf.set("mapred.output.committer.class",
+ Committer.class.getName()); // 2x and 3x newApiCommitter=false.
+ return new TaskAttemptContextImpl(jobConf, taskAttemptId);
+ }
+
+ public void setupJob() throws IOException {
+ committer.setupJob(jobContext);
+ }
+
+ public void setupTask() throws IOException {
+ committer.setupTask(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void setupTask(TaskAttemptContext taskAttemptCxt) throws IOException {
+ committer.setupTask(taskAttemptCxt);
+ }
+
+ public void writeOutput() throws Exception {
+ writeOutput(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void writeOutput(TaskAttemptContext taskAttemptCxt) throws Exception {
+ RecordWriter<Object, Object> writer = new TextOutputFormat<>().getRecordWriter(fs(),
+ taskAttemptCxt.getJobConf(),
+ CommitUtils.buildJobId(taskAttemptCxt),
+ taskAttemptCxt.getProgressible());
+ NullWritable nullKey = NullWritable.get();
+ NullWritable nullVal = NullWritable.get();
+ Object[] keys = new Object[]{KEY_1, nullKey, null, nullKey, null, KEY_2};
+ Object[] vals = new Object[]{VAL_1, nullVal, null, null, nullVal, VAL_2};
+ try {
+ assertEquals(keys.length, vals.length);
+ for (int i = 0; i < keys.length; i++) {
+ writer.write(keys[i], vals[i]);
+ }
+ } finally {
+ writer.close(Reporter.NULL);
+ }
+ }
+
+ public boolean needsTaskCommit() throws IOException {
+ return committer.needsTaskCommit(taskAttemptContext);
+ }
+
+ public void commitTask() throws IOException {
+ committer.commitTask(taskAttemptContext);
+ }
+
+ // This method simulates the scenario that the job may set up task with a different
+ // taskAttemptContext, e.g., for a spark job.
+ public void commitTask(TaskAttemptContext taskAttemptCxt) throws IOException {
+ committer.commitTask(taskAttemptCxt);
+ }
+
+ public void abortTask() throws IOException {
+ committer.abortTask(taskAttemptContext);
+ }
+
+ public void commitJob() throws IOException {
+ committer.commitJob(jobContext);
+ }
+
+ @Override
+ public Path magicPartPath() {
+ return new Path(committer.getWorkPath(), committer.jobId());
+ }
+
+ @Override
+ public Path magicPendingSetPath() {
+ return CommitUtils.magicTaskPendingSetPath(taskAttemptContext, outputPath());
+ }
+
+ public TaskAttemptContext taskAttemptContext() {
+ return taskAttemptContext;
+ }
+
+ public Committer committer() {
+ return committer;
+ }
+
+ @Override
+ public void assertNoTaskAttemptPath() throws IOException {
+ Path path = CommitUtils.magicTaskAttemptBasePath(taskAttemptContext, outputPath());
+ assertFalse(fs().exists(path), "Task attempt path should be not existing");
+ String pathToKey = ObjectUtils.pathToKey(path);
+ assertNull(storage().head(pathToKey), "Should have no task attempt path key");
+ }
+
+ @Override
+ protected boolean skipTests() {
+ return storage().bucket().isDirectory();
+ }
+
+ @Override
+ public void assertSuccessMarker() throws IOException {
+ Path succPath = CommitUtils.successMarker(outputPath());
+ assertTrue(fs().exists(succPath), String.format("%s should be exists", succPath));
+ SuccessData successData = SuccessData.deserialize(CommitUtils.load(fs(), succPath));
+ assertEquals(SuccessData.class.getName(), successData.name());
+ assertTrue(successData.success());
+ assertEquals(NetUtils.getHostname(), successData.hostname());
+ assertEquals(CommitUtils.COMMITTER_NAME, successData.committer());
+ assertEquals(
+ String.format("Task committer %s", taskAttemptContext.getTaskAttemptID()),
+ successData.description());
+ assertEquals(job().getJobID().toString(), successData.jobId());
+ assertEquals(1, successData.filenames().size());
+ assertEquals(destPartKey(), successData.filenames().get(0));
+ }
+
+ @Override
+ public void assertSummaryReport(Path reportDir) throws IOException {
+ Path reportPath = CommitUtils.summaryReport(reportDir, job().getJobID().toString());
+ assertTrue(fs().exists(reportPath), String.format("%s should be exists", reportPath));
+ SuccessData reportData = SuccessData.deserialize(CommitUtils.load(fs(), reportPath));
+ assertEquals(SuccessData.class.getName(), reportData.name());
+ assertTrue(reportData.success());
+ assertEquals(NetUtils.getHostname(), reportData.hostname());
+ assertEquals(CommitUtils.COMMITTER_NAME, reportData.committer());
+ assertEquals(String.format("Task committer %s", taskAttemptContext.getTaskAttemptID()),
+ reportData.description());
+ assertEquals(job().getJobID().toString(), reportData.jobId());
+ assertEquals(1, reportData.filenames().size());
+ assertEquals(destPartKey(), reportData.filenames().get(0));
+ assertEquals("clean", reportData.diagnostics().get("stage"));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/TestCommitter.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/TestCommitter.java
new file mode 100644
index 00000000000..48f90ec3d30
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/TestCommitter.java
@@ -0,0 +1,29 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.commit.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+
+public class TestCommitter extends CommitterTestBase {
+ @Override
+ protected Configuration newConf() {
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", String.format("tos://%s", TestUtility.bucket()));
+ return conf;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestChecksum.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestChecksum.java
new file mode 100644
index 00000000000..bdc14e05e84
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestChecksum.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+
+public class TestChecksum extends AbstractFSContractTestBase {
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+
+ private Path testCreateNewFile(String fileName, byte[] data, boolean useBuilder)
+ throws IOException {
+ describe("Foundational 'create a file' test, using builder API=" + useBuilder);
+ Path path = path(fileName, useBuilder);
+
+ writeDataset(getFileSystem(), path, data, data.length, 1024 * 1024, false, useBuilder);
+ ContractTestUtils.verifyFileContents(getFileSystem(), path, data);
+
+ return path;
+ }
+
+ private Path path(String filepath, boolean useBuilder) throws IOException {
+ return super.path(filepath + (useBuilder ? "" : "-builder"));
+ }
+
+ @Test
+ public void testCheckSumWithSimplePut() throws IOException {
+ byte[] data = dataset(256, 'a', 'z');
+ Path path1 = testCreateNewFile("file1", data, true);
+ Path path2 = testCreateNewFile("file2", data, true);
+ Path path3 = testCreateNewFile("file3", dataset(512, 'a', 'z'), true);
+
+ FileChecksum expected = getFileSystem().getFileChecksum(path1);
+ assertEquals("Checksum value should be same among objects with same content",
+ expected, getFileSystem().getFileChecksum(path2));
+ assertEquals("Checksum value should be same among multiple call for same object",
+ expected, getFileSystem().getFileChecksum(path1));
+ assertNotEquals(
+ "Checksum value should be different for different objects with different content", expected,
+ getFileSystem().getFileChecksum(path3));
+
+ Path renamed = path("renamed");
+ getFileSystem().rename(path1, renamed);
+ assertEquals("Checksum value should not change after rename",
+ expected, getFileSystem().getFileChecksum(renamed));
+ }
+
+ @Test
+ public void testCheckSumShouldSameViaPutAndMPU() throws IOException {
+ byte[] data = TestUtility.rand(11 << 20);
+
+ // simple put
+ Path singleFile = path("singleFile");
+ RawFileSystem fs = (RawFileSystem) getFileSystem();
+ fs.storage().put(ObjectUtils.pathToKey(singleFile), data);
+
+ // MPU upload data, the default threshold is 10MB
+ Path mpuFile = testCreateNewFile("mpuFile", data, true);
+
+ assertEquals(fs.getFileChecksum(singleFile), fs.getFileChecksum(mpuFile));
+ }
+
+ @Test
+ public void testDisableCheckSum() throws IOException {
+ Path path1 = testCreateNewFile("file1", dataset(256, 'a', 'z'), true);
+ Path path2 = testCreateNewFile("file2", dataset(512, 'a', 'z'), true);
+ assertNotEquals(getFileSystem().getFileChecksum(path1), getFileSystem().getFileChecksum(path2));
+
+ // disable checksum
+ Configuration newConf = new Configuration(getFileSystem().getConf());
+ newConf.setBoolean(ConfKeys.FS_CHECKSUM_ENABLED.key("tos"), false);
+ FileSystem newFS = FileSystem.get(newConf);
+
+ assertEquals(newFS.getFileChecksum(path1), newFS.getFileChecksum(path2));
+ }
+
+ @Test
+ public void testGetDirChecksum() throws IOException {
+ FileSystem fs = getFileSystem();
+
+ Path dir1 = path("dir1", true);
+ Path dir2 = path("dir2", true);
+ assertPathDoesNotExist("directory already exists", dir1);
+ assertPathDoesNotExist("directory already exists", dir2);
+ fs.mkdirs(dir1);
+
+ assertThrows("Path is not a file", FileNotFoundException.class,
+ () -> getFileSystem().getFileChecksum(dir1));
+ assertThrows("No such file or directory", FileNotFoundException.class,
+ () -> getFileSystem().getFileChecksum(dir2));
+
+ assertDeleted(dir1, false);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestCreate.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestCreate.java
new file mode 100644
index 00000000000..a9b4f083398
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestCreate.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.jupiter.api.BeforeAll;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestCreate extends AbstractContractCreateTest {
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDelete.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDelete.java
new file mode 100644
index 00000000000..31363432669
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDelete.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.tosfs.object.ObjectTestUtils.assertDirExist;
+import static org.apache.hadoop.fs.tosfs.object.ObjectTestUtils.assertObjectNotExist;
+
+public class TestDelete extends AbstractContractDeleteTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+
+ @Test
+ public void testParentDirCreatedAfterDeleteSubChildren() throws IOException {
+ Path path = path("testParentDirCreatedAfterDeleteSubChildren/");
+ Path file1 = new Path(path, "f1");
+ Path file2 = new Path(path, "f2");
+ ContractTestUtils.writeTextFile(getFileSystem(), file1,
+ "the first file", true);
+ ContractTestUtils.writeTextFile(getFileSystem(), file2,
+ "the second file", true);
+ assertPathExists("file1 not created", file1);
+ assertPathExists("file1 not created", file2);
+
+ assertObjectNotExist(path, false);
+ assertObjectNotExist(path, true);
+
+ assertDeleted(file1, false);
+ assertPathExists("parent path should exist", path);
+
+ assertObjectNotExist(path, false);
+ assertDirExist(path);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDistCp.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDistCp.java
new file mode 100644
index 00000000000..a87ac6e690b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDistCp.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestDistCp extends AbstractContractDistCpTest {
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+
+ // ignore this test case as there is intermittent IllegalStateException issue
+ @Test
+ public void testDistCpWithIterator() {
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestGetFileStatus.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestGetFileStatus.java
new file mode 100644
index 00000000000..2c631121bef
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestGetFileStatus.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+
+@RunWith(Parameterized.class)
+public class TestGetFileStatus extends AbstractContractGetFileStatusTest {
+
+ private final boolean getFileStatusEnabled;
+
+ @Parameterized.Parameters(name = "getFileStatusEnabled={0}")
+ public static List<Boolean> createParameters() {
+ return Arrays.asList(false, true);
+ }
+
+ public TestGetFileStatus(boolean getFileStatusEnabled) {
+ this.getFileStatusEnabled = getFileStatusEnabled;
+ }
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ conf.setBoolean(TosKeys.FS_TOS_GET_FILE_STATUS_ENABLED, getFileStatusEnabled);
+ conf.setBoolean(ConfKeys.FS_ASYNC_CREATE_MISSED_PARENT.key("tos"), false);
+ return new TosContract(conf);
+ }
+
+ @Test
+ public void testDirModificationTimeShouldNotBeZero() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path path = getContract().getTestPath();
+ fs.delete(path, true);
+
+ Path subfolder = path.suffix('/' + this.methodName.getMethodName() + "-" + UUIDUtils.random());
+ mkdirs(subfolder);
+
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertTrue(fileStatus.getModificationTime() > 0);
+ }
+
+ @Test
+ public void testThrowExceptionWhenListStatusForNonExistPath() {
+ FileSystem fs = getFileSystem();
+ Path path = getContract().getTestPath();
+
+ assertThrows("Path doesn't exist", FileNotFoundException.class,
+ () -> fs.listStatusIterator(new Path(path, "testListStatusForNonExistPath")));
+ }
+
+ @Test
+ public void testPathStatNonexistentFile() {
+ FileSystem fs = getFileSystem();
+ // working dir does not exist.
+ Path file = new Path(getContract().getTestPath(), this.methodName.getMethodName());
+ assertThrows("Path doesn't exist", FileNotFoundException.class, () -> fs.getFileStatus(file));
+ }
+
+ @Test
+ public void testPathStatExistentFile() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path file = new Path(getContract().getTestPath(), this.methodName.getMethodName());
+
+ int size = 1 << 20;
+ byte[] data = dataset(size, 'a', 'z');
+ createFile(fs, file, true, data);
+ FileStatus status = fs.getFileStatus(file);
+ Assert.assertTrue(status.isFile());
+ Assert.assertTrue(status.getModificationTime() > 0);
+ Assert.assertEquals(size, status.getLen());
+ }
+
+ @Test
+ public void testPathStatEmptyDirectory() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path workingPath = new Path(getContract().getTestPath(), this.methodName.getMethodName());
+ mkdirs(workingPath);
+
+ FileStatus dirStatus = fs.getFileStatus(workingPath);
+ Assert.assertTrue(dirStatus.isDirectory());
+ Assert.assertTrue(dirStatus.getModificationTime() > 0);
+ if (dirStatus instanceof RawFileStatus) {
+ Assert.assertArrayEquals(Constants.MAGIC_CHECKSUM, ((RawFileStatus) dirStatus).checksum());
+ }
+ }
+
+ @Test
+ public void testPathStatWhenCreateSubDir() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path workintPath = new Path(getContract().getTestPath(), this.methodName.getMethodName());
+ // create sub directory directly.
+ Path subDir = new Path(workintPath, UUIDUtils.random());
+ mkdirs(subDir);
+ Assert.assertTrue(fs.getFileStatus(subDir).isDirectory());
+
+ // can get FileStatus of working dir.
+ Assert.assertTrue(fs.getFileStatus(workintPath).isDirectory());
+ // delete sub directory.
+ fs.delete(subDir, true);
+ // still cat get FileStatus of working dir.
+ Assert.assertTrue(fs.getFileStatus(workintPath).isDirectory());
+ }
+
+ @Test
+ public void testPathStatDirNotExistButSubFileExist() throws IOException {
+ FileSystem fs = getFileSystem();
+ // working dir does not exist.
+ Path workintPath = new Path(getContract().getTestPath(), this.methodName.getMethodName());
+ assertThrows("Path doesn't exist", FileNotFoundException.class,
+ () -> fs.getFileStatus(workintPath));
+
+ // create sub file in working dir directly.
+ Path file = workintPath.suffix('/' + UUIDUtils.random());
+ touch(fs, file);
+
+ // can get FileStatus of working dir.
+ Assert.assertTrue(fs.getFileStatus(workintPath).isDirectory());
+
+ // delete sub file, will create parent directory.
+ fs.delete(file, false);
+ Assert.assertTrue(fs.getFileStatus(workintPath).isDirectory());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestMkdir.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestMkdir.java
new file mode 100644
index 00000000000..cfe12524bdf
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestMkdir.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
+public class TestMkdir extends AbstractContractMkdirTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestOpen.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestOpen.java
new file mode 100644
index 00000000000..29abb102416
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestOpen.java
@@ -0,0 +1,75 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.object.exceptions.ChecksumMismatchException;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+
+public class TestOpen extends AbstractContractOpenTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+
+ @Test
+ public void testOpenAExpiredFile() throws IOException {
+ Path file = path("testOpenAOutageFile");
+ FileSystem fs = getFileSystem();
+ byte[] data = dataset(256, 'a', 'z');
+ writeDataset(fs, file, data, data.length, 1024 * 1024, true);
+
+ FileStatus fileStatus = fs.getFileStatus(file);
+ if (fs instanceof RawFileSystem) {
+ byte[] expectChecksum = ((RawFileStatus) fileStatus).checksum();
+ FSDataInputStream fsDataInputStream =
+ ((RawFileSystem) fs).open(file, expectChecksum, Range.of(0, Long.MAX_VALUE));
+ fsDataInputStream.close();
+
+ // update the file
+ data = dataset(512, 'a', 'z');
+ writeDataset(fs, file, data, data.length, 1024 * 1024, true);
+
+ FSDataInputStream newStream =
+ ((RawFileSystem) fs).open(file, expectChecksum, Range.of(0, Long.MAX_VALUE));
+ assertThrows("the file is expired", ChecksumMismatchException.class, () -> newStream.read());
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRename.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRename.java
new file mode 100644
index 00000000000..d63e340516f
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRename.java
@@ -0,0 +1,281 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+
+public class TestRename extends AbstractContractRenameTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ // Add follow two keys into hadoop configuration.
+ String defaultScheme = FileSystem.getDefaultUri(conf).getScheme();
+ Configuration newConf = new Configuration(conf);
+ newConf.setLong(ConfKeys.FS_MULTIPART_SIZE.key(defaultScheme),
+ ConfKeys.FS_MULTIPART_SIZE_DEFAULT);
+ newConf.setLong(ConfKeys.FS_MULTIPART_THRESHOLD.key(defaultScheme),
+ ConfKeys.FS_MULTIPART_THRESHOLD_DEFAULT);
+
+ return new TosContract(newConf);
+ }
+
+ @Test
+ public void testSucceedRenameFile() throws IOException {
+ describe("check if source file and dest file exists when succeed to rename");
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+ FileSystem fs = getFileSystem();
+ byte[] data = dataset(256, 'a', 'z');
+ writeDataset(fs, renameSrc, data, data.length, 1024 * 1024, true);
+ boolean renamed = rename(renameSrc, renameDest);
+ assertTrue(renamed);
+ assertPathExists("dest file should exist when succeed to rename", renameDest);
+ assertPathDoesNotExist("source file should not exist when succeed to rename", renameSrc);
+ }
+
+ @Test
+ public void testSucceedRenameDir() throws IOException {
+ describe("check if source dir and dest dir exists when succeed to rename");
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+ int fileNums = 10;
+ int byteSize = 10 << 20; // trigger multipart upload
+ FileSystem fs = getFileSystem();
+ for (int i = 0; i < fileNums; i++) {
+ byte[] data = dataset(byteSize >> i, 'a', 'z');
+ writeDataset(fs, new Path(renameSrc, String.format("src%02d", i)), data, data.length,
+ 1024 * 1024, true);
+ }
+ boolean renamed = rename(renameSrc, renameDest);
+ assertTrue(renamed);
+ for (int i = 0; i < fileNums; i++) {
+ Path srcFilePath = new Path(renameSrc, String.format("src%02d", i));
+ Path dstFilePath = new Path(renameDest, String.format("src%02d", i));
+ byte[] data = dataset(byteSize >> i, 'a', 'z');
+ assertPathExists("dest file should exist when succeed to rename", dstFilePath);
+ assertPathDoesNotExist("source file should not exist when succeed to rename", srcFilePath);
+ try (InputStream is = fs.open(dstFilePath)) {
+ assertArrayEquals(data, IOUtils.toByteArray(is));
+ }
+ }
+ }
+
+ @Test
+ public void testFailedRename() throws IOException {
+ describe("check if source file and dest file exists when failed to rename");
+ Path renameSrc = path("src/renameSrc");
+ Path renameDest = path("src/renameSrc/renameDst");
+ FileSystem fs = getFileSystem();
+ byte[] data = dataset(256, 'a', 'z');
+ writeDataset(fs, renameSrc, data, data.length, 1024 * 1024, true);
+ boolean renamed;
+ try {
+ renamed = rename(renameSrc, renameDest);
+ } catch (IOException e) {
+ renamed = false;
+ }
+ assertFalse(renamed);
+ assertPathExists("source file should exist when failed to rename", renameSrc);
+ assertPathDoesNotExist("dest file should not exist when failed to rename", renameDest);
+ }
+
+ @Test
+ public void testRenameSmallFile() throws IOException {
+ testRenameFileByPut(1 << 20);
+ testRenameFileByPut(3 << 20);
+ }
+
+ @Test
+ public void testRenameLargeFile() throws IOException {
+ testRenameFileByUploadParts(16 << 20);
+ testRenameFileByUploadParts(10 << 20);
+ }
+
+ @Test
+ public void testRenameDirWithSubFileAndSubDir() throws IOException {
+ FileSystem fs = getFileSystem();
+
+ Path renameSrc = path("dir/renameSrc");
+ Path renameDest = path("dir/renameDst");
+ int size = 1024;
+ byte[] data = dataset(size, 'a', 'z');
+ String fileName = "file.txt";
+ writeDataset(fs, new Path(renameSrc, fileName), data, data.length, 1024, true);
+
+ String dirName = "dir";
+ Path dirPath = new Path(renameSrc, dirName);
+ mkdirs(dirPath);
+ assertPathExists("source dir should exist", dirPath);
+
+ boolean renamed = fs.rename(renameSrc, renameDest);
+
+ assertTrue(renamed);
+ Path srcFilePath = new Path(renameSrc, fileName);
+ Path dstFilePath = new Path(renameDest, fileName);
+ assertPathExists("dest file should exist when succeed to rename", dstFilePath);
+ assertPathDoesNotExist("source file should not exist when succeed to rename", srcFilePath);
+
+ assertPathExists("dest dir should exist when succeed to rename", new Path(renameDest, dirName));
+ assertPathDoesNotExist("source dir should not exist when succeed to rename",
+ new Path(renameSrc, dirName));
+
+ ContractTestUtils.cleanup("TEARDOWN", fs, getContract().getTestPath());
+ }
+
+ public void testRenameFileByPut(int size) throws IOException {
+ describe("check if use put method when rename file");
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+ FileSystem fs = getFileSystem();
+
+ byte[] data = dataset(size, 'a', 'z');
+ String fileName = String.format("%sMB.txt", size >> 20);
+ writeDataset(fs, new Path(renameSrc, fileName), data, data.length, 1024 * 1024, true);
+ boolean renamed = fs.rename(renameSrc, renameDest);
+
+ assertTrue(renamed);
+ Path srcFilePath = new Path(renameSrc, fileName);
+ Path dstFilePath = new Path(renameDest, fileName);
+ assertPathExists("dest file should exist when succeed to rename", dstFilePath);
+ assertPathDoesNotExist("source file should not exist when succeed to rename", srcFilePath);
+
+ assertPathExists("dest src should exist when succeed to rename", renameDest);
+ assertPathDoesNotExist("source src should not exist when succeed to rename", renameSrc);
+
+ try (InputStream is = fs.open(dstFilePath)) {
+ assertArrayEquals(data, IOUtils.toByteArray(is));
+ }
+ ContractTestUtils.cleanup("TEARDOWN", fs, getContract().getTestPath());
+ }
+
+ @Test
+ public void testCreateParentDirAfterRenameSubFile() throws IOException {
+ FileSystem fs = getFileSystem();
+
+ Path srcDir = path("srcDir");
+ Path destDir = path("destDir");
+
+ assertPathDoesNotExist("Src dir should not exist", srcDir);
+ assertPathDoesNotExist("Dest dir should not exist", destDir);
+ int size = 1 << 20;
+ byte[] data = dataset(size, 'a', 'z');
+ String fileName = String.format("%sMB.txt", size >> 20);
+ Path srcFile = new Path(srcDir, fileName);
+ Path destFile = new Path(destDir, fileName);
+ writeDataset(fs, srcFile, data, data.length, 1024 * 1024, true);
+
+ assertPathExists("Src file should exist", srcFile);
+ assertPathExists("Src dir should exist", srcDir);
+
+ mkdirs(destDir);
+ assertPathExists("Dest dir should exist", destDir);
+
+ boolean renamed = fs.rename(srcFile, destFile);
+ assertTrue(renamed);
+
+ assertPathExists("Dest file should exist", destFile);
+ assertPathExists("Dest dir should exist", destDir);
+ assertPathDoesNotExist("Src file should not exist", srcFile);
+ assertPathExists("Src dir should exist", srcDir);
+ }
+
+ @Test
+ public void testCreateParentDirAfterRenameSubDir() throws IOException {
+ FileSystem fs = getFileSystem();
+
+ Path srcDir = path("srcDir");
+ Path destDir = path("destDir");
+
+ assertPathDoesNotExist("Src dir should not exist", srcDir);
+ assertPathDoesNotExist("Dest dir should not exist", destDir);
+
+ String subDirName = String.format("subDir");
+ Path srcSubDir = new Path(srcDir, subDirName);
+ Path destDestDir = new Path(destDir, subDirName);
+ mkdirs(srcSubDir);
+
+ assertPathExists("Src sub dir should exist", srcSubDir);
+ assertPathExists("Src dir should exist", srcDir);
+
+ mkdirs(destDir);
+ assertPathExists("Dest dir should exist", destDir);
+
+ boolean renamed = fs.rename(srcSubDir, destDestDir);
+ assertTrue(renamed);
+
+ assertPathExists("Dest sub dir should exist", destDestDir);
+ assertPathExists("Dest dir should exist", destDir);
+ assertPathDoesNotExist("Src sub dir should not exist", srcSubDir);
+ assertPathExists("Src sub dir should exist", srcDir);
+ }
+
+ public void testRenameFileByUploadParts(int size) throws IOException {
+ describe("check if use upload parts method when rename file");
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+ FileSystem fs = getFileSystem();
+
+ byte[] data = dataset(size, 'a', 'z');
+ String fileName = String.format("%sMB.txt", size >> 20);
+ writeDataset(fs, new Path(renameSrc, fileName), data, data.length, 1024 * 1024, true);
+ boolean renamed = fs.rename(renameSrc, renameDest);
+
+ assertTrue(renamed);
+ Path srcFilePath = new Path(renameSrc, fileName);
+ Path dstFilePath = new Path(renameDest, fileName);
+ assertPathExists("dest file should exist when succeed to rename", dstFilePath);
+ assertPathDoesNotExist("source file should not exist when succeed to rename", srcFilePath);
+
+ try (InputStream is = fs.open(dstFilePath)) {
+ assertArrayEquals(data, IOUtils.toByteArray(is));
+ }
+ ContractTestUtils.cleanup("TEARDOWN", fs, getContract().getTestPath());
+ }
+
+ @Ignore
+ @Test
+ public void testRenameFileUnderFileSubdir() {
+ }
+
+ @Ignore
+ @Test
+ public void testRenameFileUnderFile() {
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRootDir.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRootDir.java
new file mode 100644
index 00000000000..6086d0a810b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRootDir.java
@@ -0,0 +1,37 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
+public class TestRootDir extends AbstractContractRootDirectoryTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestSeek.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestSeek.java
new file mode 100644
index 00000000000..ad5bef20508
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestSeek.java
@@ -0,0 +1,37 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
+public class TestSeek extends AbstractContractSeekTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestUnbuffer.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestUnbuffer.java
new file mode 100644
index 00000000000..06e5677c172
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestUnbuffer.java
@@ -0,0 +1,37 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
+public class TestUnbuffer extends AbstractContractUnbufferTest {
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestXAttr.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestXAttr.java
new file mode 100644
index 00000000000..7acf58d24f0
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestXAttr.java
@@ -0,0 +1,177 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+public class TestXAttr extends AbstractFSContractTestBase {
+ private static final String XATTR_NAME = "xAttrName";
+ private static final byte[] XATTR_VALUE = "xAttrValue".getBytes();
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new TosContract(conf);
+ }
+
+ @Test
+ public void testGetNonExistedXAttr() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testSetAndGet/file");
+ fs.create(path).close();
+
+ fs.setXAttr(path, XATTR_NAME, XATTR_VALUE);
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttr(path, "non-exist"));
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttrs(path, Arrays.asList("non-exist")));
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttrs(path, Arrays.asList("non-exist", XATTR_NAME)));
+ }
+
+ @Test
+ public void testSetAndGetWhenPathNotExist() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testXAttrWhenPathNotExist/file");
+ fs.delete(path);
+
+ assertThrows("No such file", FileNotFoundException.class,
+ () -> fs.setXAttr(path, XATTR_NAME, XATTR_VALUE));
+ assertThrows("No such file", FileNotFoundException.class,
+ () -> fs.getXAttrs(path));
+ assertThrows("No such file", FileNotFoundException.class,
+ () -> fs.removeXAttr(path, "name"));
+ }
+
+ @Test
+ public void testSetAndGet() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testSetAndGet/file");
+ fs.create(path).close();
+
+ fs.setXAttr(path, XATTR_NAME, XATTR_VALUE);
+ assertArrayEquals(XATTR_VALUE, fs.getXAttr(path, XATTR_NAME));
+ }
+
+ @Test
+ public void testSetAndGetNonExistedObject() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testSetAndGetOnNonExistedObject/dir-0/dir-1/file");
+ fs.create(path).close();
+
+ Path nonExistedPath = path.getParent().getParent();
+ fs.setXAttr(nonExistedPath, XATTR_NAME, XATTR_VALUE);
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttr(nonExistedPath, XATTR_NAME));
+ }
+
+ @Test
+ public void testSetAndGetOnExistedObjectDir() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testSetAndGetOnDir/dir-0/dir-1");
+ fs.mkdirs(path);
+
+ fs.setXAttr(path, XATTR_NAME, XATTR_VALUE);
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttr(path, XATTR_NAME));
+ }
+
+ @Test
+ public void testGetAndListAll() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testGetAndListAll/file");
+ fs.create(path).close();
+
+ int size = 10;
+ for (int i = 0; i < size; i++) {
+ fs.setXAttr(path, XATTR_NAME + i, Bytes.toBytes("VALUE" + i));
+ }
+
+ Map<String, byte[]> result = fs.getXAttrs(path);
+ assertEquals(size, result.size());
+ for (int i = 0; i < size; i++) {
+ assertEquals("VALUE" + i, Bytes.toString(result.get(XATTR_NAME + i)));
+ }
+
+ List<String> names = fs.listXAttrs(path);
+ assertEquals(size, names.size());
+ for (int i = 0; i < size; i++) {
+ assertTrue(names.contains(XATTR_NAME + i));
+ }
+ }
+
+ @Test
+ public void testRemove() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testRemove/file");
+ fs.create(path).close();
+
+ int size = 10;
+ for (int i = 0; i < size; i++) {
+ fs.setXAttr(path, XATTR_NAME + i, Bytes.toBytes("VALUE" + i));
+ }
+
+ for (int i = 0; i < size; i++) {
+ fs.removeXAttr(path, XATTR_NAME + i);
+ String name = XATTR_NAME + i;
+ assertThrows("Not found.", IOException.class,
+ () -> fs.getXAttr(path, name));
+ assertEquals(size - 1 - i, fs.listXAttrs(path).size());
+ }
+ }
+
+ @Test
+ public void testXAttrFlag() throws Exception {
+ FileSystem fs = getFileSystem();
+ Path path = path("testXAttrFlag/file");
+ fs.create(path).close();
+
+ String key = XATTR_NAME;
+ byte[] value = XATTR_VALUE;
+ assertThrows("The CREATE flag must be specified", IOException.class,
+ () -> fs.setXAttr(path, key, value, EnumSet.of(XAttrSetFlag.REPLACE)));
+ fs.setXAttr(path, key, value, EnumSet.of(XAttrSetFlag.CREATE));
+ assertArrayEquals(value, fs.getXAttr(path, key));
+
+ byte[] newValue = Bytes.toBytes("new value");
+ assertThrows("The REPLACE flag must be specified", IOException.class,
+ () -> fs.setXAttr(path, key, newValue, EnumSet.of(XAttrSetFlag.CREATE)));
+ fs.setXAttr(path, key, newValue, EnumSet.of(XAttrSetFlag.REPLACE));
+ assertArrayEquals(newValue, fs.getXAttr(path, key));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TosContract.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TosContract.java
new file mode 100644
index 00000000000..0f1ce3a4c75
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TosContract.java
@@ -0,0 +1,61 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.contract;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TosContract extends AbstractBondedFSContract {
+ private static final Logger LOG = LoggerFactory.getLogger(TosContract.class);
+ private final String testDir;
+
+ public TosContract(Configuration conf) {
+ super(conf);
+ addConfResource("contract/tos.xml");
+ // Set the correct contract test path if there is a provided bucket name from environment.
+ if (StringUtils.isNoneEmpty(TestUtility.bucket())) {
+ conf.set("fs.contract.test.fs.tos", String.format("tos://%s/", TestUtility.bucket()));
+ }
+
+ testDir = "/test-" + UUIDUtils.random();
+ }
+
+ @BeforeClass
+ public static void before() {
+ Assume.assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ public String getScheme() {
+ return "tos";
+ }
+
+ @Override
+ public Path getTestPath() {
+ LOG.info("the test dir is: {}", testDir);
+ return new Path(testDir);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageTestBase.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageTestBase.java
new file mode 100644
index 00000000000..484a414389b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectStorageTestBase.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class ObjectStorageTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(ObjectStorageTestBase.class);
+ private Configuration conf;
+ private Configuration tosConf;
+ private Path testDir;
+ private FileSystem fs;
+ private String scheme;
+ private ObjectStorage storage;
+
+ @TempDir
+ private java.nio.file.Path tempDir;
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ LOG.info("The test temporary folder is {}", tempDir);
+
+ String tempDirPath = tempDir.toAbsolutePath().toString();
+ conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key("filestore"), tempDirPath);
+ conf.set("fs.filestore.impl", LocalFileSystem.class.getName());
+ tosConf = new Configuration(conf);
+ // Set the environment variable for ObjectTestUtils#assertObject
+ TestUtility.setSystemEnv(FileStore.ENV_FILE_STORAGE_ROOT, tempDirPath);
+
+ testDir = new Path("filestore://" + FileStore.DEFAULT_BUCKET + "/", UUIDUtils.random());
+ fs = testDir.getFileSystem(conf);
+ scheme = testDir.toUri().getScheme();
+ storage = ObjectStorageFactory.create(scheme, testDir.toUri().getAuthority(), tosConf);
+ }
+
+ @AfterEach
+ public void tearDown() throws IOException {
+ if (storage != null) {
+ // List all keys with test dir prefix and delete them.
+ String prefix = ObjectUtils.pathToKey(testDir);
+ CommonUtils.runQuietly(() -> storage.deleteAll(prefix));
+ // List all multipart uploads and abort them.
+ CommonUtils.runQuietly(() -> {
+ for (MultipartUpload upload : storage.listUploads(prefix)) {
+ LOG.info("Abort the multipart upload {}", upload);
+ storage.abortMultipartUpload(upload.key(), upload.uploadId());
+ }
+ });
+
+ storage.close();
+ }
+ }
+
+ public Configuration conf() {
+ return conf;
+ }
+
+ public Configuration tosConf() {
+ return tosConf;
+ }
+
+ public Path testDir() {
+ return testDir;
+ }
+
+ public FileSystem fs() {
+ return fs;
+ }
+
+ public String scheme() {
+ return scheme;
+ }
+
+ public ObjectStorage getStorage() {
+ return storage;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectTestUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectTestUtils.java
new file mode 100644
index 00000000000..d8c23552c5e
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/ObjectTestUtils.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public final class ObjectTestUtils {
+
+ public static final byte[] EMPTY_BYTES = new byte[] {};
+
+ private ObjectTestUtils() {
+ }
+
+ /**
+ * Assert that all the parent directories should be existing.
+ *
+ * @param path to validate, can be directory or file.
+ */
+ public static void assertParentDirExist(Path path) throws IOException {
+ for (Path p = path.getParent(); p != null && p.getParent() != null; p = p.getParent()) {
+ assertObject(p, EMPTY_BYTES, true);
+ }
+ }
+
+ /**
+ * Assert that all the parent directories and current directory should be existing.
+ *
+ * @param path to validate, must be a directory.
+ */
+ public static void assertDirExist(Path path) throws IOException {
+ // All parent directories exist.
+ assertParentDirExist(path);
+ // The current directory exist.
+ assertObject(path, EMPTY_BYTES, true);
+ }
+
+ public static void assertObjectNotExist(Path path) throws IOException {
+ assertObjectNotExist(path, false);
+ }
+
+ public static void assertObjectNotExist(Path path, boolean isDir) throws IOException {
+ ObjectStorage store =
+ ObjectStorageFactory.create(path.toUri().getScheme(), path.toUri().getHost(),
+ new Configuration());
+ String objectKey = ObjectUtils.pathToKey(path, isDir);
+ ObjectInfo info = store.head(objectKey);
+ assertNull(info, String.format("Object key %s shouldn't exist in backend storage.", objectKey));
+
+ store.close();
+ }
+
+ public static void assertObject(Path path, byte[] data) throws IOException {
+ assertObject(path, data, false);
+ }
+
+ public static void assertObject(Path path, byte[] data, boolean isDir) throws IOException {
+ ObjectStorage store =
+ ObjectStorageFactory.create(path.toUri().getScheme(), path.toUri().getHost(),
+ new Configuration());
+ String objectKey = ObjectUtils.pathToKey(path, isDir);
+ // Verify the existence of object.
+ ObjectInfo info = store.head(objectKey);
+ assertNotNull(info, String.format("there should be an key %s in object storage", objectKey));
+ assertEquals(info.key(), objectKey);
+ assertEquals(data.length, info.size());
+ // Verify the data content.
+ try (InputStream in = store.get(objectKey, 0, -1).stream()) {
+ byte[] actual = IOUtils.toByteArray(in);
+ assertArrayEquals(data, actual, "Unexpected binary");
+ }
+
+ store.close();
+ }
+
+ public static void assertMultipartUploadExist(Path path, String uploadId) throws IOException {
+ ObjectStorage store =
+ ObjectStorageFactory.create(path.toUri().getScheme(), path.toUri().getHost(),
+ new Configuration());
+ String objectKey = ObjectUtils.pathToKey(path, false);
+
+ Iterator<MultipartUpload> uploadIterator = store.listUploads(objectKey).iterator();
+ assertTrue(uploadIterator.hasNext());
+ assertMultipartUploadIdExist(uploadIterator, uploadId);
+
+ store.close();
+ }
+
+ private static void assertMultipartUploadIdExist(Iterator<MultipartUpload> uploadIterator,
+ String uploadId) {
+ boolean exist = false;
+ while (uploadIterator.hasNext()) {
+ if (Objects.equals(uploadIterator.next().uploadId(), uploadId)) {
+ exist = true;
+ }
+ }
+ assertTrue(exist);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestDirectoryStorage.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestDirectoryStorage.java
new file mode 100644
index 00000000000..5d3965770af
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestDirectoryStorage.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hadoop.fs.tosfs.util.TestUtility.scheme;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestDirectoryStorage {
+ private final ObjectStorage storage;
+
+ public TestDirectoryStorage() {
+ Configuration conf = new Configuration();
+ storage =
+ ObjectStorageFactory.createWithPrefix(String.format("%s-%s/", scheme(), UUIDUtils.random()),
+ scheme(), TestUtility.bucket(), conf);
+ }
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @AfterEach
+ public void tearDown() {
+ CommonUtils.runQuietly(() -> storage.deleteAll(""));
+ for (MultipartUpload upload : storage.listUploads("")) {
+ storage.abortMultipartUpload(upload.key(), upload.uploadId());
+ }
+ }
+
+ @Test
+ public void testListEmptyDir() {
+ String key = "testListEmptyDir/";
+ mkdir(key);
+ assertNotNull(directoryStorage().head(key));
+
+ assertFalse(directoryStorage().listDir(key, false).iterator().hasNext());
+ assertFalse(directoryStorage().listDir(key, false).iterator().hasNext());
+ assertTrue(directoryStorage().isEmptyDir(key));
+ }
+
+ @Test
+ public void testListNonExistDir() {
+ String key = "testListNonExistDir/";
+ assertNull(directoryStorage().head(key));
+
+ assertFalse(directoryStorage().listDir(key, false).iterator().hasNext());
+ assertFalse(directoryStorage().listDir(key, false).iterator().hasNext());
+ assertTrue(directoryStorage().isEmptyDir(key));
+ }
+
+ @Test
+ public void testRecursiveList() {
+ String root = "root/";
+ String file1 = "root/file1";
+ String file2 = "root/afile2";
+ String dir1 = "root/dir1/";
+ String file3 = "root/dir1/file3";
+
+ mkdir(root);
+ mkdir(dir1);
+ touchFile(file1, TestUtility.rand(8));
+ touchFile(file2, TestUtility.rand(8));
+ touchFile(file3, TestUtility.rand(8));
+
+ Assertions.assertThat(directoryStorage().listDir(root, false))
+ .hasSize(3)
+ .extracting(ObjectInfo::key)
+ .contains(dir1, file1, file2);
+
+ Assertions.assertThat(directoryStorage().listDir(root, true))
+ .hasSize(4)
+ .extracting(ObjectInfo::key)
+ .contains(dir1, file1, file2, file3);
+ }
+
+ @Test
+ public void testRecursiveListWithSmallBatch() {
+ Configuration conf = new Configuration(directoryStorage().conf());
+ conf.setInt(TosKeys.FS_TOS_LIST_OBJECTS_COUNT, 5);
+ directoryStorage().initialize(conf, directoryStorage().bucket().name());
+
+ String root = "root/";
+ mkdir(root);
+
+ // Create 2 files start with 'a', 2 sub dirs start with 'b', 2 files start with 'c'
+ for (int i = 1; i <= 2; i++) {
+ touchFile("root/a-file-" + i, TestUtility.rand(8));
+ mkdir("root/b-dir-" + i + "/");
+ touchFile("root/c-file-" + i, TestUtility.rand(8));
+ }
+
+ // Create two files under each sub dirs.
+ for (int j = 1; j <= 2; j++) {
+ touchFile(String.format("root/b-dir-%d/file1", j), TestUtility.rand(8));
+ touchFile(String.format("root/b-dir-%d/file2", j), TestUtility.rand(8));
+ }
+
+ Assertions.assertThat(directoryStorage().listDir(root, false))
+ .hasSize(6)
+ .extracting(ObjectInfo::key)
+ .contains(
+ "root/a-file-1", "root/a-file-2",
+ "root/b-dir-1/", "root/b-dir-2/",
+ "root/c-file-1", "root/c-file-2");
+
+ Assertions.assertThat(directoryStorage().listDir(root, true))
+ .hasSize(10)
+ .extracting(ObjectInfo::key)
+ .contains(
+ "root/a-file-1", "root/a-file-2",
+ "root/b-dir-1/", "root/b-dir-1/file1", "root/b-dir-1/file2",
+ "root/b-dir-2/", "root/b-dir-2/file1", "root/b-dir-2/file2",
+ "root/c-file-1", "root/c-file-2");
+ }
+
+ @Test
+ public void testRecursiveListRoot() {
+ String root = "root/";
+ String dir1 = "root/dir1/";
+ mkdir(root);
+ mkdir(dir1);
+
+ Assertions.assertThat(directoryStorage().listDir("", true))
+ .hasSize(2)
+ .extracting(ObjectInfo::key)
+ .contains("root/", "root/dir1/");
+ }
+
+ @Test
+ public void testDeleteEmptyDir() {
+ String dir = "a/b/";
+ mkdir(dir);
+
+ directoryStorage().deleteDir(dir, false);
+ assertNull(directoryStorage().head(dir));
+ }
+
+ @Test
+ public void testDeleteNonEmptyDir() {
+ String dir = "a/b/";
+ String subDir = "a/b/c/";
+ String file = "a/b/file.txt";
+ mkdir(dir);
+ mkdir(subDir);
+ touchFile(file, new byte[10]);
+
+ assertThrows(RuntimeException.class, () -> directoryStorage().deleteDir(dir, false));
+ assertNotNull(directoryStorage().head(dir));
+ assertNotNull(directoryStorage().head(subDir));
+ assertNotNull(directoryStorage().head(file));
+
+ directoryStorage().deleteDir(dir, true);
+ assertNull(directoryStorage().head(dir));
+ assertNull(directoryStorage().head(subDir));
+ assertNull(directoryStorage().head(file));
+ }
+
+ @Test
+ public void testRecursiveDeleteDirViaTosSDK() {
+ Configuration conf = new Configuration(directoryStorage().conf());
+ conf.setBoolean(TosKeys.FS_TOS_RMR_CLIENT_ENABLE, true);
+ directoryStorage().initialize(conf, directoryStorage().bucket().name());
+
+ testDeleteNonEmptyDir();
+ }
+
+ // TOS doesn't enable recursive delete in server side currently.
+ @Disabled
+ @Test
+ public void testAtomicDeleteDir() {
+ Configuration conf = new Configuration(directoryStorage().conf());
+ conf.setBoolean(TosKeys.FS_TOS_RMR_SERVER_ENABLED, true);
+ directoryStorage().initialize(conf, directoryStorage().bucket().name());
+
+ testDeleteNonEmptyDir();
+ }
+
+ private void touchFile(String key, byte[] data) {
+ directoryStorage().put(key, data);
+ }
+
+ private void mkdir(String key) {
+ directoryStorage().put(key, new byte[0]);
+ }
+
+ private DirectoryStorage directoryStorage() {
+ assumeTrue(storage.bucket().isDirectory());
+ return (DirectoryStorage) storage;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectMultiRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectMultiRangeInputStream.java
new file mode 100644
index 00000000000..ee73e9db6d1
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectMultiRangeInputStream.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.object.exceptions.ChecksumMismatchException;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestObjectMultiRangeInputStream extends ObjectStorageTestBase {
+ private static ExecutorService threadPool;
+
+ @BeforeAll
+ public static void beforeClass() {
+ threadPool = ThreadPools.newWorkerPool("TestObjectInputStream-pool");
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ if (!threadPool.isShutdown()) {
+ threadPool.shutdown();
+ }
+ }
+
+ @Test
+ public void testSequentialAndRandomRead() throws IOException {
+ Path outPath = new Path(testDir(), "testSequentialAndRandomRead.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(5 << 20);
+ getStorage().put(key, rawData);
+
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(),
+ ObjectUtils.pathToKey(outPath), rawData.length, Long.MAX_VALUE, content.checksum())) {
+ // sequential read
+ assertEquals(0, in.getPos());
+ assertEquals(0, in.nextExpectPos());
+
+ byte[] b = new byte[1024];
+ int readCnt = in.read(b);
+ assertEquals(readCnt, b.length);
+ assertArrayEquals(Arrays.copyOfRange(rawData, 0, 1024), b);
+ assertEquals(1024, in.getPos());
+ assertEquals(1024, in.nextExpectPos());
+
+ readCnt = in.read(b);
+ assertEquals(readCnt, b.length);
+ assertArrayEquals(Arrays.copyOfRange(rawData, 1024, 2048), b);
+ assertEquals(2048, in.getPos());
+ assertEquals(2048, in.nextExpectPos());
+
+ // random read forward
+ in.seek(4 << 20);
+ assertEquals(4 << 20, in.getPos());
+ assertEquals(2048, in.nextExpectPos());
+
+ readCnt = in.read(b);
+ assertEquals(readCnt, b.length);
+ assertArrayEquals(Arrays.copyOfRange(rawData, 4 << 20, 1024 + (4 << 20)), b);
+ assertEquals((4 << 20) + 1024, in.getPos());
+ assertEquals((4 << 20) + 1024, in.nextExpectPos());
+
+ // random read back
+ in.seek(2 << 20);
+ assertEquals(2 << 20, in.getPos());
+ assertEquals((4 << 20) + 1024, in.nextExpectPos());
+
+ readCnt = in.read(b);
+ assertEquals(readCnt, b.length);
+ assertArrayEquals(Arrays.copyOfRange(rawData, 2 << 20, 1024 + (2 << 20)), b);
+ assertEquals((2 << 20) + 1024, in.getPos());
+ assertEquals((2 << 20) + 1024, in.nextExpectPos());
+ }
+ }
+
+ private InputStream getStream(String key) {
+ return getStorage().get(key).stream();
+ }
+
+ @Test
+ public void testReadSingleByte() throws IOException {
+ int len = 10;
+ Path outPath = new Path(testDir(), "testReadSingleByte.txt");
+ byte[] data = TestUtility.rand(len);
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] checksum = getStorage().put(key, data);
+
+ try (InputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(), key,
+ data.length, Long.MAX_VALUE, checksum)) {
+ for (int i = 0; i < data.length; i++) {
+ assertTrue(in.read() >= 0);
+ }
+ assertEquals(-1, in.read());
+ }
+ }
+
+ @Test
+ public void testReadStreamButTheFileChangedDuringReading() throws IOException {
+ int len = 2048;
+ Path outPath = new Path(testDir(), "testReadStreamButTheFileChangedDuringReading.txt");
+ byte[] data = TestUtility.rand(len);
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] checksum = getStorage().put(key, data);
+
+ try (InputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(), key,
+ data.length, 1024, checksum)) {
+ byte[] read = new byte[1024];
+ int n = in.read(read);
+ assertEquals(1024, n);
+
+ getStorage().put(key, TestUtility.rand(1024));
+ assertThrows(ChecksumMismatchException.class, () -> in.read(read), "The file is staled");
+ }
+ }
+
+ @Test
+ public void testRead100M() throws IOException {
+ testSequentialReadData(100 << 20, 6 << 20);
+ testSequentialReadData(100 << 20, 5 << 20);
+ }
+
+ @Test
+ public void testRead10M() throws IOException {
+ testSequentialReadData(10 << 20, 4 << 20);
+ testSequentialReadData(10 << 20, 5 << 20);
+ }
+
+ @Test
+ public void testParallelRead10M() throws IOException, ExecutionException, InterruptedException {
+ testParallelRandomRead(10 << 20, 4 << 20);
+ testParallelRandomRead(10 << 20, 5 << 20);
+ }
+
+ @Test
+ public void testRead100b() throws IOException {
+ testSequentialReadData(100, 40);
+ testSequentialReadData(100, 50);
+ testSequentialReadData(100, 100);
+ testSequentialReadData(100, 101);
+ }
+
+ private void testSequentialReadData(int dataSize, int partSize) throws IOException {
+ Path outPath = new Path(testDir(), String.format("%d-%d.txt", dataSize, partSize));
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(dataSize);
+ getStorage().put(key, rawData);
+
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ int batchSize = (dataSize - 1) / partSize + 1;
+ try (InputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(),
+ ObjectUtils.pathToKey(outPath), rawData.length, Long.MAX_VALUE, content.checksum())) {
+ for (int i = 0; i < batchSize; i++) {
+ int start = i * partSize;
+ int end = Math.min(dataSize, start + partSize);
+ byte[] expectArr = Arrays.copyOfRange(rawData, start, end);
+
+ byte[] b = new byte[end - start];
+ int ret = in.read(b, 0, b.length);
+
+ assertEquals(b.length, ret);
+ assertArrayEquals(expectArr, b, String.format("the read bytes mismatched at batch: %d", i));
+ }
+ assertEquals(-1, in.read());
+ }
+ }
+
+ private void testParallelRandomRead(int dataSize, int partSize)
+ throws IOException, ExecutionException, InterruptedException {
+
+ Path outPath = new Path(testDir(), String.format("%d-%d.txt", dataSize, partSize));
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(dataSize);
+ getStorage().put(key, rawData);
+
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ Random random = new Random();
+ List<Future<Boolean>> tasks = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ int position = random.nextInt(rawData.length);
+ tasks.add(threadPool.submit(
+ () -> testReadDataFromSpecificPosition(rawData, outPath, position, partSize,
+ content.checksum())));
+ }
+
+ for (Future<Boolean> task : tasks) {
+ assertTrue(task.get());
+ }
+ }
+
+ private boolean testReadDataFromSpecificPosition(
+ final byte[] rawData,
+ final Path objPath,
+ final int startPosition,
+ final int partSize,
+ byte[] checksum) {
+ int rawDataSize = rawData.length;
+ int batchSize = (rawDataSize - startPosition - 1) / partSize + 1;
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(),
+ ObjectUtils.pathToKey(objPath), rawDataSize, Long.MAX_VALUE, checksum)) {
+ in.seek(startPosition);
+
+ for (int i = 0; i < batchSize; i++) {
+ int start = startPosition + i * partSize;
+ int end = Math.min(rawDataSize, start + partSize);
+ byte[] expectArr = Arrays.copyOfRange(rawData, start, end);
+
+ byte[] b = new byte[end - start];
+ int ret = in.read(b, 0, b.length);
+
+ assertEquals(b.length, ret);
+ assertArrayEquals(expectArr, b, String.format("the read bytes mismatched at batch: %d", i));
+ }
+ assertEquals(-1, in.read());
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Test
+ public void testParallelReadFromOneInputStream() throws IOException, ExecutionException,
+ InterruptedException {
+ testParallelReadFromOneInputStreamImpl(10 << 20, 512, 10);
+ testParallelReadFromOneInputStreamImpl(10 << 20, 64, 100);
+ testParallelReadFromOneInputStreamImpl(1 << 20, 2 << 20, 5);
+ }
+
+ public void testParallelReadFromOneInputStreamImpl(int dataSize, int batchSize, int parallel)
+ throws IOException, ExecutionException, InterruptedException {
+
+ Path outPath = new Path(testDir(),
+ String.format("%d-%d-testParallelReadFromOneInputStreamImpl.txt", dataSize, batchSize));
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(dataSize);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ AtomicInteger sum = new AtomicInteger(0);
+ CopyOnWriteArrayList<byte[]> readBytes = new CopyOnWriteArrayList();
+ List<Future<?>> futures = new ArrayList<>();
+ try (ObjectMultiRangeInputStream inputStream = new ObjectMultiRangeInputStream(threadPool,
+ getStorage(), ObjectUtils.pathToKey(outPath), rawData.length, Long.MAX_VALUE,
+ content.checksum())) {
+ for (int i = 0; i < parallel; i++) {
+ futures.add(threadPool.submit(() -> {
+ byte[] data = new byte[batchSize];
+ try {
+ int count;
+ while ((count = inputStream.read(data)) != -1) {
+ sum.getAndAdd(count);
+ readBytes.add(Arrays.copyOfRange(data, 0, count));
+ data = new byte[batchSize];
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+ }
+
+ for (Future<?> future : futures) {
+ future.get();
+
+ }
+ assertEquals(rawData.length, sum.get());
+ }
+
+ byte[] actualBytes = new byte[rawData.length];
+ int offset = 0;
+ for (byte[] bytes : readBytes) {
+ System.arraycopy(bytes, 0, actualBytes, offset, bytes.length);
+ offset += bytes.length;
+ }
+
+ Arrays.sort(actualBytes);
+ Arrays.sort(rawData);
+ assertArrayEquals(rawData, actualBytes);
+ }
+
+ @Test
+ public void testPositionalRead() throws IOException {
+ Path outPath = new Path(testDir(), "testPositionalRead.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ int fileSize = 5 << 20;
+ byte[] rawData = TestUtility.rand(fileSize);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ Random rand = ThreadLocalRandom.current();
+
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(),
+ ObjectUtils.pathToKey(outPath), fileSize, Long.MAX_VALUE, content.checksum())) {
+ for (int i = 0; i < 100; i++) {
+ int pos = rand.nextInt(fileSize);
+ int len = rand.nextInt(fileSize);
+
+ int expectSize = Math.min(fileSize - pos, len);
+ byte[] actual = new byte[expectSize];
+ int actualLen = in.read(pos, actual, 0, expectSize);
+
+ assertEquals(expectSize, actualLen);
+ assertArrayEquals(Bytes.toBytes(rawData, pos, expectSize), actual);
+ }
+ }
+ }
+
+ @Test
+ public void testReadAcrossRange() throws IOException {
+ Path outPath = new Path(testDir(), "testReadAcrossRange.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ int fileSize = 1 << 10;
+ byte[] rawData = TestUtility.rand(fileSize);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(
+ ThreadPools.defaultWorkerPool(), getStorage(), key, fileSize, 10, content.checksum())) {
+ byte[] data = new byte[fileSize / 2];
+ for (int i = 0; i < 2; i++) {
+ assertEquals(data.length, in.read(data));
+ assertEquals((i + 1) * data.length, in.getPos());
+ assertArrayEquals(Bytes.toBytes(rawData, i * data.length, data.length), data);
+ }
+ }
+ }
+
+ @Test
+ public void testStorageRange() throws IOException {
+ Path outPath = new Path(testDir(), "testStorageRange.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ int fileSize = 5 << 20;
+ byte[] rawData = TestUtility.rand(fileSize);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ int oneMB = 1 << 20;
+ long rangeOpenLen = oneMB;
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(
+ ThreadPools.defaultWorkerPool(), getStorage(), key, fileSize, rangeOpenLen,
+ content.checksum())) {
+ assertNull(in.stream());
+
+ // Init range.
+ in.read();
+ assertEquals(Range.of(0, rangeOpenLen), in.stream().range());
+ // Range doesn't change.
+ in.read(new byte[(int) (rangeOpenLen - 1)], 0, (int) (rangeOpenLen - 1));
+ assertEquals(Range.of(0, rangeOpenLen), in.stream().range());
+
+ // Move to next range.
+ in.read();
+ assertEquals(Range.of(rangeOpenLen, rangeOpenLen), in.stream().range());
+
+ // Seek and move.
+ in.seek(rangeOpenLen * 3 + 10);
+ in.read();
+ assertEquals(Range.of(rangeOpenLen * 3, rangeOpenLen), in.stream().range());
+
+ // Seek small and range doesn't change.
+ in.seek(in.getPos() + 1);
+ in.read();
+ assertEquals(Range.of(rangeOpenLen * 3, rangeOpenLen), in.stream().range());
+
+ // Seek big and range changes.
+ in.seek(rangeOpenLen * 2);
+ in.read(new byte[(int) (rangeOpenLen - 10)], 0, (int) (rangeOpenLen - 10));
+ assertEquals(Range.of(rangeOpenLen * 2, rangeOpenLen), in.stream().range());
+ // Old range has 10 bytes left. Seek 10 bytes then read 10 bytes. Old range can't read any
+ // bytes, so range changes.
+ assertEquals(rangeOpenLen * 3 - 10, in.getPos());
+ in.seek(rangeOpenLen * 3);
+ in.read(new byte[10], 0, 10);
+ assertEquals(Range.of(rangeOpenLen * 3, rangeOpenLen), in.stream().range());
+
+ // Read big buffer.
+ in.seek(10);
+ in.read(new byte[oneMB * 3], 0, oneMB * 3);
+ assertEquals(oneMB * 3 + 10, in.getPos());
+ assertEquals(Range.of(3 * rangeOpenLen, rangeOpenLen), in.stream().range());
+ }
+
+ try (ObjectMultiRangeInputStream in = new ObjectMultiRangeInputStream(threadPool, getStorage(),
+ ObjectUtils.pathToKey(outPath), fileSize, Long.MAX_VALUE, content.checksum())) {
+ assertNull(in.stream());
+
+ // Init range.
+ in.read();
+ assertEquals(Range.of(0, fileSize), in.stream().range());
+
+ // Range doesn't change.
+ in.read(new byte[oneMB], 0, oneMB);
+ assertEquals(Range.of(0, fileSize), in.stream().range());
+
+ // Seek and move.
+ long pos = oneMB * 3 + 10;
+ in.seek(pos);
+ in.read();
+ assertEquals(Range.of(0, fileSize), in.stream().range());
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
new file mode 100644
index 00000000000..7c459dc29de
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.staging.StagingPart;
+import org.apache.hadoop.fs.tosfs.object.staging.State;
+import org.apache.hadoop.fs.tosfs.util.FSUtils;
+import org.apache.hadoop.fs.tosfs.util.TempFiles;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.apache.hadoop.util.Lists;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestObjectOutputStream extends ObjectStorageTestBase {
+
+ private static ExecutorService threadPool;
+
+ @BeforeAll
+ public static void beforeClass() {
+ threadPool = ThreadPools.newWorkerPool("TestObjectOutputStream-pool");
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ if (!threadPool.isShutdown()) {
+ threadPool.shutdown();
+ }
+ }
+
+ @Test
+ public void testMkStagingDir() throws ExecutionException, InterruptedException, IOException {
+ try (TempFiles tmp = TempFiles.of()) {
+ List<String> tmpDirs = Lists.newArrayList();
+ for (int i = 0; i < 3; i++) {
+ tmpDirs.add(tmp.newDir());
+ }
+ Configuration newConf = new Configuration(tosConf());
+ newConf.set(ConfKeys.FS_MULTIPART_STAGING_DIR.key("filestore"), Joiner.on(",").join(tmpDirs));
+
+ // Start multiple threads to open streams to create staging dir.
+ List<Future<ObjectOutputStream>> futures = Collections.synchronizedList(new ArrayList<>());
+ for (int i = 0; i < 10; i++) {
+ futures.add(threadPool.submit(() ->
+ new ObjectOutputStream(getStorage(), threadPool, newConf, path("none.txt"), true)));
+ }
+ for (Future<ObjectOutputStream> f : futures) {
+ f.get().close();
+ }
+ }
+ }
+
+ @Test
+ public void testWriteZeroByte() throws IOException {
+ Path zeroByteTxt = path("zero-byte.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), zeroByteTxt, true);
+ // write zero-byte and close.
+ out.write(new byte[0], 0, 0);
+ out.close();
+ assertStagingPart(0, out.stagingParts());
+
+ // Read and validate the dest object contents
+ ObjectTestUtils.assertObject(zeroByteTxt, ObjectTestUtils.EMPTY_BYTES);
+ }
+
+ @Test
+ public void testWriteZeroByteWithoutAllowPut() throws IOException {
+ Path zeroByteTxt = path("zero-byte-without-allow-put.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), zeroByteTxt, false);
+ // write zero-byte and close.
+ out.close();
+ assertStagingPart(0, out.stagingParts());
+
+ // Read and validate the dest object content.
+ ObjectTestUtils.assertObject(zeroByteTxt, ObjectTestUtils.EMPTY_BYTES);
+ }
+
+ @Test
+ public void testDeleteStagingFileWhenUploadPartsOK() throws IOException {
+ Path path = path("data.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), path, true);
+ byte[] data = TestUtility.rand((int) (ConfKeys.FS_MULTIPART_SIZE_DEFAULT * 2));
+ out.write(data);
+ out.waitForPartsUpload();
+ for (StagingPart part : out.stagingParts()) {
+ assertEquals(State.CLEANED, part.state());
+ }
+ out.close();
+ for (StagingPart part : out.stagingParts()) {
+ assertEquals(State.CLEANED, part.state());
+ }
+ }
+
+ @Test
+ public void testDeleteStagingFileWithClose() throws IOException {
+ Path path = path("data.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), path, true);
+ byte[] data = TestUtility.rand((int) (ConfKeys.FS_MULTIPART_SIZE_DEFAULT * 2));
+ out.write(data);
+ out.close();
+ for (StagingPart part : out.stagingParts()) {
+ assertEquals(State.CLEANED, part.state());
+ }
+ }
+
+ @Test
+ public void testDeleteSimplePutStagingFile() throws IOException {
+ Path smallTxt = path("small.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), smallTxt, true);
+ byte[] data = TestUtility.rand(4 << 20);
+ out.write(data);
+ for (StagingPart part : out.stagingParts()) {
+ assertTrue(part.size() > 0);
+ }
+ out.close();
+ for (StagingPart part : out.stagingParts()) {
+ assertEquals(State.CLEANED, part.state());
+ }
+ }
+
+ @Test
+ public void testSimplePut() throws IOException {
+ Path smallTxt = path("small.txt");
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), smallTxt, true);
+ byte[] data = TestUtility.rand(4 << 20);
+ out.write(data);
+ out.close();
+ assertStagingPart(1, out.stagingParts());
+ assertNull(out.upload(), "Should use the simple PUT to upload object for small file.");
+
+ // Read and validate the dest object content.
+ ObjectTestUtils.assertObject(smallTxt, data);
+ }
+
+ public void testWrite(int uploadPartSize, int len) throws IOException {
+ Configuration newConf = new Configuration(tosConf());
+ newConf.setLong(ConfKeys.FS_MULTIPART_SIZE.key(FSUtils.scheme(conf(), testDir().toUri())),
+ uploadPartSize);
+
+ Path outPath = path(len + ".txt");
+ int partNum = (len - 1) / uploadPartSize + 1;
+
+ byte[] data = TestUtility.rand(len);
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, newConf, outPath, true);
+ try {
+ out.write(data);
+ } finally {
+ out.close();
+ }
+
+ assertStagingPart(partNum, out.stagingParts());
+ ObjectTestUtils.assertObject(outPath, data);
+
+ // List multipart uploads
+ int uploadsNum = 0;
+ for (MultipartUpload ignored : getStorage().listUploads(out.destKey())) {
+ uploadsNum += 1;
+ }
+ assertEquals(0L, uploadsNum);
+ }
+
+ @Test
+ public void testParallelWriteOneOutPutStream() throws IOException, ExecutionException,
+ InterruptedException {
+ testParallelWriteOneOutPutStreamImpl(5 << 20, 10, 128);
+ testParallelWriteOneOutPutStreamImpl(5 << 20, 10, 1 << 20);
+ testParallelWriteOneOutPutStreamImpl(5 << 20, 10, 2 << 20);
+ testParallelWriteOneOutPutStreamImpl(5 << 20, 10, 6 << 20);
+ }
+
+ public void testParallelWriteOneOutPutStreamImpl(int partSize, int epochs, int batchSize)
+ throws IOException, ExecutionException, InterruptedException {
+ Configuration newConf = new Configuration(tosConf());
+ newConf.setLong(ConfKeys.FS_MULTIPART_SIZE.key(FSUtils.scheme(conf(), testDir().toUri())),
+ partSize);
+
+ String file =
+ String.format("%d-%d-%d-testParallelWriteOneOutPutStream.txt", partSize, epochs, batchSize);
+ Path outPath = path(file);
+ try (ObjectOutputStream out = new ObjectOutputStream(getStorage(), threadPool, newConf, outPath,
+ true)) {
+ List<Future<?>> futures = new ArrayList<>();
+ for (int i = 0; i < epochs; i++) {
+ final int index = i;
+ futures.add(threadPool.submit(() -> {
+ try {
+ out.write(dataset(batchSize, index));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+ }
+
+ // wait for all tasks finished
+ for (Future<?> future : futures) {
+ future.get();
+ }
+ }
+
+ try (InputStream inputStream = getStorage().get(ObjectUtils.pathToKey(outPath)).stream()) {
+ List<byte[]> ret = new ArrayList<>();
+ byte[] data = new byte[batchSize];
+ while (inputStream.read(data) != -1) {
+ ret.add(data);
+ data = new byte[batchSize];
+ }
+
+ assertEquals(epochs, ret.size());
+ List<byte[]> sortedRet = ret.stream()
+ .sorted(Comparator.comparingInt(o -> o[0]))
+ .collect(Collectors.toList());
+
+ int j = 0;
+ for (byte[] e : sortedRet) {
+ assertArrayEquals(dataset(batchSize, j), e);
+ j++;
+ }
+ }
+ }
+
+ public static byte[] dataset(int len, int base) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base);
+ }
+ return dataset;
+ }
+
+ @Test
+ public void testWrite1MB() throws IOException {
+ testWrite(5 << 20, 1 << 20);
+ testWrite(8 << 20, 1 << 20);
+ testWrite(16 << 20, 1 << 20);
+ }
+
+ @Test
+ public void testWrite24MB() throws IOException {
+ testWrite(5 << 20, 24 << 20);
+ testWrite(8 << 20, 24 << 20);
+ testWrite(16 << 20, 24 << 20);
+ }
+
+ @Test
+ public void testWrite100MB() throws IOException {
+ testWrite(5 << 20, 100 << 20);
+ testWrite(8 << 20, 100 << 20);
+ testWrite(16 << 20, 100 << 20);
+ }
+
+ private void testMultipartThreshold(int partSize, int multipartThreshold, int dataSize)
+ throws IOException {
+ Configuration newConf = new Configuration(tosConf());
+ newConf.setLong(ConfKeys.FS_MULTIPART_SIZE.key(scheme()), partSize);
+ newConf.setLong(ConfKeys.FS_MULTIPART_THRESHOLD.key(scheme()), multipartThreshold);
+ Path outPath =
+ path(String.format("threshold-%d-%d-%d.txt", partSize, multipartThreshold, dataSize));
+
+ byte[] data = TestUtility.rand(dataSize);
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, newConf, outPath, true);
+ try {
+ // Verify for every 1MB data writing, unless reaching the threshold.
+ int upperLimit = Math.min(multipartThreshold, dataSize);
+ int curOff = 0;
+ for (; curOff < upperLimit; curOff += (1 << 20)) {
+ int end = Math.min(curOff + (1 << 20), upperLimit);
+ out.write(Arrays.copyOfRange(data, curOff, end));
+
+ List<MultipartUpload> uploads = Lists.newArrayList(getStorage().listUploads(out.destKey()));
+ if (end < multipartThreshold) {
+ assertEquals(0, uploads.size(),
+ "Shouldn't has any uploads because it just use simple PUT");
+ } else {
+ assertEquals(1, uploads.size(), "Switch to use MPU.");
+ }
+ assertEquals((end - 1) / partSize + 1, out.stagingParts().size());
+ }
+
+ // Verify for every 1MB data writing, unless reaching the data size.
+ for (; curOff < dataSize; curOff += (1 << 20)) {
+ int end = Math.min(curOff + (1 << 20), dataSize);
+ out.write(Arrays.copyOfRange(data, curOff, end));
+
+ List<MultipartUpload> uploads = Lists.newArrayList(getStorage().listUploads(out.destKey()));
+ assertEquals(1, uploads.size());
+ assertEquals(out.destKey(), uploads.get(0).key());
+ assertEquals((end - 1) / partSize + 1, out.stagingParts().size());
+ }
+ } finally {
+ out.close();
+ }
+
+ assertStagingPart((dataSize - 1) / partSize + 1, out.stagingParts());
+ ObjectTestUtils.assertObject(outPath, data);
+
+ List<MultipartUpload> uploads = Lists.newArrayList(getStorage().listUploads(out.destKey()));
+ assertEquals(0, uploads.size());
+ }
+
+ @Test
+ public void testMultipartThreshold2MB() throws IOException {
+ testMultipartThreshold(5 << 20, 2 << 20, 1 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, (2 << 20) - 1);
+ testMultipartThreshold(5 << 20, 2 << 20, 2 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, 4 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, 5 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, (5 << 20) + 1);
+ testMultipartThreshold(5 << 20, 2 << 20, 6 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, 10 << 20);
+ testMultipartThreshold(5 << 20, 2 << 20, 20 << 20);
+ }
+
+ @Test
+ public void testMultipartThreshold5MB() throws IOException {
+ testMultipartThreshold(5 << 20, 5 << 20, 1 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 4 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 5 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 5 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 6 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 10 << 20);
+ testMultipartThreshold(5 << 20, 5 << 20, 20 << 20);
+ }
+
+ @Test
+ public void testMultipartThreshold10MB() throws IOException {
+ testMultipartThreshold(5 << 20, 10 << 20, 1 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 10 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 11 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 15 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 20 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 40 << 20);
+ testMultipartThreshold(5 << 20, 10 << 20, 30 << 20);
+ }
+
+ @Test
+ public void testCloseStreamTwice() throws IOException {
+ int len = 100;
+ Path outPath = path(len + ".txt");
+ int partNum = 1;
+
+ byte[] data = TestUtility.rand(len);
+ ObjectOutputStream out =
+ new ObjectOutputStream(getStorage(), threadPool, tosConf(), outPath, true);
+ try {
+ out.write(data);
+ out.close();
+ } finally {
+ out.close();
+ }
+
+ assertStagingPart(partNum, out.stagingParts());
+ ObjectTestUtils.assertObject(outPath, data);
+ }
+
+ @Test
+ public void testWriteClosedStream() throws IOException {
+ byte[] data = TestUtility.rand(10);
+ Path outPath = path("testWriteClosedStream.txt");
+ try (ObjectOutputStream out = new ObjectOutputStream(getStorage(), threadPool, tosConf(),
+ outPath, true)) {
+ out.close();
+ out.write(data);
+ } catch (IllegalStateException e) {
+ assertEquals("OutputStream is closed.", e.getMessage());
+ }
+ }
+
+ private static void assertStagingPart(int expectedNum, List<StagingPart> parts) {
+ assertEquals(expectedNum, parts.size());
+ for (StagingPart part : parts) {
+ assertTrue(part.size() > 0);
+ }
+ }
+
+ private Path path(String name) {
+ return new Path(testDir(), name);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectRangeInputStream.java
new file mode 100644
index 00000000000..45db5c66fb6
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectRangeInputStream.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestObjectRangeInputStream extends ObjectStorageTestBase {
+
+ @Test
+ public void testRead() throws IOException {
+ Path outPath = new Path(testDir(), "testRead.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(1 << 10);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ int position = 100;
+ int len = 200;
+ try (ObjectRangeInputStream ri = new ObjectRangeInputStream(getStorage(), key,
+ Range.of(position, len), content.checksum())) {
+ // Test read byte.
+ assertEquals(rawData[position] & 0xff, ri.read());
+
+ // Test read buffer.
+ byte[] buffer = new byte[len];
+ assertEquals(buffer.length - 1, ri.read(buffer, 0, buffer.length));
+ assertArrayEquals(
+ Arrays.copyOfRange(rawData, position + 1, position + len),
+ Arrays.copyOfRange(buffer, 0, buffer.length - 1));
+ assertEquals(0, ri.available());
+
+ assertEquals(-1, ri.read());
+ assertEquals(-1, ri.read(buffer, 0, buffer.length));
+ }
+ }
+
+ @Test
+ public void testRangeExceedInnerStream() throws IOException {
+ Path outPath = new Path(testDir(), "testRangeExceedInnerStream.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(10);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ int position = 10;
+ int badLen = 10;
+ try (ObjectRangeInputStream ri = new ObjectRangeInputStream(getStorage(), key,
+ Range.of(position, badLen), content.checksum())) {
+ byte[] buffer = new byte[1];
+ assertEquals(-1, ri.read());
+ assertEquals(-1, ri.read(buffer, 0, buffer.length));
+ }
+ }
+
+ @Test
+ public void testRangeInclude() throws IOException {
+ Path outPath = new Path(testDir(), "testRangeInclude.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(10);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ long pos = 100;
+ long len = 300;
+
+ try (ObjectRangeInputStream in = new ObjectRangeInputStream(getStorage(), key,
+ Range.of(pos, len), content.checksum())) {
+ assertEquals(Range.of(pos, len), in.range());
+
+ assertTrue(in.include(pos));
+ assertTrue(in.include((pos + len) / 2));
+ assertTrue(in.include(pos + len - 1));
+
+ assertFalse(in.include(pos - 1));
+ assertFalse(in.include(pos + len));
+ }
+ }
+
+ @Test
+ public void testSeek() throws IOException {
+ Path outPath = new Path(testDir(), "testSeek.txt");
+ String key = ObjectUtils.pathToKey(outPath);
+ byte[] rawData = TestUtility.rand(1 << 10);
+ getStorage().put(key, rawData);
+ ObjectContent content = getStorage().get(key);
+ assertArrayEquals(rawData, IOUtils.toByteArray(content.stream()));
+
+ long pos = 100;
+ long len = 300;
+
+ try (ObjectRangeInputStream in = new ObjectRangeInputStream(getStorage(), key,
+ Range.of(pos, len), content.checksum())) {
+ assertEquals(pos, in.getPos());
+
+ Exception error = assertThrows(IllegalArgumentException.class, () -> in.seek(-1), "Overflow");
+ assertTrue(error.getMessage().contains("must be in range Range{offset=100, length=300}"));
+ error = assertThrows(IllegalArgumentException.class, () -> in.seek(99), "Overflow");
+ assertTrue(error.getMessage().contains("must be in range Range{offset=100, length=300}"));
+ error = assertThrows(IllegalArgumentException.class, () -> in.seek(401), "Overflow");
+ assertTrue(error.getMessage().contains("must be in range Range{offset=100, length=300}"));
+ error = assertThrows(IllegalArgumentException.class, () -> in.seek(1 << 20), "Overflow");
+ assertTrue(error.getMessage().contains("must be in range Range{offset=100, length=300}"));
+
+ in.seek(399);
+ assertTrue(0 <= in.read());
+ assertEquals(-1, in.read());
+
+ in.seek(100);
+ assertTrue(in.read() >= 0);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
new file mode 100644
index 00000000000..1c18219404f
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
@@ -0,0 +1,1495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import com.volcengine.tos.TosServerException;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.exceptions.InvalidObjectKeyException;
+import org.apache.hadoop.fs.tosfs.object.exceptions.NotAppendableException;
+import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
+import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TempFiles;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestObjectStorage {
+ private static final String FILE_STORE_ROOT = TempFiles.newTempDir("TestObjectStorage");
+ private ObjectStorage storage;
+
+ public static Stream<Arguments> provideArguments() {
+ assumeTrue(TestEnv.checkTestEnabled());
+
+ List<Arguments> values = new ArrayList<>();
+ for (ObjectStorage store : TestUtility.createTestObjectStorage(FILE_STORE_ROOT)) {
+ values.add(Arguments.of(store));
+ }
+ return values.stream();
+ }
+
+ private void setEnv(ObjectStorage objectStore) {
+ this.storage = objectStore;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ CommonUtils.runQuietly(() -> storage.deleteAll(""));
+ for (MultipartUpload upload : storage.listUploads("")) {
+ storage.abortMultipartUpload(upload.key(), upload.uploadId());
+ }
+ }
+
+ @AfterAll
+ public static void afterClass() throws Exception {
+ CommonUtils.runQuietly(() -> TempFiles.deleteDir(FILE_STORE_ROOT));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testHeadNonExistObject(ObjectStorage store) {
+ setEnv(store);
+ assertNull(storage.head("a/b/c.txt"));
+
+ byte[] data = TestUtility.rand(256);
+ storage.put("a/b/c.txt", data);
+ assertNotNull(storage.head("a/b/c.txt"));
+
+ assertNull(storage.head("a/b/c/non-exits"));
+ if (storage.bucket().isDirectory()) {
+ assertThrows(InvalidObjectKeyException.class, () -> storage.head("a/b/c.txt/non-exits"));
+ } else {
+ assertNull(storage.head("a/b/c.txt/non-exits"));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testHeadExistObject(ObjectStorage store) {
+ setEnv(store);
+ byte[] data = TestUtility.rand(256);
+ String key = "testHeadExistObject.txt";
+ storage.put(key, data);
+
+ ObjectInfo obj = storage.head(key);
+ assertEquals(key, obj.key());
+ assertFalse(obj.isDir());
+ if (storage.bucket().isDirectory()) {
+ assertThrows(InvalidObjectKeyException.class, () -> storage.head(key + "/"));
+ } else {
+ assertNull(storage.head(key + "/"));
+ }
+
+ String dirKey = "testHeadExistObject/";
+ storage.put(dirKey, new byte[0]);
+ obj = storage.head(dirKey);
+ assertEquals(dirKey, obj.key());
+ assertTrue(obj.isDir());
+
+ if (storage.bucket().isDirectory()) {
+ obj = storage.head("testHeadExistObject");
+ assertEquals("testHeadExistObject", obj.key());
+ assertTrue(obj.isDir());
+ } else {
+ assertNull(storage.head("testHeadExistObject"));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testGetAndDeleteNonExistFile(ObjectStorage store) {
+ setEnv(store);
+ // ensure file is not exist
+ assertNull(storage.head("a/b/c.txt"));
+
+ assertThrows(RuntimeException.class, () -> storage.get("a/b/c.txt", 0, 0));
+ assertThrows(RuntimeException.class, () -> storage.get("a/b/c.txt", 0, 1));
+
+ // Allow to delete a non-exist object.
+ storage.delete("a/b/c.txt");
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutAndDeleteFileWithEmptyKey(ObjectStorage store) {
+ setEnv(store);
+ assertThrows(RuntimeException.class, () -> storage.put("", new byte[0]));
+ assertThrows(RuntimeException.class, () -> storage.put(null, new byte[0]));
+ assertThrows(RuntimeException.class, () -> storage.delete(null));
+ assertThrows(RuntimeException.class, () -> storage.head(""));
+ assertThrows(RuntimeException.class, () -> storage.head(null));
+ assertThrows(RuntimeException.class, () -> getStream(""));
+ assertThrows(RuntimeException.class, () -> getStream(null));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutObjectButContentLengthDisMatch(ObjectStorage store) throws IOException {
+ setEnv(store);
+ byte[] data = TestUtility.rand(256);
+ String key = "a/truncated.txt";
+
+ // The final object data will be truncated if content length is smaller.
+ byte[] checksum = storage.put(key, () -> new ByteArrayInputStream(data), 200);
+ assertArrayEquals(Arrays.copyOfRange(data, 0, 200), IOUtils.toByteArray(getStream(key)));
+ ObjectInfo info = storage.head(key);
+ assertEquals(key, info.key());
+ assertEquals(200, info.size());
+ assertArrayEquals(checksum, info.checksum());
+
+ // Will create object failed is the content length is bigger.
+ assertThrows(RuntimeException.class,
+ () -> storage.put(key, () -> new ByteArrayInputStream(data), 300));
+ }
+
+ private InputStream getStream(String key) {
+ return storage.get(key).stream();
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutAndGetFile(ObjectStorage store) throws IOException {
+ setEnv(store);
+ byte[] data = TestUtility.rand(256);
+ String key = "a/test.txt";
+ byte[] checksum = storage.put(key, data);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+
+ if (storage.bucket().isDirectory()) {
+ // Directory bucket will create missed parent dir.
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream("a")));
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream("a/")));
+ } else {
+ assertNull(storage.head("a"));
+ assertNull(storage.head("a/"));
+ }
+
+ ObjectInfo info = storage.head(key);
+ assertEquals(key, info.key());
+ assertEquals(data.length, info.size());
+ assertArrayEquals(checksum, info.checksum());
+
+ ObjectContent content = storage.get(key);
+ assertArrayEquals(info.checksum(), content.checksum());
+ assertArrayEquals(data, IOUtils.toByteArray(content.stream()));
+
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key, 0, -1)));
+ assertThrows(RuntimeException.class, () -> storage.get(key, -1, -1), "offset is negative");
+ assertThrows(RuntimeException.class, () -> storage.get(key + "/", 0, -1),
+ "path not found or resource type is invalid");
+
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key, 0, 256)));
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key, 0, 512)));
+
+ byte[] secondHalfData = Arrays.copyOfRange(data, 128, 256);
+ assertArrayEquals(secondHalfData, IOUtils.toByteArray(getStream(key, 128, -1)));
+ assertArrayEquals(secondHalfData, IOUtils.toByteArray(getStream(key, 128, 256)));
+ assertArrayEquals(secondHalfData, IOUtils.toByteArray(getStream(key, 128, 257)));
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream(key, 128, 0)));
+
+ ObjectContent partContent = storage.get(key, 8, 32);
+ assertArrayEquals(info.checksum(), partContent.checksum());
+ assertArrayEquals(Arrays.copyOfRange(data, 8, 40),
+ IOUtils.toByteArray(partContent.stream()));
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+
+ assertThrows(RuntimeException.class, () -> storage.get(key, 257, 8),
+ "offset is bigger than object length");
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream(key, 256, 8)));
+
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream(key, 0, 0)));
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream(key, 1, 0)));
+
+
+ // assert the original data is not changed during random get request
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+
+ storage.delete(key);
+ assertNull(storage.head(key));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testAppendAndGetFile(ObjectStorage store) throws Exception {
+ setEnv(store);
+ String key = "a/testAppendAndGetFile.txt";
+
+ // Append zero bytes.
+ assertThrows(NotAppendableException.class, () -> storage.append(key, new byte[0]),
+ "Append non-existed object with zero byte is not supported.");
+
+ // Append 256 bytes.
+ byte[] data = TestUtility.rand(256);
+ byte[] checksum = storage.append(key, data);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+
+ // Append zero bytes.
+ byte[] newChecksum = storage.append(key, new byte[0]);
+ assertArrayEquals(checksum, newChecksum);
+ checksum = newChecksum;
+
+ // Append one byte.
+ newChecksum = storage.append(key, new byte[1]);
+ assertFalse(Arrays.equals(checksum, newChecksum));
+ assertArrayEquals(newChecksum, storage.head(key).checksum());
+ checksum = newChecksum;
+
+ // Append 1024 byte.
+ data = TestUtility.rand(1024);
+ newChecksum = storage.append(key, data);
+ assertFalse(Arrays.equals(checksum, newChecksum));
+ assertArrayEquals(newChecksum, storage.head(key).checksum());
+
+ storage.delete(key);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testAppendLengthNotMatch(ObjectStorage store) {
+ setEnv(store);
+ byte[] data = TestUtility.rand(256);
+ String key = "a/testAppendLengthNotMatch.txt";
+ storage.append(key, () -> new ByteArrayInputStream(data), 128);
+ assertEquals(128, storage.head(key).size());
+
+ assertThrows(RuntimeException.class,
+ () -> storage.append(key, () -> new ByteArrayInputStream(data), 1024),
+ "Expect unexpected end of stream error.");
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testHeadAndListAndObjectStatusShouldGetSameObjectInfo(ObjectStorage store) {
+ setEnv(store);
+ String key = "testHeadAndListObjectCheckSum.txt";
+ byte[] data = TestUtility.rand(256);
+ byte[] checksum = storage.put(key, data);
+
+ ObjectInfo obj = storage.head(key);
+ assertEquals(obj, storage.objectStatus(key));
+ if (!storage.bucket().isDirectory()) {
+ List<ObjectInfo> objects = toList(storage.list(key, null, 1));
+ assertEquals(1, objects.size());
+ assertEquals(obj, objects.get(0));
+ assertArrayEquals(checksum, objects.get(0).checksum());
+ }
+
+
+ key = "testHeadAndListObjectCheckSum/";
+ checksum = storage.put(key, new byte[0]);
+ obj = storage.head(key);
+ assertEquals(obj, storage.objectStatus(key));
+ if (!storage.bucket().isDirectory()) {
+ List<ObjectInfo> objects = toList(storage.list(key, null, 1));
+ assertEquals(1, objects.size());
+ assertEquals(obj, objects.get(0));
+ assertArrayEquals(checksum, objects.get(0).checksum());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testObjectStatus(ObjectStorage store) {
+ setEnv(store);
+ // test get file status
+ String key = "a/b/testObjectStatus.txt";
+ byte[] data = TestUtility.rand(256);
+ byte[] checksum = storage.put(key, data);
+
+ ObjectInfo obj = storage.head(key);
+ assertArrayEquals(checksum, obj.checksum());
+ assertEquals(obj, storage.objectStatus(key));
+
+ if (storage.bucket().isDirectory()) {
+ assertThrows(InvalidObjectKeyException.class, () -> storage.head(key + "/"));
+ assertThrows(InvalidObjectKeyException.class, () -> storage.objectStatus(key + "/"));
+ } else {
+ assertNull(storage.head(key + "/"));
+ assertNull(storage.objectStatus(key + "/"));
+ }
+
+ // test get dir status
+ String dirKey = "a/b/dir/";
+ checksum = storage.put(dirKey, new byte[0]);
+ obj = storage.head(dirKey);
+ assertEquals(Constants.MAGIC_CHECKSUM, checksum);
+ assertArrayEquals(Constants.MAGIC_CHECKSUM, checksum);
+ assertArrayEquals(checksum, obj.checksum());
+ assertTrue(obj.isDir());
+ assertEquals(dirKey, obj.key());
+ assertEquals(obj, storage.objectStatus(dirKey));
+
+ if (storage.bucket().isDirectory()) {
+ assertNotNull(storage.head("a/b/dir"));
+ assertEquals("a/b/dir", storage.objectStatus("a/b/dir").key());
+ } else {
+ assertNull(storage.head("a/b/dir"));
+ assertEquals(dirKey, storage.objectStatus("a/b/dir").key());
+ }
+
+ // test get dir status of prefix
+ String prefix = "a/b/";
+ obj = storage.objectStatus(prefix);
+ assertEquals(prefix, obj.key());
+ assertEquals(Constants.MAGIC_CHECKSUM, obj.checksum());
+ assertTrue(obj.isDir());
+
+ if (storage.bucket().isDirectory()) {
+ assertEquals(obj, storage.head(prefix));
+ assertEquals("a/b", storage.objectStatus("a/b").key());
+ } else {
+ assertNull(storage.head(prefix));
+ assertEquals(prefix, storage.objectStatus("a/b").key());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutAndGetDirectory(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String key = "a/b/";
+ byte[] data = new byte[0];
+ storage.put(key, data);
+
+ ObjectInfo info = storage.head(key);
+ assertEquals(key, info.key());
+ assertEquals(data.length, info.size());
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key, 0, 256)));
+
+ // test create the same dir again
+ storage.put(key, data);
+
+ storage.delete(key);
+ assertNull(storage.head(key));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testOverwriteFile(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String key = "a/test.txt";
+ byte[] data1 = TestUtility.rand(256);
+ byte[] data2 = TestUtility.rand(128);
+
+ storage.put(key, data1);
+ assertArrayEquals(data1, IOUtils.toByteArray(getStream(key, 0, -1)));
+
+ storage.put(key, data2);
+ assertArrayEquals(data2, IOUtils.toByteArray(getStream(key, 0, -1)));
+
+ storage.delete(key);
+ assertNull(storage.head(key));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectsWithEmptyDelimiters(ObjectStorage store) {
+ setEnv(store);
+ // Directory bucket only supports list with delimiter = '/' currently.
+ assumeFalse(storage.bucket().isDirectory());
+ String key1 = "a/b/c/d";
+ String key2 = "a/b";
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%s/file-%d.txt", key1, i), data);
+ storage.put(String.format("%s/file-%d.txt", key2, i), data);
+ }
+
+ // list 100 objects under 'a/', there are total 20 objects.
+ ListObjectsResponse response = list("a/", "", 100, "");
+ assertEquals(20, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(19).key());
+
+ // list 20 objects and there only have 20 objects under 'a/'
+ response = list("a/", "", 20, "");
+ assertEquals(20, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(19).key());
+
+ // list the top 10 objects among 20 objects
+ response = list("a/", "", 10, "");
+ assertEquals(10, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/c/d/file-9.txt", response.objects().get(9).key());
+
+ // list the next 5 objects behind a/b/c/d/file-9.txt among 20 objects
+ response = list("a/", "a/b/c/d/file-9.txt", 5, "");
+ assertEquals(5, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-4.txt", response.objects().get(4).key());
+
+ // list the next 10 objects behind a/b/c/d/file-9.txt among 20 objects
+ response = list("a/", "a/b/c/d/file-9.txt", 10, "");
+ assertEquals(10, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(9).key());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListEmptyDirWithSlashDelimiter(ObjectStorage store) {
+ setEnv(store);
+ String key = "a/b/";
+ storage.put(key, new byte[0]);
+
+ ListObjectsResponse response = list(key, null, 10, "/");
+ assertEquals(1, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/", response.objects().get(0).key());
+
+ response = list(key, key, 10, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDeleteMultipleKeys(ObjectStorage store) {
+ setEnv(store);
+ String prefix = "a/b";
+ byte[] data = TestUtility.rand(256);
+
+ List<String> keys = Lists.newArrayList();
+ for (int i = 0; i < 50; i++) {
+ String existingKey = String.format("%s/existing-file-%d.txt", prefix, i);
+ storage.put(existingKey, data);
+ keys.add(existingKey);
+
+ String unExistingKey = String.format("%s/unExisting-file-%d.txt", prefix, i);
+ keys.add(unExistingKey);
+ }
+
+ List<String> failedKeys = storage.batchDelete(keys);
+
+ for (String key : failedKeys) {
+ assertNotNull(storage.head(key));
+ }
+
+ for (String key : keys) {
+ if (!failedKeys.contains(key)) {
+ assertNull(storage.head(key));
+ }
+ }
+
+ assertThrows(IllegalArgumentException.class, () -> storage.batchDelete(
+ IntStream.range(0, 1001).mapToObj(String::valueOf).collect(Collectors.toList())),
+ "The deleted keys size should be <= 1000");
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectsWithEmptyMarkers(ObjectStorage store) {
+ setEnv(store);
+ String key1 = "a/b/c/d";
+ String key2 = "a/b";
+ String key3 = "a1/b1";
+
+ // create the folder to compatible with directory bucket.
+ storage.put("a/", new byte[0]);
+ storage.put("a/b/", new byte[0]);
+ storage.put("a/b/c/", new byte[0]);
+ storage.put("a/b/c/d/", new byte[0]);
+ storage.put("a1/", new byte[0]);
+ storage.put("a1/b1/", new byte[0]);
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%s/file-%d.txt", key1, i), data);
+ storage.put(String.format("%s/file-%d.txt", key2, i), data);
+ storage.put(String.format("%s/file-%d.txt", key3, i), data);
+ }
+
+ // group objects by '/' under 'a/'
+ ListObjectsResponse response = list("a/", null, 100, "/");
+ assertEquals(1, response.objects().size());
+ assertEquals("a/", response.objects().get(0).key());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+
+ response = list("a", null, 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(2, response.commonPrefixes().size());
+ assertEquals("a/", response.commonPrefixes().get(0));
+ assertEquals("a1/", response.commonPrefixes().get(1));
+
+ // group objects by '/' under 'a/b/' and group objects by 'b/' under 'a', they are same
+ response = list("a/b/", null, 100, "/");
+ assertEquals(11, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/", response.objects().get(0).key());
+ assertEquals("a/b/file-0.txt", response.objects().get(1).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(10).key());
+
+ response = list("a/b", null, 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+
+ if (!storage.bucket().isDirectory()) {
+ // Directory bucket only supports list with delimiter = '/' currently.
+ response = list("a", null, 100, "b/");
+ assertEquals(13, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a/", response.objects().get(0).key());
+ assertEquals("a1/", response.objects().get(1).key());
+ assertEquals("a1/b1/", response.objects().get(2).key());
+ assertEquals("a1/b1/file-0.txt", response.objects().get(3).key());
+ assertEquals("a1/b1/file-9.txt", response.objects().get(12).key());
+
+ response = list("a/", null, 100, "b/");
+ assertEquals(1, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a/", response.objects().get(0).key());
+ }
+
+ // group objects by different delimiter under 'a/b/c/d/' or 'a/b/c/d'
+ response = list("a/b/c/d/", null, 100, "/");
+ assertEquals(11, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/", response.objects().get(0).key());
+
+ response = list("a/b/c/d/", null, 5, "/");
+ assertEquals(5, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/", response.objects().get(0).key());
+
+ response = list("a/b/c/d", null, 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/", response.commonPrefixes().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectWithLimitObjectAndCommonPrefixes(ObjectStorage store) {
+ setEnv(store);
+ String key1 = "a/b/c/d";
+ String key2 = "a/b";
+ String key3 = "a1/b1";
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%s/file-%d.txt", key1, i), data);
+ storage.put(String.format("%s/file-%d.txt", key2, i), data);
+ storage.put(String.format("%s/file-%d.txt", key3, i), data);
+ }
+
+ List<String> dirKeys = Lists.newArrayList("a/b/d/", "a/b/e/", "a/b/f/", "a/b/g/");
+ for (String key : dirKeys) {
+ storage.put(key, new byte[0]);
+ }
+
+ // group objects by '/' under 'a/b/', and limit top 5 objects among 10 objects and 1 common
+ // prefix.
+ ListObjectsResponse response = list("a/b/", "a/b/", 5, "/");
+ assertEquals(1, response.objects().size());
+ assertEquals(4, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ assertEquals("a/b/e/", response.commonPrefixes().get(2));
+ assertEquals("a/b/f/", response.commonPrefixes().get(3));
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+
+ response = list("a/b/", "a/b/", 14, "/");
+ assertEquals(10, response.objects().size());
+ assertEquals(4, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ assertEquals("a/b/e/", response.commonPrefixes().get(2));
+ assertEquals("a/b/f/", response.commonPrefixes().get(3));
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(9).key());
+
+ response = list("a/b/", "a/b/", 15, "/");
+ assertEquals(10, response.objects().size());
+ assertEquals(5, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ assertEquals("a/b/e/", response.commonPrefixes().get(2));
+ assertEquals("a/b/f/", response.commonPrefixes().get(3));
+ assertEquals("a/b/g/", response.commonPrefixes().get(4));
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(9).key());
+
+ // a/b/h-file-0.txt is behind from a/b/g/
+ storage.put("a/b/h-file-0.txt", data);
+ response = list("a/b/", "a/b/", 15, "/");
+ assertEquals(10, response.objects().size());
+ assertEquals(5, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ assertEquals("a/b/e/", response.commonPrefixes().get(2));
+ assertEquals("a/b/f/", response.commonPrefixes().get(3));
+ assertEquals("a/b/g/", response.commonPrefixes().get(4));
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/file-9.txt", response.objects().get(9).key());
+
+ response = list("a/b/", "a/b/", 20, "/");
+ assertEquals(11, response.objects().size());
+ assertEquals(5, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ assertEquals("a/b/e/", response.commonPrefixes().get(2));
+ assertEquals("a/b/f/", response.commonPrefixes().get(3));
+ assertEquals("a/b/g/", response.commonPrefixes().get(4));
+ assertEquals("a/b/file-0.txt", response.objects().get(0).key());
+ assertEquals("a/b/h-file-0.txt", response.objects().get(10).key());
+
+ response = list("a/b/", "a/b/", 1, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+
+ response = list("a/b/", "a/b/", 2, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(2, response.commonPrefixes().size());
+ assertEquals("a/b/c/", response.commonPrefixes().get(0));
+ assertEquals("a/b/d/", response.commonPrefixes().get(1));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListedIteratorIsIdempotent(ObjectStorage store) {
+ setEnv(store);
+ String key1 = "a/b/c/d";
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%s/file-%d.txt", key1, i), data);
+ }
+
+ Iterable<ObjectInfo> res;
+ if (storage.bucket().isDirectory()) {
+ res = ((DirectoryStorage) storage).listDir("a/b/c/d/", true);
+ } else {
+ res = storage.list("a/b/c/d/", "a/b/c/d/", 10);
+ }
+ Iterator<ObjectInfo> batch1 = res.iterator();
+ Iterator<ObjectInfo> batch2 = res.iterator();
+
+ for (int i = 0; i < 10; i++) {
+ assertTrue(batch1.hasNext());
+ ObjectInfo obj = batch1.next();
+ assertEquals(String.format("a/b/c/d/file-%d.txt", i), obj.key());
+ }
+ assertFalse(batch1.hasNext());
+
+ for (int i = 0; i < 10; i++) {
+ assertTrue(batch2.hasNext());
+ ObjectInfo obj = batch2.next();
+ assertEquals(String.format("a/b/c/d/file-%d.txt", i), obj.key());
+ }
+ assertFalse(batch2.hasNext());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectsWithSmallBatch(ObjectStorage store) {
+ setEnv(store);
+ assumeFalse(storage.bucket().isDirectory());
+ String key1 = "a/b/c/d/";
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%sfile-%d.txt", key1, i), data);
+ }
+
+ // change list object count
+ Configuration newConf = new Configuration(storage.conf());
+ newConf.setInt(TosKeys.FS_TOS_LIST_OBJECTS_COUNT, 5);
+ storage.initialize(newConf, storage.bucket().name());
+
+ List<Integer> maxKeys = Arrays.asList(5, 10, 9, 20, -1);
+ for (int maxKey : maxKeys) {
+ Iterator<ObjectInfo> objs = storage.list(key1, key1, maxKey).iterator();
+ int end = Math.min(maxKey == -1 ? 10 : maxKey, 10);
+ for (int i = 0; i < end; i++) {
+ assertTrue(objs.hasNext());
+ ObjectInfo obj = objs.next();
+ assertEquals(String.format("a/b/c/d/file-%d.txt", i), obj.key());
+ }
+ assertFalse(objs.hasNext());
+ }
+
+ // reset list object count
+ newConf = new Configuration(storage.conf());
+ newConf.setInt(TosKeys.FS_TOS_LIST_OBJECTS_COUNT, 1000);
+ storage.initialize(newConf, storage.bucket().name());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectsWithSpecificDelimiters(ObjectStorage store) {
+ setEnv(store);
+ assumeFalse(storage.bucket().isDirectory());
+ String key1 = "a/b/c/d";
+ String key2 = "a/b";
+ String key3 = "a1/b1";
+
+ byte[] data = TestUtility.rand(256);
+ for (int i = 0; i < 10; i++) {
+ storage.put(String.format("%s/file-%d.txt", key1, i), data);
+ storage.put(String.format("%s/file-%d.txt", key2, i), data);
+ storage.put(String.format("%s/file-%d.txt", key3, i), data);
+ }
+
+ ListObjectsResponse response = list("a", "", 11, "b/");
+ assertEquals(10, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a1/b1/file-0.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-9.txt", response.objects().get(9).key());
+
+ response = list("a", "", 5, "b/");
+ assertEquals(4, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a1/b1/file-0.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-3.txt", response.objects().get(3).key());
+
+ response = list("a", "a1/b1/file-3.txt", 5, "b/");
+ assertEquals(5, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a1/b1/file-4.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-8.txt", response.objects().get(4).key());
+
+ response = list("a", "a1/b1/file-3.txt", 6, "b/");
+ assertEquals(6, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ assertEquals("a1/b1/file-4.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-9.txt", response.objects().get(5).key());
+
+ response = list("a", "a/b/file-3.txt", 5, "b/");
+ assertEquals(4, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a1/b1/file-0.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-3.txt", response.objects().get(3).key());
+
+ response = list("a", "a/b/file-3.txt", 10, "b/");
+ assertEquals(9, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a1/b1/file-0.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-8.txt", response.objects().get(8).key());
+
+ response = list("a", "a/b/file-3.txt", 11, "b/");
+ assertEquals(10, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/", response.commonPrefixes().get(0));
+ assertEquals("a1/b1/file-0.txt", response.objects().get(0).key());
+ assertEquals("a1/b1/file-9.txt", response.objects().get(9).key());
+
+ response = list("a", "a/b/", 1, "b/");
+ assertEquals(1, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+
+ response = list("a/b/c/d", "", 100, "/file");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/file", response.commonPrefixes().get(0));
+
+ response = list("a/b/c/d/", "", 100, "file");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a/b/c/d/file", response.commonPrefixes().get(0));
+
+
+ // group objects by different delimiter under 'a1' or 'a1/'
+ response = list("a1", "", 100, "");
+ assertEquals(10, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+
+ response = list("a1", "", 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a1/", response.commonPrefixes().get(0));
+
+ response = list("a1/", "", 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a1/b1/", response.commonPrefixes().get(0));
+
+ response = list("a1/", "", 1, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(1, response.commonPrefixes().size());
+ assertEquals("a1/b1/", response.commonPrefixes().get(0));
+
+ // group objects by non-exist delimiter under 'a1' or 'a1/'
+ response = list("a1", "", 100, "non-exist");
+ assertEquals(10, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+
+ response = list("a1/", "", 100, "non-exist");
+ assertEquals(10, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+
+ // test the sequent of common prefixes
+ response = list("a", "", 100, "b");
+ assertEquals(0, response.objects().size());
+ assertEquals(2, response.commonPrefixes().size());
+ assertEquals("a/b", response.commonPrefixes().get(0));
+ assertEquals("a1/b", response.commonPrefixes().get(1));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testOverwriteDirectoryWithAFile(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String dirKey = "a/b/";
+ String key = "a/b";
+ storage.delete("a/");
+
+ byte[] data1 = new byte[0];
+ byte[] data2 = TestUtility.rand(128);
+
+ storage.put(dirKey, data1);
+ assertArrayEquals(data1, IOUtils.toByteArray(getStream(dirKey, 0, 256)));
+
+ if (!storage.bucket().isDirectory()) {
+ // Directory bucket doesn't allow overwrote if the resource type is changed.
+ storage.put(key, data2);
+ assertArrayEquals(data2, IOUtils.toByteArray(getStream(key, 0, 256)));
+ }
+
+ storage.delete(key);
+ storage.delete(dirKey);
+ assertNull(storage.head(key));
+ assertNull(storage.head(dirKey));
+ }
+
+ private InputStream getStream(String key, long off, long limit) {
+ return storage.get(key, off, limit).stream();
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDeleteNonEmptyDir(ObjectStorage store) throws IOException {
+ setEnv(store);
+ storage.put("a/", new byte[0]);
+ storage.put("a/b/", new byte[0]);
+ assertArrayEquals(new byte[0], IOUtils.toByteArray(getStream("a/b/", 0, 256)));
+
+ ListObjectsResponse response = list("a/b/", "a/b/", 100, "/");
+ assertEquals(0, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+
+ if (!storage.bucket().isDirectory()) {
+ // Directory bucket only supports list with delimiter = '/'.
+ response = list("a/b/", "a/b/", 100, null);
+ assertEquals(0, response.objects().size());
+ assertEquals(0, response.commonPrefixes().size());
+ }
+
+ storage.delete("a/b/");
+ assertNull(storage.head("a/b/"));
+ assertNull(storage.head("a/b"));
+ assertNotNull(storage.head("a/"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRecursiveDelete(ObjectStorage store) {
+ setEnv(store);
+ storage.put("a/", new byte[0]);
+ storage.put("a/b/", new byte[0]);
+ storage.put("a/b/c1/", new byte[0]);
+ storage.put("a/b/c2/", new byte[0]);
+ storage.put("a/b/c3/", new byte[0]);
+ assertNotNull(storage.head("a/"));
+ assertNotNull(storage.head("a/b/"));
+ assertNotNull(storage.head("a/b/c1/"));
+ assertNotNull(storage.head("a/b/c2/"));
+ assertNotNull(storage.head("a/b/c3/"));
+
+ storage.delete("a/b/c3/");
+ assertNull(storage.head("a/b/c3/"));
+
+ storage.deleteAll("");
+ assertNull(storage.head("a/b/c1/"));
+ assertNull(storage.head("a/b/c2/"));
+ assertNull(storage.head("a/b/"));
+ assertNull(storage.head("a/"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjectKeys(ObjectStorage store) {
+ setEnv(store);
+ assumeFalse(storage.bucket().isDirectory());
+ byte[] dirBytes = new byte[0];
+ byte[] fileBytes = TestUtility.rand(128);
+ storage.put("a/b1/", dirBytes);
+ storage.put("a/b2/c0/", dirBytes);
+ storage.put("a/b2/c1/d1.txt", fileBytes);
+ storage.put("a/b2/c1/e1.txt", fileBytes);
+ storage.put("a/b2/c2.txt", fileBytes);
+
+ // list single dir
+ List<ObjectInfo> ret = toList(storage.list("a/b1", "", 10));
+ assertEquals(1, ret.size());
+ assertEquals("a/b1/", ret.get(0).key());
+ assertEquals(0, ret.get(0).size());
+
+ ret = toList(storage.list("a/b1/", "", 10));
+ assertEquals(1, ret.size());
+ assertEquals("a/b1/", ret.get(0).key());
+ assertEquals(0, ret.get(0).size());
+
+ // list single file
+ ret = toList(storage.list("a/b2/c1/d1.txt", "", 10));
+ assertEquals(1, ret.size());
+ assertEquals("a/b2/c1/d1.txt", ret.get(0).key());
+ assertEquals(fileBytes.length, ret.get(0).size());
+
+ // list multiple files & dirs
+ ret = toList(storage.list("a/b2", "", 10));
+ assertEquals(4, ret.size());
+ assertEquals("a/b2/c0/", ret.get(0).key());
+ assertEquals("a/b2/c1/d1.txt", ret.get(1).key());
+ assertEquals("a/b2/c1/e1.txt", ret.get(2).key());
+ assertEquals("a/b2/c2.txt", ret.get(3).key());
+ assertEquals(dirBytes.length, ret.get(0).size());
+
+ // list single file with marker
+ ret = toList(storage.list("a/b2", "a/b2/c1/e1.txt", 10));
+ assertEquals(1, ret.size());
+ assertEquals("a/b2/c2.txt", ret.get(0).key());
+ assertEquals(fileBytes.length, ret.get(0).size());
+
+ // list multiple files with marker
+ ret = toList(storage.list("a/b2", "a/b2/c1/", 10));
+ assertEquals(3, ret.size());
+ assertEquals("a/b2/c1/d1.txt", ret.get(0).key());
+ assertEquals("a/b2/c1/e1.txt", ret.get(1).key());
+ assertEquals("a/b2/c2.txt", ret.get(2).key());
+ assertEquals(fileBytes.length, ret.get(0).size());
+
+ // list multiple files & dirs with part path as prefix
+ ret = toList(storage.list("a/b2/c", "", 10));
+ assertEquals(4, ret.size());
+ assertEquals("a/b2/c0/", ret.get(0).key());
+ assertEquals("a/b2/c1/d1.txt", ret.get(1).key());
+ assertEquals("a/b2/c1/e1.txt", ret.get(2).key());
+ assertEquals("a/b2/c2.txt", ret.get(3).key());
+ assertEquals(dirBytes.length, ret.get(0).size());
+
+ ret = toList(storage.list("a/b2/c", "", 2));
+ assertEquals(2, ret.size());
+ assertEquals("a/b2/c0/", ret.get(0).key());
+
+ ret = toList(storage.list("a/b2/c1/d1.", "", 10));
+ assertEquals(1, ret.size());
+ assertEquals("a/b2/c1/d1.txt", ret.get(0).key());
+ assertEquals(fileBytes.length, ret.get(0).size());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListAllObjectKeys(ObjectStorage store) {
+ setEnv(store);
+ assumeFalse(storage.bucket().isDirectory());
+ byte[] dirBytes = new byte[0];
+ byte[] fileBytes = TestUtility.rand(128);
+ storage.put("a/b1/", dirBytes);
+ storage.put("a/b2/c0/", dirBytes);
+ storage.put("a/b2/c1/d1.txt", fileBytes);
+ storage.put("a/b2/c1/e1.txt", fileBytes);
+ storage.put("a/b2/c2.txt", dirBytes);
+
+ // list single dir
+ List<ObjectInfo> ret = Lists.newArrayList(storage.listAll("a/b1", ""));
+ assertEquals(1, ret.size());
+ assertEquals("a/b1/", ret.get(0).key());
+ assertEquals(0, ret.get(0).size());
+
+ // list single file
+ ret = Lists.newArrayList(storage.listAll("a/b2/c1/d1.txt", ""));
+ assertEquals(1, ret.size());
+ assertEquals("a/b2/c1/d1.txt", ret.get(0).key());
+ assertEquals(fileBytes.length, ret.get(0).size());
+
+ // list multiple files & dirs
+ ret = Lists.newArrayList(storage.listAll("a/b2", ""));
+ assertEquals(4, ret.size());
+ assertEquals("a/b2/c0/", ret.get(0).key());
+ assertEquals("a/b2/c1/d1.txt", ret.get(1).key());
+ assertEquals("a/b2/c1/e1.txt", ret.get(2).key());
+ assertEquals("a/b2/c2.txt", ret.get(3).key());
+ assertEquals(dirBytes.length, ret.get(0).size());
+
+ // list multiple files & dirs with part path as prefix
+ ret = Lists.newArrayList(storage.listAll("a/b2/c", ""));
+ assertEquals(4, ret.size());
+ assertEquals("a/b2/c0/", ret.get(0).key());
+ assertEquals("a/b2/c1/d1.txt", ret.get(1).key());
+ assertEquals("a/b2/c1/e1.txt", ret.get(2).key());
+ assertEquals("a/b2/c2.txt", ret.get(3).key());
+ assertEquals(dirBytes.length, ret.get(0).size());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListEmptyKeys(ObjectStorage store) {
+ setEnv(store);
+ if (storage.bucket().isDirectory()) {
+ assertEquals(0,
+ Lists.newArrayList(((DirectoryStorage) storage).listDir("not-exist", true)).size());
+ } else {
+ assertEquals(0, Lists.newArrayList(storage.list("not-exist", "", 2)).size());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testMultiUploadEmptyFile(ObjectStorage store) {
+ setEnv(store);
+ String key = "a/b/empty.txt";
+ MultipartUpload upload = storage.createMultipartUpload(key);
+ assertThrows(Exception.class,
+ () -> storage.completeUpload(key, upload.uploadId(), Lists.newArrayList()));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testMultiUploadZeroByte(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String key = "a/b/zero.txt";
+ MultipartUpload upload = storage.createMultipartUpload(key);
+ Part part =
+ storage.uploadPart(key, upload.uploadId(), 1, () -> new ByteArrayInputStream(new byte[0]),
+ 0);
+ storage.completeUpload(key, upload.uploadId(), Lists.newArrayList(part));
+ assertArrayEquals(ObjectTestUtils.EMPTY_BYTES, IOUtils.toByteArray(getStream(key)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testMultiUploadFile(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String key1 = "a/b/c/e.txt";
+ String uploadId1 = storage.createMultipartUpload(key1).uploadId();
+ assertNotEquals(uploadId1, "");
+
+ byte[] dataset = multipleUpload(key1, uploadId1, 2, true);
+ assertArrayEquals(dataset, IOUtils.toByteArray(getStream(key1)));
+
+ String key2 = "a/b/e/e.txt";
+ String uploadId2 = storage.createMultipartUpload(key2).uploadId();
+ assertNotEquals(uploadId2, "");
+
+ dataset = multipleUpload(key2, uploadId2, 3, true);
+ assertArrayEquals(dataset, IOUtils.toByteArray(getStream(key2)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutAndCompleteMPUWithSameContent(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String mpu = "a/b/mpu.txt";
+ String put = "a/b/put.txt";
+ byte[] dataset = TestUtility.rand(11 << 20);
+ byte[] checksum = multipleUpload(mpu, dataset);
+
+ storage.put(put, dataset);
+
+ ObjectInfo mputObj = storage.head(mpu);
+ ObjectInfo putObj = storage.head(put);
+ assertArrayEquals(checksum, mputObj.checksum());
+ assertArrayEquals(checksum, putObj.checksum());
+
+ if (!storage.bucket().isDirectory()) {
+ List<ObjectInfo> objectInfo = toList(storage.list(mpu, null, 10));
+ assertEquals(mputObj, objectInfo.get(0));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListUploads(ObjectStorage store) {
+ setEnv(store);
+ String key1 = "a/b/c/e.txt";
+ String uploadId1 = storage.createMultipartUpload(key1).uploadId();
+ assertNotEquals(uploadId1, "");
+ multipleUpload(key1, uploadId1, 2, false);
+
+ String key2 = "a/b/e/e.txt";
+ String uploadId2 = storage.createMultipartUpload(key2).uploadId();
+ assertNotEquals(uploadId2, "");
+ multipleUpload(key2, uploadId2, 3, false);
+
+ Iterable<MultipartUpload> iterable = storage.listUploads("");
+ List<MultipartUpload> uploads = Lists.newArrayList(iterable.iterator());
+ assertEquals(2, uploads.size());
+ assertEquals(key1, uploads.get(0).key());
+ assertEquals(uploadId1, uploads.get(0).uploadId());
+ assertEquals(key2, uploads.get(1).key());
+ assertEquals(uploadId2, uploads.get(1).uploadId());
+
+ // check iterator is idempotent
+ uploads = Lists.newArrayList(iterable.iterator());
+ assertEquals(2, uploads.size());
+ assertEquals(key1, uploads.get(0).key());
+ assertEquals(uploadId1, uploads.get(0).uploadId());
+ assertEquals(key2, uploads.get(1).key());
+ assertEquals(uploadId2, uploads.get(1).uploadId());
+
+ uploads = Lists.newArrayList(storage.listUploads("a/b/"));
+ assertEquals(2, uploads.size());
+ assertEquals(key1, uploads.get(0).key());
+ assertEquals(uploadId1, uploads.get(0).uploadId());
+ assertEquals(key2, uploads.get(1).key());
+ assertEquals(uploadId2, uploads.get(1).uploadId());
+
+ uploads = Lists.newArrayList(storage.listUploads("a/b/c/"));
+ assertEquals(1, uploads.size());
+ assertEquals(key1, uploads.get(0).key());
+ assertEquals(uploadId1, uploads.get(0).uploadId());
+
+ storage.abortMultipartUpload(key1, uploadId1);
+ storage.abortMultipartUpload(key2, uploadId2);
+ assertEquals(0, Lists.newArrayList((storage.listUploads("a/b/"))).size());
+ }
+
+ private byte[] multipleUpload(String key, String uploadId, int partCnt, boolean completeUpload) {
+ int partSize = 5 * 1024 * 1024;
+ byte[] dataset = new byte[partCnt * partSize];
+ byte[] partData = TestUtility.rand(partSize);
+ try {
+ int offset = 0;
+ List<Part> parts = new ArrayList<>();
+ for (int i = 1; i <= partCnt; i++) {
+ Part part = storage.uploadPart(key, uploadId, i, () -> new ByteArrayInputStream(partData),
+ partData.length);
+ parts.add(part);
+ System.arraycopy(partData, 0, dataset, offset, partData.length);
+ offset += partData.length;
+ }
+ if (completeUpload) {
+ storage.completeUpload(key, uploadId, parts);
+ }
+ } catch (RuntimeException e) {
+ storage.abortMultipartUpload(key, uploadId);
+ }
+ return dataset;
+ }
+
+ private byte[] multipleUpload(String key, byte[] dataset) throws IOException {
+ int partSize = 5 * 1024 * 1024;
+ int partCnt = (int) Math.ceil((double) dataset.length / partSize);
+
+ String uploadId = storage.createMultipartUpload(key).uploadId();
+ assertNotEquals(uploadId, "");
+
+ try {
+ List<Part> parts = new ArrayList<>();
+ for (int i = 0; i < partCnt; i++) {
+ int start = i * partSize;
+ int end = Math.min(dataset.length, start + partSize);
+ byte[] partData = Arrays.copyOfRange(dataset, start, end);
+
+ Part part =
+ storage.uploadPart(key, uploadId, i + 1, () -> new ByteArrayInputStream(partData),
+ partData.length);
+
+ assertEquals(DigestUtils.md5Hex(partData), part.eTag().replace("\"", ""));
+ parts.add(part);
+ }
+
+ byte[] checksum = storage.completeUpload(key, uploadId, parts);
+ assertArrayEquals(dataset, IOUtils.toByteArray(getStream(key)));
+
+ return checksum;
+ } catch (IOException | RuntimeException e) {
+ storage.abortMultipartUpload(key, uploadId);
+ throw e;
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testUploadPartCopy10MB(ObjectStorage store) {
+ setEnv(store);
+ String srcKey = "src10MB.txt";
+ String dstKey = "dst10MB.txt";
+ testUploadPartCopy(srcKey, dstKey, 10 << 20); // 10MB
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testUploadPartCopy100MB(ObjectStorage store) {
+ setEnv(store);
+ String srcKey = "src100MB.txt";
+ String dstKey = "dst100MB.txt";
+ testUploadPartCopy(srcKey, dstKey, 100 << 20); // 100MB
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testUploadPartCopy65MB(ObjectStorage store) {
+ setEnv(store);
+ String srcKey = "src65MB.txt";
+ String dstKey = "dst65MB.txt";
+ testUploadPartCopy(srcKey, dstKey, 65 << 20); // 65MB
+ }
+
+ private void testUploadPartCopy(String srcKey, String key, int fileSize) {
+ MultipartUpload srcMultipartUpload = storage.createMultipartUpload(srcKey);
+ long partSize = 5 << 20;
+ int partCnt = (int) (fileSize / partSize + (fileSize % partSize == 0 ? 0 : 1));
+ byte[] data =
+ multipleUpload(srcMultipartUpload.key(), srcMultipartUpload.uploadId(), partCnt, true);
+ MultipartUpload dstMultipartUpload = storage.createMultipartUpload(key);
+ long copyPartRangeStart = 0L;
+ List<Part> results = Lists.newArrayList();
+ try {
+ for (int i = 0; i < partCnt; i++) {
+ Part result = storage.uploadPartCopy(srcKey, key, dstMultipartUpload.uploadId(), i + 1,
+ copyPartRangeStart, Math.min(copyPartRangeStart + partSize, fileSize) - 1);
+ results.add(result);
+ copyPartRangeStart += partSize;
+ }
+ storage.completeUpload(key, dstMultipartUpload.uploadId(), results);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(key)));
+ } catch (Exception e) {
+ storage.abortMultipartUpload(key, dstMultipartUpload.uploadId());
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testCopy0MB(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String srcKey = "src0MB.txt";
+ String dstKey = "dst0MB.txt";
+ testCopy(srcKey, dstKey, 0);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testCopy5MB(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String srcKey = "src5MB.txt";
+ String dstKey = "dst5MB.txt";
+ testCopy(srcKey, dstKey, 5 << 20);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testCopy10MB(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String srcKey = "src10MB.txt";
+ String dstKey = "dst10MB.txt";
+ testCopy(srcKey, dstKey, 10 << 20);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRename(ObjectStorage store) throws IOException {
+ setEnv(store);
+ String srcKey = "src.txt";
+ String dstKey = "dst.txt";
+
+ // Rename source to a un-exist object
+ renameObject(srcKey, dstKey, 256);
+ renameObject(srcKey, dstKey, 0);
+
+ // Overwrite an existing object
+ renameObjectWhenDestExist(srcKey, dstKey, 256, 0);
+ renameObjectWhenDestExist(srcKey, dstKey, 0, 256);
+
+ assertNull(storage.head(srcKey));
+ assertThrows(RuntimeException.class, () -> storage.rename(srcKey, dstKey),
+ "Source key not found");
+
+ assertThrows(RuntimeException.class, () -> renameObject(srcKey, srcKey, 256),
+ "Cannot rename to the same object");
+ }
+
+ private void renameObjectWhenDestExist(String srcKey, String dstKey, int srcSize, int destSize)
+ throws IOException {
+ byte[] dstData = new byte[destSize];
+ storage.put(dstKey, dstData, 0, destSize);
+ assertArrayEquals(dstData, IOUtils.toByteArray(getStream(dstKey)));
+
+ renameObject(srcKey, dstKey, srcSize);
+ }
+
+ private void renameObject(String srcKey, String dstKey, int fileSize) throws IOException {
+ byte[] data = new byte[fileSize];
+ storage.put(srcKey, data, 0, fileSize);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(srcKey)));
+
+ storage.rename(srcKey, dstKey);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(dstKey)));
+ assertNull(storage.head(srcKey));
+
+ storage.delete(dstKey);
+ assertNull(storage.head(dstKey));
+ }
+
+ private void testCopy(String srcKey, String dstKey, int fileSize) throws IOException {
+ byte[] data = new byte[fileSize];
+ storage.put(srcKey, data, 0, fileSize);
+ storage.copy(srcKey, dstKey);
+ assertArrayEquals(data, IOUtils.toByteArray(getStream(dstKey)));
+ }
+
+ private ListObjectsResponse list(String prefix, String startAfter, int limit, String delimiter) {
+ Preconditions.checkArgument(limit <= 1000, "Cannot list more than 1000 objects.");
+ ListObjectsRequest request = ListObjectsRequest.builder()
+ .prefix(prefix)
+ .startAfter(startAfter)
+ .maxKeys(limit)
+ .delimiter(delimiter)
+ .build();
+ Iterator<ListObjectsResponse> iterator = storage.list(request).iterator();
+ if (iterator.hasNext()) {
+ return iterator.next();
+ } else {
+ return new ListObjectsResponse(new ArrayList<>(), new ArrayList<>());
+ }
+ }
+
+ private static <T> List<T> toList(final Iterable<T> iterable) {
+ return StreamSupport.stream(iterable.spliterator(), false)
+ .collect(Collectors.toList());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testObjectTagging(ObjectStorage store) {
+ setEnv(store);
+ assumeFalse(storage.bucket().isDirectory());
+ if (storage instanceof FileStore) {
+ return;
+ }
+
+ // create key.
+ String key = "ObjectTagging";
+ String tagPrefix = "tag" + UUIDUtils.random() + "_";
+ String valuePrefix = "value" + UUIDUtils.random() + "_";
+ storage.put(key, new byte[0], 0, 0);
+
+ Map<String, String> tagsMap = new HashMap<>();
+ for (int i = 0; i < 10; i++) {
+ tagsMap.put(tagPrefix + i, valuePrefix + i);
+ }
+
+ // 1. put and get when key exists.
+ storage.putTags(key, tagsMap);
+ Map<String, String> tags = storage.getTags(key);
+ assertEquals(10, tags.keySet().size());
+ assertTrue(Maps.difference(tagsMap, tags).areEqual());
+
+ // 2. put and get when key doesn't exist.
+ assertThrows(TosServerException.class, () -> storage.putTags("non-exist-key", tagsMap),
+ "NoSuchKey");
+ assertThrows(TosServerException.class, () -> storage.getTags("non-exist-key"), "doesn't exist");
+
+ // 3. tag threshold.
+ Map<String, String> bigMap = new HashMap<>(tagsMap);
+ bigMap.put(tagPrefix + 11, valuePrefix + 11);
+ assertThrows(RuntimeException.class, () -> storage.putTags(key, bigMap), "exceed limit of 10");
+
+ // 4. put tag with null tagName.
+ Map<String, String> nullKeyTag = new HashMap<>();
+ nullKeyTag.put(null, "some value");
+ assertThrows(TosServerException.class, () -> storage.putTags(key, nullKeyTag),
+ "TagKey you have provided is invalid");
+
+ // 5. put tag with null value.
+ Map<String, String> nullValueTag = new HashMap<>();
+ nullValueTag.put("some-key", null);
+ storage.putTags(key, nullValueTag);
+ assertNull(storage.getTags(key).get("some-key"));
+
+ // 6. remove tags.
+ Map<String, String> emptyTag = new HashMap<>();
+ storage.putTags(key, emptyTag);
+ assertEquals(0, storage.getTags(key).size());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testObjectChecksum(ObjectStorage store) throws IOException {
+ setEnv(store);
+ byte[] data = TestUtility.rand(256);
+ String key = "a/truncated.txt";
+
+ // Read object at the end offset.
+ byte[] checksum = storage.put(key, () -> new ByteArrayInputStream(data), 200);
+ ObjectContent objContent = storage.get(key, 200, -1);
+ objContent.stream().close();
+ assertArrayEquals(checksum, objContent.checksum());
+
+ // Read empty object.
+ checksum = storage.put(key, () -> new ByteArrayInputStream(new byte[0]), 0);
+ objContent = storage.get(key, 0, -1);
+ objContent.stream().close();
+ assertArrayEquals(checksum, objContent.checksum());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestChainTOSInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestChainTOSInputStream.java
new file mode 100644
index 00000000000..26533d74c7d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestChainTOSInputStream.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import com.volcengine.tos.model.object.GetObjectBasicOutput;
+import com.volcengine.tos.model.object.GetObjectV2Output;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestChainTOSInputStream {
+
+ private static final int DATA_SIZE = 1 << 20;
+ private static final byte[] DATA = TestUtility.rand(DATA_SIZE);
+
+ @Test
+ public void testRetryReadData() throws IOException {
+ int readLen = DATA_SIZE - 1;
+ int cutOff = readLen / 2;
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // The read length is more than the cut-off position, and equal to data length,
+ // so the first stream will throw IOException, and fallback to the second stream.
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, 0, readLen), data);
+ }
+
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // The read length is more than data length, so the first stream will throw IOException,
+ // and fallback to the second stream.
+ byte[] data = new byte[readLen + 2];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, 0, readLen), Bytes.toBytes(data, 0, n));
+ }
+
+ readLen = DATA_SIZE / 3;
+ cutOff = DATA_SIZE / 2;
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE, 1024,
+ cutOff)) {
+ for (int i = 0; i <= 3; i++) {
+ // The cut-off position is between (readLen, 2 * readLen), so the data of first read come
+ // from the first stream, and then the second read will meet IOException, and fallback to
+ // the second stream.
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+
+ int off = i * readLen;
+ int len = Math.min(readLen, DATA_SIZE - off);
+
+ assertEquals(len, n);
+ assertArrayEquals(Bytes.toBytes(DATA, off, len), Bytes.toBytes(data, 0, len));
+ }
+ }
+
+ int smallDataSize = 1 << 10;
+ cutOff = smallDataSize / 2;
+ byte[] smallData = TestUtility.rand(1 << 10);
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(smallData, 0, smallDataSize,
+ 1024, cutOff)) {
+ for (int i = 0; i < smallDataSize; i++) {
+ // The cut-off position is 512, the 512th read operation will meet IOException,
+ // and then fallback to the second stream.
+ int read = stream.read();
+ assertEquals(smallData[i] & 0xFF, read);
+ }
+ }
+ }
+
+ @Test
+ public void testSkipAndRead() throws IOException {
+ int cutOff = (DATA_SIZE - 1) / 2;
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // The skip pos is equal to cut-off pos, once skip finished, the first read operation will
+ // meet IOException, and the fallback to the second stream.
+ int readPos = (DATA_SIZE - 1) / 2;
+ stream.skip(readPos);
+
+ int readLen = 1024;
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, readPos, readLen), data);
+ }
+
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // The skip pos is more than cut-off pos, the skip operation will throw IOException,
+ // and the fallback to the second stream and skip(readPos) again
+ int readPos = cutOff + 1024;
+ stream.skip(readPos);
+
+ int readLen = 1024;
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, readPos, readLen), data);
+ }
+
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // The skip pos = cut-off pos - 1025, the skip operation will succeed on the first stream,
+ // the 1024 bytes read operation also succeed on the first stream,
+ // but the next 1024 bytes read operation will fail on the first stream, and fallback to the
+ // second stream
+ int readPos = cutOff - 1024 - 1;
+ stream.skip(readPos);
+
+ int readLen = 1024;
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, readPos, readLen), data);
+
+ n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, readPos + 1024, readLen), data);
+ }
+
+ try (ChainTOSInputStream stream = createTestChainTOSInputStream(DATA, 0, DATA_SIZE - 1, 1024,
+ cutOff)) {
+ // 1. Skip 1024 bytes and then read 1024 bytes from the first stream.
+ // 2. And then skip cut-off - 512 bytes, the target off = 1024 + 1024 + cut-off - 512,
+ // which is bigger than cut-off pos, so the second skip operation will fail,
+ // and then fallback to the second stream.
+ // 3. Read 1024 bytes
+ int readPos = 1024;
+ stream.skip(readPos);
+
+ int readLen = 1024;
+ byte[] data = new byte[readLen];
+ int n = stream.read(data);
+ assertEquals(readLen, n);
+ assertArrayEquals(Bytes.toBytes(DATA, readPos, readLen), data);
+
+ int skipPos = cutOff - 512;
+ stream.skip(skipPos);
+
+ n = stream.read(data);
+ assertEquals(readLen, n);
+ int targetOff = readPos + 1024 + skipPos;
+ assertArrayEquals(Bytes.toBytes(DATA, targetOff, readLen), data);
+ }
+ }
+
+ /**
+ * The ChainTOSInputStream contains two stream created by TestObjectFactory.
+ * Once the read pos of first stream is more than cutPos, the stream will throw IOException with
+ * unexpect end of stream error msg, but the second stream will contain the remaining data.
+ */
+ private ChainTOSInputStream createTestChainTOSInputStream(byte[] data, long startOff, long endOff,
+ long maxDrainSize, long cutPos) {
+ String key = "dummy-key";
+ TOS.GetObjectFactory factory = new TestObjectFactory(data, Arrays.asList(cutPos, -1L));
+ return new ChainTOSInputStream(factory, key, startOff, endOff, maxDrainSize, 1);
+ }
+
+ private static class TestObjectFactory implements TOS.GetObjectFactory {
+ private final byte[] data;
+ private final List<Long> streamBreakPoses;
+ private int streamIndex = 0;
+
+ TestObjectFactory(byte[] data, List<Long> streamBreakPoses) {
+ this.data = data;
+ this.streamBreakPoses = streamBreakPoses;
+ }
+
+ @Override
+ public GetObjectOutput create(String key, long offset, long end) {
+ long len = Math.min(end, data.length) - offset;
+ ByteArrayInputStream dataIn = new ByteArrayInputStream(this.data, (int) offset, (int) len);
+
+ if (streamIndex < streamBreakPoses.size()) {
+ return new GetObjectOutput(new GetObjectV2Output(new GetObjectBasicOutput(),
+ new UnExpectedEndOfStream(dataIn, streamBreakPoses.get(streamIndex++))),
+ Constants.MAGIC_CHECKSUM);
+ } else {
+ throw new RuntimeException("No more output");
+ }
+ }
+ }
+
+ private static class UnExpectedEndOfStream extends InputStream {
+ private final ByteArrayInputStream delegate;
+ private final long breakPos;
+ private int readPos;
+
+ UnExpectedEndOfStream(ByteArrayInputStream stream, long breakPos) {
+ delegate = stream;
+ this.breakPos = breakPos;
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (breakPos != -1 && readPos >= breakPos) {
+ throw new IOException("unexpected end of stream on dummy source.");
+ } else {
+ int n = delegate.read();
+ readPos += 1;
+ return n;
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ if (breakPos != -1 && readPos >= breakPos) {
+ throw new IOException("unexpected end of stream on dummy source.");
+ } else {
+ int n = delegate.read(b, off, len);
+ readPos += n;
+ return n;
+ }
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestDelegationClientBuilder.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestDelegationClientBuilder.java
new file mode 100644
index 00000000000..fabc26c23c8
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestDelegationClientBuilder.java
@@ -0,0 +1,462 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import com.volcengine.tos.TOSV2;
+import com.volcengine.tos.TOSV2ClientBuilder;
+import com.volcengine.tos.TosClientException;
+import com.volcengine.tos.TosException;
+import com.volcengine.tos.TosServerException;
+import com.volcengine.tos.auth.Credential;
+import com.volcengine.tos.auth.StaticCredentials;
+import com.volcengine.tos.comm.HttpStatus;
+import com.volcengine.tos.model.object.DeleteObjectInput;
+import com.volcengine.tos.model.object.HeadObjectV2Input;
+import com.volcengine.tos.model.object.HeadObjectV2Output;
+import com.volcengine.tos.model.object.ListObjectsV2Input;
+import com.volcengine.tos.model.object.ListObjectsV2Output;
+import com.volcengine.tos.model.object.PutObjectInput;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.common.Tasks;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.tos.auth.EnvironmentCredentialsProvider;
+import org.apache.hadoop.fs.tosfs.object.tos.auth.SimpleCredentialsProvider;
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.TestReporter;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+import javax.net.ssl.SSLException;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.DelegationClient.isRetryableException;
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestDelegationClientBuilder {
+
+ private static final String TEST_KEY = UUIDUtils.random();
+ private static final String TEST_DATA = "1234567890";
+ private static final String ENV_ACCESS_KEY =
+ ParseUtils.envAsString(TOS.ENV_TOS_ACCESS_KEY_ID, false);
+ private static final String ENV_SECRET_KEY =
+ ParseUtils.envAsString(TOS.ENV_TOS_SECRET_ACCESS_KEY, false);
+ private static final String ENV_ENDPOINT = ParseUtils.envAsString(TOS.ENV_TOS_ENDPOINT, false);
+
+ // Maximum retry times of the tos http client.
+ public static final String MAX_RETRY_COUNT_KEY = "fs.tos.http.maxRetryCount";
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @BeforeEach
+ public void setUp() {
+ TOSV2 tosSdkClientV2 =
+ new TOSV2ClientBuilder().build(TestUtility.region(), TestUtility.endpoint(),
+ new StaticCredentials(ENV_ACCESS_KEY, ENV_SECRET_KEY));
+ try (ByteArrayInputStream stream = new ByteArrayInputStream(TEST_DATA.getBytes())) {
+ PutObjectInput putObjectInput =
+ new PutObjectInput().setBucket(TestUtility.bucket()).setKey(TEST_KEY).setContent(stream);
+ tosSdkClientV2.putObject(putObjectInput);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testHeadApiRetry() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME),
+ "https://test.tos-cn-beijing.ivolces.com");
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, false);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key("test"), "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key("test"), "SECRET_KEY");
+
+ DelegationClient tosV2 = new DelegationClientBuilder().bucket("test").conf(conf).build();
+ TOSV2 mockClient = mock(TOSV2.class);
+ tosV2.setClient(mockClient);
+ tosV2.setMaxRetryTimes(5);
+
+ HeadObjectV2Input input = HeadObjectV2Input.builder().bucket("test").build();
+ when(tosV2.headObject(input)).thenThrow(
+ new TosServerException(HttpStatus.INTERNAL_SERVER_ERROR),
+ new TosServerException(HttpStatus.TOO_MANY_REQUESTS),
+ new TosClientException("fake toe", new IOException("fake ioe")),
+ new TosException(new SocketException("fake msg")),
+ new TosException(new UnknownHostException("fake msg")),
+ new TosException(new SSLException("fake msg")),
+ new TosException(new InterruptedException("fake msg")),
+ new TosException(new InterruptedException("fake msg")))
+ .thenReturn(new HeadObjectV2Output());
+
+ RuntimeException exception =
+ assertThrows(RuntimeException.class, () -> tosV2.headObject(input));
+ assertTrue(exception instanceof TosException);
+ assertTrue(exception.getCause() instanceof UnknownHostException);
+ verify(tosV2.client(), times(5)).headObject(input);
+
+ HeadObjectV2Input inputOneTime = HeadObjectV2Input.builder().bucket("inputOneTime").build();
+ HeadObjectV2Output output = new HeadObjectV2Output();
+ when(tosV2.headObject(inputOneTime)).thenReturn(output);
+ HeadObjectV2Output headObject = tosV2.headObject(inputOneTime);
+ assertEquals(headObject, output);
+ verify(tosV2.client(), times(1)).headObject(inputOneTime);
+ tosV2.close();
+
+ DelegationClient newClient = new DelegationClientBuilder().bucket("test").conf(conf).build();
+ mockClient = mock(TOSV2.class);
+ newClient.setClient(mockClient);
+ newClient.setMaxRetryTimes(5);
+ when(newClient.headObject(input)).thenThrow(
+ new TosClientException("fake toe", new EOFException("fake eof")),
+ new TosServerException(HttpStatus.INTERNAL_SERVER_ERROR),
+ new TosServerException(HttpStatus.TOO_MANY_REQUESTS)).thenReturn(new HeadObjectV2Output());
+
+ exception = assertThrows(RuntimeException.class, () -> newClient.headObject(input));
+ assertTrue(exception instanceof TosClientException);
+ assertTrue(exception.getCause() instanceof EOFException);
+ verify(newClient.client(), times(1)).headObject(input);
+ newClient.close();
+ }
+
+ @Test
+ public void testEnableCrcCheck(TestInfo testInfo, TestReporter testReporter) throws IOException {
+ String bucket = testInfo.getTestMethod().map(method -> method.getName()).orElse("Unknown");
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME),
+ "https://test.tos-cn-beijing.ivolces.com");
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, true);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(bucket), "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(bucket), "SECRET_KEY");
+
+ DelegationClient tosV2 = new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ assertTrue(tosV2.config().isEnableCrc());
+
+ conf.setBoolean(TosKeys.FS_TOS_CRC_CHECK_ENABLED, false);
+ tosV2 = new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ assertFalse(tosV2.config().isEnableCrc());
+
+ tosV2.close();
+ }
+
+ @Test
+ public void testClientCache(TestInfo testInfo, TestReporter testReporter) throws IOException {
+ String bucket = testInfo.getTestMethod().map(method -> method.getName()).orElse("Unknown");
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME),
+ "https://test.tos-cn-beijing.ivolces.com");
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, false);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(bucket), "ACCESS_KEY_A");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(bucket), "SECRET_KEY_A");
+
+ DelegationClient tosV2 = new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ DelegationClient tosV2Cached = new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ assertEquals(tosV2Cached, tosV2, "client must be load in cache");
+ assertEquals("ACCESS_KEY_A", tosV2.usedCredential().getAccessKeyId());
+ tosV2Cached.close();
+
+ String newBucket = "new-test-bucket";
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(newBucket), "ACCESS_KEY_B");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(newBucket), "SECRET_KEY_B");
+ DelegationClient changeBucketClient =
+ new DelegationClientBuilder().bucket(newBucket).conf(conf).build();
+ assertNotEquals(changeBucketClient, tosV2, "client should be created entirely new");
+ assertEquals("ACCESS_KEY_B", changeBucketClient.usedCredential().getAccessKeyId());
+ changeBucketClient.close();
+
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, true); // disable cache: true
+ DelegationClient tosV2NotCached =
+ new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ assertNotEquals(tosV2NotCached, tosV2, "client should be created entirely new");
+ assertEquals("ACCESS_KEY_A", tosV2NotCached.usedCredential().getAccessKeyId());
+ tosV2NotCached.close();
+
+ tosV2.close();
+ }
+
+ @Test
+ public void testOverwriteHttpConfig() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME),
+ "https://tos-cn-beijing.ivolces.com");
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key("test"), "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key("test"), "SECRET_KEY");
+ conf.setInt(TosKeys.FS_TOS_HTTP_MAX_CONNECTIONS, 24);
+ conf.setInt(MAX_RETRY_COUNT_KEY, 24);
+ conf.setInt(TosKeys.FS_TOS_REQUEST_MAX_RETRY_TIMES, 24);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, true);
+
+ DelegationClient tosV2 = new DelegationClientBuilder().bucket("test").conf(conf).build();
+ assertEquals("ACCESS_KEY", tosV2.usedCredential().getAccessKeyId());
+ assertEquals(24, tosV2.config().getTransportConfig().getMaxConnections(),
+ "http max connection overwrite to 24 from 1024, must be 24");
+ assertEquals(DelegationClientBuilder.DISABLE_TOS_RETRY_VALUE,
+ tosV2.config().getTransportConfig().getMaxRetryCount(),
+ "tos maxRetryCount disabled, must be -1");
+ assertEquals(24, tosV2.maxRetryTimes(), "maxRetryTimes must be 24");
+ assertEquals("https://tos-cn-beijing.ivolces.com", tosV2.config().getEndpoint(),
+ "endpoint must be equals to https://tos-cn-beijing.ivolces.com");
+
+ tosV2.close();
+ }
+
+ @Test
+ public void testDynamicRefreshAkSk() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME), ENV_ENDPOINT);
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(TestUtility.bucket()), ENV_ACCESS_KEY);
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(TestUtility.bucket()), ENV_SECRET_KEY);
+ conf.setInt(TosKeys.FS_TOS_HTTP_MAX_CONNECTIONS, 24);
+ conf.setInt(MAX_RETRY_COUNT_KEY, 24);
+
+ TOSV2 tosSdkClientV2 =
+ new TOSV2ClientBuilder().build(TestUtility.region(), TestUtility.endpoint(),
+ new StaticCredentials("a", "b"));
+ DelegationClient delegationClientV2 =
+ new DelegationClientBuilder().bucket(TestUtility.bucket()).conf(conf).build();
+
+ ListObjectsV2Input inputV2 =
+ ListObjectsV2Input.builder().bucket(TestUtility.bucket()).prefix(TEST_KEY).marker("")
+ .maxKeys(10).build();
+
+ assertThrows(TosServerException.class, () -> tosSdkClientV2.listObjects(inputV2));
+
+ tosSdkClientV2.changeCredentials(new StaticCredentials(ENV_ACCESS_KEY, ENV_SECRET_KEY));
+
+ ListObjectsV2Output tosSdkOutput = tosSdkClientV2.listObjects(inputV2);
+ ListObjectsV2Output delegateOutput = delegationClientV2.listObjects(inputV2);
+ int nativeContentSize =
+ tosSdkOutput.getContents() == null ? -1 : tosSdkOutput.getContents().size();
+ int delegateContentSize =
+ delegateOutput.getContents() == null ? -1 : delegateOutput.getContents().size();
+
+ assertEquals(nativeContentSize, delegateContentSize,
+ "delegation client must same as native client");
+ assertEquals(ENV_ACCESS_KEY, delegationClientV2.usedCredential().getAccessKeyId());
+
+ delegationClientV2.close();
+ }
+
+ @Test
+ public void testCreateClientWithEnvironmentCredentials() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME), ENV_ENDPOINT);
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, EnvironmentCredentialsProvider.NAME);
+
+ DelegationClient tosV2 =
+ new DelegationClientBuilder().bucket(TestUtility.bucket()).conf(conf).build();
+ Credential cred = tosV2.usedCredential();
+
+ String assertMsg =
+ String.format("expect %s, but got %s", ENV_ACCESS_KEY, cred.getAccessKeyId());
+ assertEquals(cred.getAccessKeyId(), ENV_ACCESS_KEY, assertMsg);
+ assertMsg = String.format("expect %s, but got %s", ENV_SECRET_KEY, cred.getAccessKeySecret());
+ assertEquals(cred.getAccessKeySecret(), ENV_SECRET_KEY, assertMsg);
+
+ tosV2.close();
+ }
+
+ @Test
+ public void testCreateClientWithSimpleCredentials() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME), ENV_ENDPOINT);
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(TestUtility.bucket()), ENV_ACCESS_KEY);
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(TestUtility.bucket()), ENV_SECRET_KEY);
+ conf.setInt(TosKeys.FS_TOS_HTTP_MAX_CONNECTIONS, 24);
+ conf.setInt(MAX_RETRY_COUNT_KEY, 24);
+
+ ListObjectsV2Input input =
+ ListObjectsV2Input.builder().bucket(TestUtility.bucket()).prefix(TEST_KEY).marker("")
+ .maxKeys(10).build();
+
+ TOSV2 v2 = new TOSV2ClientBuilder().build(TestUtility.region(), TestUtility.endpoint(),
+ new StaticCredentials(ENV_ACCESS_KEY, ENV_SECRET_KEY));
+ ListObjectsV2Output outputV2 = v2.listObjects(input);
+
+ DelegationClient tosV2 =
+ new DelegationClientBuilder().bucket(TestUtility.bucket()).conf(conf).build();
+
+ ListObjectsV2Output output = tosV2.listObjects(input);
+ assertEquals(outputV2.getContents().size(), output.getContents().size(),
+ "delegation client must be same as native client");
+
+ tosV2.close();
+ }
+
+ @Test
+ public void testCachedConcurrently(TestInfo testInfo, TestReporter testReporter) {
+ String bucketName = testInfo.getTestMethod().map(method -> method.getName()).orElse("Unknown");
+
+ Function<String, Configuration> commonConf = bucket -> {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME), ENV_ENDPOINT);
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key(bucket), ENV_ACCESS_KEY);
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key(bucket), ENV_SECRET_KEY);
+ return conf;
+ };
+
+ // enable cache
+ Function<String, Configuration> enableCachedConf = bucket -> {
+ Configuration conf = commonConf.apply(bucket);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, false);
+ return conf;
+ };
+
+ ExecutorService es = ThreadPools.newWorkerPool("testCachedConcurrently", 32);
+ int bucketCount = 5;
+ int taskCount = 10000;
+
+ AtomicInteger success = new AtomicInteger(0);
+ AtomicInteger failure = new AtomicInteger(0);
+ Tasks.foreach(IntStream.range(0, taskCount).boxed().map(i -> bucketName + (i % bucketCount)))
+ .executeWith(es).run(bucket -> {
+ try {
+ Configuration conf = enableCachedConf.apply(bucket);
+ DelegationClient client =
+ new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ client.close();
+ success.incrementAndGet();
+ } catch (Exception e) {
+ failure.incrementAndGet();
+ }
+ });
+
+ assertEquals(bucketCount, DelegationClientBuilder.CACHE.size());
+ assertEquals(taskCount, success.get());
+ assertEquals(0, failure.get());
+
+ // clear cache
+ DelegationClientBuilder.CACHE.clear();
+
+ // disable cache
+ Function<String, Configuration> disableCachedConf = bucket -> {
+ Configuration conf = commonConf.apply(bucket);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, true);
+ return conf;
+ };
+
+ success.set(0);
+ failure.set(0);
+ Tasks.foreach(IntStream.range(0, taskCount).boxed().map(i -> bucketName + (i % bucketCount)))
+ .executeWith(es).run(bucket -> {
+ try {
+ Configuration conf = disableCachedConf.apply(bucket);
+ DelegationClient client =
+ new DelegationClientBuilder().bucket(bucket).conf(conf).build();
+ client.close();
+ success.incrementAndGet();
+ } catch (Exception e) {
+ failure.incrementAndGet();
+ }
+ });
+
+ assertTrue(DelegationClientBuilder.CACHE.isEmpty());
+ assertEquals(taskCount, success.get());
+ assertEquals(0, failure.get());
+
+ es.shutdown();
+ }
+
+ @AfterEach
+ public void deleteAllTestData() throws IOException {
+ TOSV2 tosSdkClientV2 =
+ new TOSV2ClientBuilder().build(TestUtility.region(), TestUtility.endpoint(),
+ new StaticCredentials(ENV_ACCESS_KEY, ENV_SECRET_KEY));
+ tosSdkClientV2.deleteObject(
+ DeleteObjectInput.builder().bucket(TestUtility.bucket()).key(TEST_KEY).build());
+
+ tosSdkClientV2.close();
+ DelegationClientBuilder.CACHE.clear();
+ }
+
+ @Test
+ public void testRetryableException() {
+ assertTrue(retryableException(new TosServerException(500)));
+ assertTrue(retryableException(new TosServerException(501)));
+ assertTrue(retryableException(new TosServerException(429)));
+ assertFalse(retryableException(new TosServerException(404)));
+
+ assertTrue(retryableException(new TosException(new SocketException())));
+ assertTrue(retryableException(new TosException(new UnknownHostException())));
+ assertTrue(retryableException(new TosException(new SSLException("fake ssl"))));
+ assertTrue(retryableException(new TosException(new SocketTimeoutException())));
+ assertTrue(retryableException(new TosException(new InterruptedException())));
+
+ assertTrue(retryableException(new TosClientException("fake ioe", new IOException())));
+ assertFalse(retryableException(new TosClientException("fake eof", new EOFException())));
+
+ assertTrue(retryableException(new TosServerException(409)));
+ assertTrue(
+ retryableException(new TosServerException(409).setEc(TOSErrorCodes.PATH_LOCK_CONFLICT)));
+ assertFalse(
+ retryableException(new TosServerException(409).setEc(TOSErrorCodes.DELETE_NON_EMPTY_DIR)));
+ assertFalse(
+ retryableException(new TosServerException(409).setEc(TOSErrorCodes.LOCATED_UNDER_A_FILE)));
+ assertFalse(retryableException(
+ new TosServerException(409).setEc(TOSErrorCodes.COPY_BETWEEN_DIR_AND_FILE)));
+ assertFalse(retryableException(
+ new TosServerException(409).setEc(TOSErrorCodes.RENAME_TO_AN_EXISTED_DIR)));
+ assertFalse(
+ retryableException(new TosServerException(409).setEc(TOSErrorCodes.RENAME_TO_SUB_DIR)));
+ assertFalse(retryableException(
+ new TosServerException(409).setEc(TOSErrorCodes.RENAME_BETWEEN_DIR_AND_FILE)));
+ }
+
+ private boolean retryableException(TosException e) {
+ return isRetryableException(e,
+ Arrays.asList(TOSErrorCodes.FAST_FAILURE_CONFLICT_ERROR_CODES.split(",")));
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSInputStream.java
new file mode 100644
index 00000000000..48449196e47
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSInputStream.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import com.volcengine.tos.internal.util.aborthook.AbortInputStreamHook;
+import com.volcengine.tos.model.object.GetObjectBasicOutput;
+import com.volcengine.tos.model.object.GetObjectV2Output;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.thirdparty.com.google.common.io.ByteStreams;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class TestTOSInputStream {
+
+ private static final int DATA_SIZE = 1 << 20;
+ private static final byte[] DATA = TestUtility.rand(DATA_SIZE);
+
+ @Test
+ public void testForceClose() throws IOException {
+ TOSInputStream stream = createStream(DATA, 0, DATA_SIZE - 1, 1024);
+ stream.close();
+ assertTrue("Expected force close", cast(stream).isForceClose());
+
+ stream = createStream(DATA, 0, DATA_SIZE - 1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1024 - 1);
+ stream.close();
+ assertTrue("Expected force close", cast(stream).isForceClose());
+
+ stream = createStream(DATA, 0, -1, 1024);
+ stream.close();
+ assertTrue("Expected force close", cast(stream).isForceClose());
+
+ stream = createStream(DATA, 0, -1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1024 - 1);
+ stream.close();
+ assertTrue("Expected force close", cast(stream).isForceClose());
+
+ stream = createStream(DATA, 0, -1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1024);
+ stream.close();
+ assertTrue("Expected force close", cast(stream).isForceClose());
+ }
+
+ @Test
+ public void testClose() throws IOException {
+ TOSInputStream stream = createStream(DATA, 0, DATA_SIZE - 1, DATA_SIZE);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+
+ stream = createStream(DATA, 0, DATA_SIZE - 1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1024);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+
+ stream = createStream(DATA, 0, DATA_SIZE - 1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1023);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+
+ stream = createStream(DATA, 0, -1, DATA_SIZE + 1);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+
+ stream = createStream(DATA, 0, -1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE - 1023);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+
+ stream = createStream(DATA, 0, -1, 1024);
+ ByteStreams.skipFully(stream, DATA_SIZE);
+ stream.close();
+ assertFalse(cast(stream).isForceClose(), "Expected close by skipping bytes");
+ }
+
+ private TestInputStream cast(TOSInputStream stream) throws IOException {
+ InputStream content = stream.getObjectOutput().verifiedContent(Constants.MAGIC_CHECKSUM);
+ assertTrue("Not a TestInputStream", content instanceof TestInputStream);
+ return (TestInputStream) content;
+ }
+
+ private TOSInputStream createStream(byte[] data, long startOff, long endOff, long maxDrainSize)
+ throws IOException {
+ TestInputStream stream =
+ new TestInputStream(data, (int) startOff, (int) (data.length - startOff));
+ GetObjectV2Output output = new GetObjectV2Output(new GetObjectBasicOutput(), stream).setHook(
+ new ForceCloseHook(stream));
+
+ return new TOSInputStream(new GetObjectOutput(output, Constants.MAGIC_CHECKSUM), startOff,
+ endOff, maxDrainSize, Constants.MAGIC_CHECKSUM);
+ }
+
+ private final static class TestInputStream extends ByteArrayInputStream {
+ // -1 means call close()
+ // 0 means neither call close() nor forceClose()
+ // 1 means call forceClose()
+ private int cloeState = 0;
+
+ private TestInputStream(byte[] buf, int off, int len) {
+ super(buf, off, len);
+ }
+
+ @Override
+ public void close() {
+ cloeState = -1;
+ }
+
+ public void forceClose() {
+ cloeState = 1;
+ }
+
+ boolean isForceClose() {
+ assertTrue("Neither call close() nor forceClose()", cloeState == -1 || cloeState == 1);
+ return cloeState == 1;
+ }
+ }
+
+ private final static class ForceCloseHook implements AbortInputStreamHook {
+ private final TestInputStream in;
+
+ private ForceCloseHook(TestInputStream in) {
+ this.in = in;
+ }
+
+ @Override public void abort() {
+ if (in != null) {
+ in.forceClose();
+ }
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSObjectStorage.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSObjectStorage.java
new file mode 100644
index 00000000000..282d39e204d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSObjectStorage.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import com.volcengine.tos.internal.model.CRC64Checksum;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.ChecksumType;
+import org.apache.hadoop.fs.tosfs.object.Constants;
+import org.apache.hadoop.fs.tosfs.object.MultipartUpload;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.object.exceptions.NotAppendableException;
+import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
+import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.apache.hadoop.util.PureJavaCrc32C;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Stream;
+import java.util.zip.Checksum;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestTOSObjectStorage {
+ public static Stream<Arguments> provideArguments() {
+ assumeTrue(TestEnv.checkTestEnabled());
+
+ List<Arguments> values = new ArrayList<>();
+
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_CHECKSUM_TYPE, ChecksumType.CRC64ECMA.name());
+ values.add(Arguments.of(
+ ObjectStorageFactory.createWithPrefix(String.format("tos-%s/", UUIDUtils.random()),
+ TOS_SCHEME, TestUtility.bucket(), conf),
+ new CRC64Checksum(),
+ ChecksumType.CRC64ECMA));
+
+ conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_CHECKSUM_TYPE, ChecksumType.CRC32C.name());
+ values.add(Arguments.of(
+ ObjectStorageFactory.createWithPrefix(String.format("tos-%s/", UUIDUtils.random()),
+ TOS_SCHEME, TestUtility.bucket(), conf),
+ new PureJavaCrc32C(),
+ ChecksumType.CRC32C));
+
+ return values.stream();
+ }
+
+ private ObjectStorage tos;
+ private ChecksumType type;
+
+ private void setEnv(ObjectStorage objectStore, ChecksumType csType) {
+ this.tos = objectStore;
+ this.type = csType;
+ }
+
+ @AfterEach
+ public void tearDown() throws Exception {
+ CommonUtils.runQuietly(() -> tos.deleteAll(""));
+ for (MultipartUpload upload : tos.listUploads("")) {
+ tos.abortMultipartUpload(upload.key(), upload.uploadId());
+ }
+ tos.close();
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testHeadObj(ObjectStorage objectStore, Checksum ckmer, ChecksumType csType) {
+ setEnv(objectStore, csType);
+
+ String key = "testPutChecksum";
+ byte[] data = TestUtility.rand(1024);
+ ckmer.update(data, 0, data.length);
+ assertEquals(ckmer.getValue(), parseChecksum(objectStore.put(key, data)));
+
+ ObjectInfo objInfo = objectStore.head(key);
+ assertEquals(ckmer.getValue(), parseChecksum(objInfo.checksum()));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testGetFileStatus(ObjectStorage objectStore, Checksum ckmer, ChecksumType csType) {
+ setEnv(objectStore, csType);
+ assumeFalse(objectStore.bucket().isDirectory());
+
+ Configuration conf = new Configuration(objectStore.conf());
+ conf.setBoolean(TosKeys.FS_TOS_GET_FILE_STATUS_ENABLED, true);
+ objectStore.initialize(conf, objectStore.bucket().name());
+
+ String key = "testFileStatus";
+ byte[] data = TestUtility.rand(256);
+ byte[] checksum = objectStore.put(key, data);
+
+ ObjectInfo obj1 = objectStore.objectStatus(key);
+ assertArrayEquals(checksum, obj1.checksum());
+ assertEquals(key, obj1.key());
+ assertEquals(obj1, objectStore.head(key));
+
+ ObjectInfo obj2 = objectStore.objectStatus(key + "/");
+ assertNull(obj2);
+
+ String dirKey = "testDirStatus/";
+ checksum = objectStore.put(dirKey, new byte[0]);
+
+ ObjectInfo obj3 = objectStore.objectStatus("testDirStatus");
+ assertArrayEquals(checksum, obj3.checksum());
+ assertEquals(dirKey, obj3.key());
+ assertEquals(obj3, objectStore.head(dirKey));
+ assertNull(objectStore.head("testDirStatus"));
+ ObjectInfo obj4 = objectStore.objectStatus(dirKey);
+ assertArrayEquals(checksum, obj4.checksum());
+ assertEquals(dirKey, obj4.key());
+ assertEquals(obj4, objectStore.head(dirKey));
+
+ String prefix = "testPrefix/";
+ objectStore.put(prefix + "subfile", data);
+ ObjectInfo obj5 = objectStore.objectStatus(prefix);
+ assertEquals(prefix, obj5.key());
+ assertArrayEquals(Constants.MAGIC_CHECKSUM, obj5.checksum());
+ assertNull(objectStore.head(prefix));
+ ObjectInfo obj6 = objectStore.objectStatus("testPrefix");
+ assertEquals(prefix, obj6.key());
+ assertArrayEquals(Constants.MAGIC_CHECKSUM, obj6.checksum());
+ assertNull(objectStore.head("testPrefix"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testObjectStatus(ObjectStorage objectStore, Checksum checksum, ChecksumType csType) {
+ setEnv(objectStore, csType);
+ assumeFalse(objectStore.bucket().isDirectory());
+
+ String key = "testObjectStatus";
+ byte[] data = TestUtility.rand(1024);
+ checksum.update(data, 0, data.length);
+ assertEquals(checksum.getValue(), parseChecksum(objectStore.put(key, data)));
+
+ ObjectInfo objInfo = objectStore.objectStatus(key);
+ assertEquals(checksum.getValue(), parseChecksum(objInfo.checksum()));
+
+ objInfo = objectStore.head(key);
+ assertEquals(checksum.getValue(), parseChecksum(objInfo.checksum()));
+
+ String dir = key + "/";
+ objectStore.put(dir, new byte[0]);
+ objInfo = objectStore.objectStatus(dir);
+ assertEquals(Constants.MAGIC_CHECKSUM, objInfo.checksum());
+
+ objInfo = objectStore.head(dir);
+ assertEquals(Constants.MAGIC_CHECKSUM, objInfo.checksum());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListObjs(ObjectStorage objectStore, Checksum checksum, ChecksumType csType) {
+ setEnv(objectStore, csType);
+
+ String key = "testListObjs";
+ byte[] data = TestUtility.rand(1024);
+ checksum.update(data, 0, data.length);
+ for (int i = 0; i < 5; i++) {
+ assertEquals(checksum.getValue(), parseChecksum(objectStore.put(key, data)));
+ }
+
+ ListObjectsRequest request =
+ ListObjectsRequest.builder().prefix(key).startAfter(null).maxKeys(-1).delimiter("/")
+ .build();
+ Iterator<ListObjectsResponse> iter = objectStore.list(request).iterator();
+ while (iter.hasNext()) {
+ List<ObjectInfo> objs = iter.next().objects();
+ for (ObjectInfo obj : objs) {
+ assertEquals(checksum.getValue(), parseChecksum(obj.checksum()));
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testPutChecksum(ObjectStorage objectStore, Checksum checksum, ChecksumType csType) {
+ setEnv(objectStore, csType);
+
+ String key = "testPutChecksum";
+ byte[] data = TestUtility.rand(1024);
+ checksum.update(data, 0, data.length);
+
+ byte[] checksumStr = objectStore.put(key, data);
+
+ assertEquals(checksum.getValue(), parseChecksum(checksumStr));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testMPUChecksum(ObjectStorage objectStore, Checksum checksum, ChecksumType csType) {
+ setEnv(objectStore, csType);
+
+ int partNumber = 2;
+ String key = "testMPUChecksum";
+ MultipartUpload mpu = objectStore.createMultipartUpload(key);
+ byte[] data = TestUtility.rand(mpu.minPartSize() * partNumber);
+ checksum.update(data, 0, data.length);
+
+ List<Part> parts = new ArrayList<>();
+ for (int i = 0; i < partNumber; i++) {
+ final int index = i;
+ Part part = objectStore.uploadPart(key, mpu.uploadId(), index + 1,
+ () -> new ByteArrayInputStream(data, index * mpu.minPartSize(), mpu.minPartSize()),
+ mpu.minPartSize());
+ parts.add(part);
+ }
+
+ byte[] checksumStr = objectStore.completeUpload(key, mpu.uploadId(), parts);
+ assertEquals(checksum.getValue(), parseChecksum(checksumStr));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testAppendable(ObjectStorage objectStore, Checksum checksum, ChecksumType csType) {
+ setEnv(objectStore, csType);
+ assumeFalse(objectStore.bucket().isDirectory());
+
+ // Test create object with append then append.
+ byte[] data = TestUtility.rand(256);
+ String prefix = "a/testAppendable/";
+ String key = prefix + "object.txt";
+ objectStore.append(key, data);
+
+ objectStore.append(key, new byte[0]);
+
+ // Test create object with put then append.
+ data = TestUtility.rand(256);
+ objectStore.put(key, data);
+
+ assertThrows("Expect not appendable.", NotAppendableException.class,
+ () -> objectStore.append(key, new byte[0]));
+
+ objectStore.delete(key);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDirectoryBucketAppendable(ObjectStorage objectStore, Checksum checksum,
+ ChecksumType csType) {
+ setEnv(objectStore, csType);
+ assumeTrue(objectStore.bucket().isDirectory());
+
+ byte[] data = TestUtility.rand(256);
+ String prefix = "a/testAppendable/";
+ String key = prefix + "object.txt";
+ objectStore.put(key, data);
+
+ objectStore.append(key, new byte[1024]);
+
+ objectStore.delete(key);
+ }
+
+ private long parseChecksum(byte[] checksum) {
+ switch (type) {
+ case CRC32C:
+ case CRC64ECMA:
+ return Bytes.toLong(checksum);
+ default:
+ throw new IllegalArgumentException(
+ String.format("Checksum type %s is not supported by TOS.", type.name()));
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSRetryPolicy.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSRetryPolicy.java
new file mode 100644
index 00000000000..2f841fce838
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSRetryPolicy.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos;
+
+import com.volcengine.tos.TOSV2;
+import com.volcengine.tos.TosException;
+import com.volcengine.tos.TosServerException;
+import com.volcengine.tos.comm.HttpStatus;
+import com.volcengine.tos.model.RequestInfo;
+import com.volcengine.tos.model.object.PutObjectOutput;
+import com.volcengine.tos.model.object.UploadPartV2Output;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.InputStreamProvider;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.object.tos.auth.SimpleCredentialsProvider;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.net.ssl.SSLException;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestTOSRetryPolicy {
+
+ private final String retryKey = "retryKey.txt";
+ private TOSV2 tosClient;
+ private DelegationClient client;
+
+ @BeforeAll
+ public static void before() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @BeforeEach
+ public void setUp() {
+ client = createRetryableDelegationClient();
+ tosClient = mock(TOSV2.class);
+ client.setClient(tosClient);
+ }
+
+ @AfterEach
+ public void tearDown() throws IOException {
+ tosClient.close();
+ client.close();
+ }
+
+ private DelegationClient createRetryableDelegationClient() {
+ Configuration conf = new Configuration();
+ conf.set(ConfKeys.FS_OBJECT_STORAGE_ENDPOINT.key(TOS_SCHEME),
+ "https://tos-cn-beijing.ivolces.com");
+ conf.set(TosKeys.FS_TOS_CREDENTIALS_PROVIDER, SimpleCredentialsProvider.NAME);
+ conf.setBoolean(TosKeys.FS_TOS_DISABLE_CLIENT_CACHE, true);
+ conf.set(TosKeys.FS_TOS_ACCESS_KEY_ID, "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY, "SECRET_KEY");
+ return new DelegationClientBuilder().bucket("test").conf(conf).build();
+ }
+
+ @Test
+ public void testShouldThrowExceptionAfterRunOut5RetryTimesIfNoRetryConfigSet()
+ throws IOException {
+ TOS storage =
+ (TOS) ObjectStorageFactory.create(TOS_SCHEME, TestUtility.bucket(), new Configuration());
+ storage.setClient(client);
+ client.setMaxRetryTimes(5);
+
+ PutObjectOutput response = mock(PutObjectOutput.class);
+ InputStreamProvider streamProvider = mock(InputStreamProvider.class);
+
+ when(tosClient.putObject(any())).thenThrow(
+ new TosServerException(HttpStatus.INTERNAL_SERVER_ERROR),
+ new TosServerException(HttpStatus.TOO_MANY_REQUESTS),
+ new TosException(new SocketException("fake msg")),
+ new TosException(new UnknownHostException("fake msg")),
+ new TosException(new SSLException("fake msg")),
+ new TosException(new InterruptedException("fake msg")),
+ new TosException(new InterruptedException("fake msg"))).thenReturn(response);
+
+ // after run out retry times, should throw exception
+ RuntimeException exception =
+ assertThrows(RuntimeException.class, () -> storage.put(retryKey, streamProvider, 0));
+ assertTrue(exception instanceof TosException);
+ assertTrue(exception.getCause() instanceof SSLException);
+
+ // the newStream method of stream provider should be called 5 times
+ verify(streamProvider, times(5)).newStream();
+
+ storage.close();
+ }
+
+ @Test
+ public void testShouldReturnResultAfterRetry8TimesIfConfigured10TimesRetry()
+ throws IOException {
+ TOS storage =
+ (TOS) ObjectStorageFactory.create(TOS_SCHEME, TestUtility.bucket(), new Configuration());
+ DelegationClient delegationClient = createRetryableDelegationClient();
+ delegationClient.setClient(tosClient);
+ delegationClient.setMaxRetryTimes(10);
+ storage.setClient(delegationClient);
+
+ UploadPartV2Output response = new UploadPartV2Output().setPartNumber(1).setEtag("etag");
+
+ InputStream in = mock(InputStream.class);
+ InputStreamProvider streamProvider = mock(InputStreamProvider.class);
+ when(streamProvider.newStream()).thenReturn(in);
+
+ when(tosClient.uploadPart(any())).thenThrow(
+ new TosServerException(HttpStatus.INTERNAL_SERVER_ERROR),
+ new TosServerException(HttpStatus.TOO_MANY_REQUESTS),
+ new TosException(new SocketException("fake msg")),
+ new TosException(new UnknownHostException("fake msg")),
+ new TosException(new SSLException("fake msg")),
+ new TosException(new InterruptedException("fake msg")),
+ new TosException(new InterruptedException("fake msg"))).thenReturn(response);
+
+ // after run out retry times, should throw exception
+ Part part = storage.uploadPart(retryKey, "uploadId", 1, streamProvider, 0);
+ assertEquals(1, part.num());
+ assertEquals("etag", part.eTag());
+
+ // the newStream method of stream provider should be called 8 times
+ verify(streamProvider, times(8)).newStream();
+
+ storage.close();
+ }
+
+ @Test
+ public void testShouldReturnResultIfRetry3TimesSucceed() throws IOException {
+ TOS storage =
+ (TOS) ObjectStorageFactory.create(TOS_SCHEME, TestUtility.bucket(), new Configuration());
+ storage.setClient(client);
+
+ PutObjectOutput response = mock(PutObjectOutput.class);
+ InputStreamProvider streamProvider = mock(InputStreamProvider.class);
+
+ RequestInfo requestInfo = mock(RequestInfo.class);
+ Map<String, String> header = new HashMap<>();
+ when(response.getRequestInfo()).thenReturn(requestInfo);
+ when(requestInfo.getHeader()).thenReturn(header);
+
+ when(tosClient.putObject(any())).thenThrow(
+ new TosServerException(HttpStatus.INTERNAL_SERVER_ERROR),
+ new TosServerException(HttpStatus.TOO_MANY_REQUESTS)).thenReturn(response);
+
+ storage.put(retryKey, streamProvider, 0);
+ // the newStream method of stream provider should be called 3 times
+ verify(streamProvider, times(3)).newStream();
+
+ storage.close();
+ }
+
+ @Test
+ public void testShouldNotRetryIfThrowUnRetryException() throws IOException {
+ TOS storage =
+ (TOS) ObjectStorageFactory.create(TOS_SCHEME, TestUtility.bucket(), new Configuration());
+ storage.setClient(client);
+
+ InputStreamProvider streamProvider = mock(InputStreamProvider.class);
+
+ when(tosClient.putObject(any())).thenThrow(
+ new TosException(new NullPointerException("fake msg.")));
+
+ RuntimeException exception =
+ assertThrows(RuntimeException.class, () -> storage.put(retryKey, streamProvider, 0));
+ assertTrue(exception instanceof TosException);
+ assertTrue(exception.getCause() instanceof NullPointerException);
+
+ // the newStream method of stream provider should be only called once.
+ verify(streamProvider, times(1)).newStream();
+
+ storage.close();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestAbstractCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestAbstractCredentialsProvider.java
new file mode 100644
index 00000000000..cbdbc7e4f7b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestAbstractCredentialsProvider.java
@@ -0,0 +1,60 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos.auth;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+
+public abstract class TestAbstractCredentialsProvider {
+ private String envAccessKeyId;
+ private String envSecretAccessKey;
+ private String envSessionToken;
+
+ protected Configuration getConf() {
+ return new Configuration();
+ }
+
+ protected void saveOsCredEnv() {
+ if (StringUtils.isNotEmpty(System.getenv(TOS.ENV_TOS_ACCESS_KEY_ID))) {
+ envAccessKeyId = System.getenv(TOS.ENV_TOS_ACCESS_KEY_ID);
+ }
+
+ if (StringUtils.isNotEmpty(System.getenv(TOS.ENV_TOS_SECRET_ACCESS_KEY))) {
+ envSecretAccessKey = System.getenv(TOS.ENV_TOS_SECRET_ACCESS_KEY);
+ }
+
+ if (StringUtils.isNotEmpty(System.getenv(TOS.ENV_TOS_SESSION_TOKEN))) {
+ envSessionToken = System.getenv(TOS.ENV_TOS_SESSION_TOKEN);
+ }
+ }
+
+ protected void resetOsCredEnv() {
+ resetOsCredEnv(TOS.ENV_TOS_ACCESS_KEY_ID, envAccessKeyId);
+ resetOsCredEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY, envSecretAccessKey);
+ resetOsCredEnv(TOS.ENV_TOS_SESSION_TOKEN, envSessionToken);
+ }
+
+ private void resetOsCredEnv(String key, String value) {
+ if (StringUtils.isNotEmpty(value)) {
+ TestUtility.setSystemEnv(key, value);
+ } else {
+ TestUtility.removeSystemEnv(key);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestDefaultCredentialsProviderChain.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestDefaultCredentialsProviderChain.java
new file mode 100644
index 00000000000..cc235da5824
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestDefaultCredentialsProviderChain.java
@@ -0,0 +1,209 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos.auth;
+
+import com.volcengine.tos.TosException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hadoop.fs.tosfs.util.TestUtility.removeSystemEnv;
+import static org.apache.hadoop.fs.tosfs.util.TestUtility.setSystemEnv;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestDefaultCredentialsProviderChain extends TestAbstractCredentialsProvider {
+
+ private static final String MOCK_TEST_AK = "AK";
+ private static final String MOCK_TEST_SK = "SK";
+ private static final String MOCK_TEST_TST_TOKEN = "STS_TOKEN";
+
+ private static final String MOCK_TEST_AK_WITH_BUCKET = "AK_WITH_BUCKET";
+ private static final String MOCK_TEST_SK_WITH_BUCKET = "SK_WITH_BUCKET";
+ private static final String MOCK_TEST_STS_TOKEN_WITH_BUCKET = "STS_TOKEN_WITH_BUCKET";
+
+ private static final String MOCK_TEST_ENV_AK = "ENV_AK";
+ private static final String MOCK_TEST_ENV_SK = "ENV_SK";
+ private static final String MOCK_TEST_ENV_STS_TOKEN = "ENV_STS_TOKEN";
+
+ private static final String MOCK_TEST_BUCKET = "test";
+ private static final String MOCK_TEST_ROLE_NAME = "roleName";
+ private static final String MOCK_PATH = "/volcstack/latest/iam/security_credentials/";
+ private static final String API_ENDPOINT = MOCK_PATH + MOCK_TEST_ROLE_NAME;
+ private static final String EXPIRED_TIME_PATTERN = "yyyy-MM-dd'T'HH:mm:ssXXX";
+
+ @Override
+ public Configuration getConf() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key("test"), MOCK_TEST_AK_WITH_BUCKET);
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key("test"), MOCK_TEST_SK_WITH_BUCKET);
+ conf.set(TosKeys.FS_TOS_BUCKET_SESSION_TOKEN.key("test"), MOCK_TEST_STS_TOKEN_WITH_BUCKET);
+ conf.set(TosKeys.FS_TOS_ACCESS_KEY_ID, MOCK_TEST_AK);
+ conf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY, MOCK_TEST_SK);
+ conf.set(TosKeys.FS_TOS_SESSION_TOKEN, MOCK_TEST_TST_TOKEN);
+ return conf;
+ }
+
+ @BeforeEach
+ public void setUp() {
+ saveOsCredEnv();
+ }
+
+ @Test
+ public void testLoadCredFromEnvProvider() {
+ Configuration conf = getConf();
+ setSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID, MOCK_TEST_ENV_AK);
+ setSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY, MOCK_TEST_ENV_SK);
+ setSystemEnv(TOS.ENV_TOS_SESSION_TOKEN, MOCK_TEST_ENV_STS_TOKEN);
+ DefaultCredentialsProviderChain chain = new DefaultCredentialsProviderChain();
+ chain.initialize(conf, null);
+
+ assertEquals(chain.credential().getAccessKeyId(),
+ MOCK_TEST_ENV_AK, String.format("expect %s", MOCK_TEST_ENV_AK));
+ assertEquals(chain.credential().getAccessKeySecret(), MOCK_TEST_ENV_SK,
+ String.format("expect %s", MOCK_TEST_ENV_SK));
+ assertEquals(chain.credential().getSecurityToken(), MOCK_TEST_ENV_STS_TOKEN,
+ String.format("expect %s", MOCK_TEST_ENV_STS_TOKEN));
+ assertTrue(chain.lastUsedProvider() instanceof EnvironmentCredentialsProvider);
+ }
+
+ @Test
+ public void testLoadCredFromSimpleProviderWithBucket() {
+ Configuration conf = getConf();
+ removeSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID);
+ removeSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY);
+ removeSystemEnv(TOS.ENV_TOS_SESSION_TOKEN);
+ DefaultCredentialsProviderChain chain = new DefaultCredentialsProviderChain();
+ chain.initialize(conf, MOCK_TEST_BUCKET);
+
+ assertEquals(chain.credential().getAccessKeyId(), MOCK_TEST_AK_WITH_BUCKET,
+ String.format("expect %s", MOCK_TEST_AK_WITH_BUCKET));
+ assertEquals(chain.credential().getAccessKeySecret(), MOCK_TEST_SK_WITH_BUCKET,
+ String.format("expect %s", MOCK_TEST_SK_WITH_BUCKET));
+ assertEquals(chain.credential().getSecurityToken(), MOCK_TEST_STS_TOKEN_WITH_BUCKET,
+ String.format("expect %s", MOCK_TEST_STS_TOKEN_WITH_BUCKET));
+ assertTrue(chain.lastUsedProvider() instanceof SimpleCredentialsProvider);
+ }
+
+ @Test
+ public void testLoadCredFromSimpleProvider() {
+ Configuration conf = getConf();
+ removeSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID);
+ removeSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY);
+ DefaultCredentialsProviderChain chain = new DefaultCredentialsProviderChain();
+ chain.initialize(conf, "test-bucket");
+
+ assertEquals(chain.credential().getAccessKeyId(), MOCK_TEST_AK,
+ String.format("expect %s", MOCK_TEST_AK));
+ assertEquals(chain.credential().getAccessKeySecret(), MOCK_TEST_SK,
+ String.format("expect %s", MOCK_TEST_SK));
+ assertTrue(chain.lastUsedProvider() instanceof SimpleCredentialsProvider);
+ }
+
+ @Test
+ public void testNotFoundAnyProvider() {
+ removeSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID);
+ removeSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY);
+ DefaultCredentialsProviderChain chain = new DefaultCredentialsProviderChain();
+ chain.initialize(new Configuration(), MOCK_TEST_BUCKET);
+ assertThrows(RuntimeException.class, chain::credential);
+ }
+
+ @AfterEach
+ public void after() {
+ resetOsCredEnv();
+ }
+
+ @Test
+ public void testShouldReturnAKSKFollowByProviderSequence() {
+ setSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID, "ENV_ACCESS_KEY");
+ setSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY, "ENV_SECRET_KEY");
+
+ // use the simple credential provider at first.
+ String providerClassesStr = SimpleCredentialsProvider.class.getName() + ','
+ + EnvironmentCredentialsProvider.class.getName();
+
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES, providerClassesStr);
+ conf.set(TosKeys.FS_TOS_ACCESS_KEY_ID, MOCK_TEST_AK);
+ conf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY, MOCK_TEST_SK);
+ conf.set(TosKeys.FS_TOS_SESSION_TOKEN, MOCK_TEST_TST_TOKEN);
+
+ DefaultCredentialsProviderChain provider = new DefaultCredentialsProviderChain();
+ provider.initialize(conf, MOCK_TEST_BUCKET);
+
+ ExpireableCredential cred = provider.createCredential();
+ assertEquals(MOCK_TEST_AK, cred.getAccessKeyId());
+ assertEquals(MOCK_TEST_SK, cred.getAccessKeySecret());
+
+ assertFalse(cred.isExpired());
+
+ // use the env credential provider at first.
+ providerClassesStr = EnvironmentCredentialsProvider.class.getName() + ','
+ + SimpleCredentialsProvider.class.getName();
+ conf.set(TosKeys.FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES, providerClassesStr);
+
+ provider = new DefaultCredentialsProviderChain();
+ provider.initialize(conf, MOCK_TEST_BUCKET);
+ cred = provider.createCredential();
+ assertEquals("ENV_ACCESS_KEY", cred.getAccessKeyId());
+ assertEquals("ENV_SECRET_KEY", cred.getAccessKeySecret());
+ assertFalse(cred.isExpired());
+
+ removeSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID);
+ removeSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY);
+ }
+
+ @Test
+ public void testShouldThrowExceptionWhenCustomClassNotFound() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES,
+ SimpleCredentialsProvider.class.getName() + "NotExist");
+
+ DefaultCredentialsProviderChain provider = new DefaultCredentialsProviderChain();
+ TosException tosException =
+ assertThrows(TosException.class, () -> provider.initialize(conf, null));
+ assertTrue(tosException.getCause() instanceof ClassNotFoundException);
+ }
+
+ @Test
+ public void testShouldThrowExceptionIfNoDefaultConstructorFound() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_CUSTOM_CREDENTIAL_PROVIDER_CLASSES,
+ TestCredentialProviderNoDefaultConstructor.class.getName());
+ DefaultCredentialsProviderChain provider = new DefaultCredentialsProviderChain();
+ RuntimeException exception =
+ assertThrows(RuntimeException.class, () -> provider.initialize(conf, null));
+ assertTrue(exception.getMessage().contains("java.lang.NoSuchMethodException"));
+ }
+
+ static class TestCredentialProviderNoDefaultConstructor extends AbstractCredentialsProvider {
+
+ TestCredentialProviderNoDefaultConstructor(String fake) {
+ }
+
+ @Override
+ protected ExpireableCredential createCredential() {
+ return null;
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestEnvironmentCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestEnvironmentCredentialsProvider.java
new file mode 100644
index 00000000000..d5d9addc9d6
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestEnvironmentCredentialsProvider.java
@@ -0,0 +1,67 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos.auth;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+public class TestEnvironmentCredentialsProvider extends TestAbstractCredentialsProvider {
+
+ @BeforeEach
+ public void setUp() {
+ saveOsCredEnv();
+ }
+
+ @Test
+ public void testLoadAkSkFromEnvProvider() {
+ TestUtility.setSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID, "AccessKeyId");
+ TestUtility.setSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY, "SecretAccessKey");
+
+ EnvironmentCredentialsProvider provider = new EnvironmentCredentialsProvider();
+ provider.initialize(new Configuration(), null);
+
+ ExpireableCredential oldCred = provider.credential();
+ assertEquals(oldCred.getAccessKeyId(), "AccessKeyId", "provider ak must be equals to env ak");
+ assertEquals(oldCred.getAccessKeySecret(), "SecretAccessKey",
+ "provider sk must be equals to env sk");
+
+ TestUtility.setSystemEnv(TOS.ENV_TOS_ACCESS_KEY_ID, "newAccessKeyId");
+ TestUtility.setSystemEnv(TOS.ENV_TOS_SECRET_ACCESS_KEY, "newSecretAccessKey");
+ TestUtility.setSystemEnv(TOS.ENV_TOS_SESSION_TOKEN, "newSessionToken");
+
+ assertFalse(oldCred.isExpired());
+
+ ExpireableCredential newCred = provider.credential();
+ assertEquals(newCred.getAccessKeyId(), "AccessKeyId", "provider ak must be equals to env ak");
+ assertEquals(newCred.getAccessKeySecret(), "SecretAccessKey",
+ "provider sk must be equals to env sk");
+
+ assertFalse(newCred.isExpired());
+ }
+
+ @AfterEach
+ public void resetEnv() {
+ resetOsCredEnv();
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestSimpleCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestSimpleCredentialsProvider.java
new file mode 100644
index 00000000000..6e3dc8825a7
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/auth/TestSimpleCredentialsProvider.java
@@ -0,0 +1,76 @@
+/*
+ * ByteDance Volcengine EMR, Copyright 2022.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object.tos.auth;
+
+import com.volcengine.tos.auth.Credential;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.conf.TosKeys;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestSimpleCredentialsProvider extends TestAbstractCredentialsProvider {
+
+ @Test
+ public void testStaticCredentials() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_ACCESS_KEY_ID, "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY, "SECRET_KEY");
+ conf.set(TosKeys.FS_TOS_SESSION_TOKEN, "STS_TOKEN");
+ SimpleCredentialsProvider provider = new SimpleCredentialsProvider();
+ provider.initialize(conf, "test");
+ Credential credentials = provider.credential();
+ assertEquals("ACCESS_KEY", credentials.getAccessKeyId(), "access key must be ACCESS_KEY");
+ assertEquals("SECRET_KEY", credentials.getAccessKeySecret(), "secret key must be SECRET_KEY");
+ assertEquals("STS_TOKEN", credentials.getSecurityToken(), "sts token must be STS_TOKEN");
+ }
+
+ @Test
+ public void testStaticCredentialsWithBucket() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key("test"), "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key("test"), "SECRET_KEY");
+ conf.set(TosKeys.FS_TOS_BUCKET_SESSION_TOKEN.key("test"), "STS_TOKEN");
+ SimpleCredentialsProvider provider = new SimpleCredentialsProvider();
+ provider.initialize(conf, "test");
+ Credential credentials = provider.credential();
+ assertEquals("ACCESS_KEY", credentials.getAccessKeyId(), "access key must be ACCESS_KEY");
+ assertEquals("SECRET_KEY", credentials.getAccessKeySecret(), "secret key must be SECRET_KEY");
+ assertEquals("STS_TOKEN", credentials.getSecurityToken(), "sts token must be STS_TOKEN");
+ }
+
+ @Test
+ public void testStaticCredentialsWithPriority() {
+ Configuration conf = new Configuration();
+ conf.set(TosKeys.FS_TOS_ACCESS_KEY_ID, "ACCESS_KEY");
+ conf.set(TosKeys.FS_TOS_SECRET_ACCESS_KEY, "SECRET_KEY");
+ conf.set(TosKeys.FS_TOS_SESSION_TOKEN, "STS_TOKEN");
+ conf.set(TosKeys.FS_TOS_BUCKET_ACCESS_KEY_ID.key("test"), "ACCESS_KEY_BUCKET");
+ conf.set(TosKeys.FS_TOS_BUCKET_SECRET_ACCESS_KEY.key("test"), "SECRET_KEY_BUCKET");
+ conf.set(TosKeys.FS_TOS_BUCKET_SESSION_TOKEN.key("test"), "STS_TOKEN_BUCKET");
+
+ SimpleCredentialsProvider provider = new SimpleCredentialsProvider();
+ provider.initialize(conf, "test");
+ Credential credentials = provider.credential();
+ assertEquals("ACCESS_KEY_BUCKET", credentials.getAccessKeyId(),
+ "access key must be ACCESS_KEY_BUCKET");
+ assertEquals("SECRET_KEY_BUCKET", credentials.getAccessKeySecret(),
+ "secret key must be SECRET_KEY_BUCKET");
+ assertEquals("STS_TOKEN_BUCKET", credentials.getSecurityToken(),
+ "sts token must be STS_TOKEN_BUCKET");
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseFsOps.java
new file mode 100644
index 00000000000..3048f05eef3
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseFsOps.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.ops;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public abstract class TestBaseFsOps implements TestBaseOps {
+
+ private ObjectStorage storage;
+
+ @Override
+ public ObjectStorage storage() {
+ return storage;
+ }
+
+ private void setStorage(ObjectStorage storage) {
+ this.storage = storage;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ CommonUtils.runQuietly(() -> storage.deleteAll(""));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDeleteFile(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path path = new Path("/a/b");
+ touchFile(path, TestUtility.rand(8));
+ assertFileExist(path);
+
+ fsOps.deleteFile(path);
+ assertFileDoesNotExist(path);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDeleteEmptyDir(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path path = new Path("/a/b/");
+ mkdir(path);
+
+ fsOps.deleteDir(path, false);
+ assertDirDoesNotExist(path);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testDeleteNonEmptyDir(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path dirPath = new Path("/a/b/");
+ Path subDirPath = new Path("/a/b/c/");
+ Path filePath = new Path("/a/b/file.txt");
+ mkdir(dirPath);
+ mkdir(subDirPath);
+ touchFile(filePath, new byte[10]);
+
+ assertThrows(PathIsNotEmptyDirectoryException.class, () -> fsOps.deleteDir(dirPath, false));
+ assertDirExist(dirPath);
+ assertDirExist(subDirPath);
+ assertFileExist(filePath);
+
+ fsOps.deleteDir(dirPath, true);
+ assertDirDoesNotExist(dirPath);
+ assertDirDoesNotExist(subDirPath);
+ assertFileDoesNotExist(filePath);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testCreateDirRecursive(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path path = new Path("/aa/bb/cc");
+ String key = ObjectUtils.pathToKey(path, true);
+ String parentKey = ObjectUtils.pathToKey(path.getParent(), true);
+ String grandparents = ObjectUtils.pathToKey(path.getParent().getParent(), true);
+
+ assertDirDoesNotExist(parentKey);
+ assertDirDoesNotExist(grandparents);
+
+ fsOps.mkdirs(path);
+ assertDirExist(key);
+ assertDirExist(parentKey);
+ assertDirExist(grandparents);
+
+ store.delete(key);
+ assertDirExist(parentKey);
+ assertDirExist(grandparents);
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListEmptyDir(ObjectStorage store, FsOps fsOps) {
+ setStorage(store);
+ Path dir = path("testListEmptyDir");
+ mkdir(dir);
+
+ assertFalse(listDir(fsOps, dir, false).iterator().hasNext());
+ assertFalse(listDir(fsOps, dir, true).iterator().hasNext());
+ assertTrue(fsOps.isEmptyDirectory(dir));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListNonExistDir(ObjectStorage store, FsOps fsOps) {
+ setStorage(store);
+ Path dir = path("testListNonExistDir");
+ assertDirDoesNotExist(dir);
+
+ assertFalse(listDir(fsOps, dir, false).iterator().hasNext());
+ assertFalse(listDir(fsOps, dir, false).iterator().hasNext());
+ assertTrue(fsOps.isEmptyDirectory(dir));
+ }
+
+ private Iterable<RawFileStatus> listDir(FsOps fsOps, Path dir, boolean recursive) {
+ return fsOps.listDir(dir, recursive, s -> true);
+ }
+
+ private Iterable<RawFileStatus> listFiles(FsOps fsOps, Path dir, boolean recursive) {
+ return fsOps.listDir(dir, recursive, s -> !ObjectInfo.isDir(s));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListAFileViaListDir(ObjectStorage store, FsOps fsOps) {
+ setStorage(store);
+ Path file = new Path("testListFileViaListDir");
+ touchFile(file, TestUtility.rand(8));
+ assertFalse(listDir(fsOps, file, false).iterator().hasNext());
+ assertFalse(listDir(fsOps, file, true).iterator().hasNext());
+
+ Path nonExistFile = new Path("testListFileViaListDir-nonExist");
+ assertFileDoesNotExist(nonExistFile);
+ assertFalse(listDir(fsOps, nonExistFile, false).iterator().hasNext());
+ assertFalse(listDir(fsOps, nonExistFile, true).iterator().hasNext());
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testListFiles(ObjectStorage store, FsOps fsOps) {
+ setStorage(store);
+ Path dir = path("testListEmptyFiles");
+ mkdir(dir);
+
+ assertFalse(listFiles(fsOps, dir, false).iterator().hasNext());
+ assertFalse(listFiles(fsOps, dir, true).iterator().hasNext());
+
+ mkdir(new Path(dir, "subDir"));
+ assertFalse(listFiles(fsOps, dir, false).iterator().hasNext());
+ assertFalse(listFiles(fsOps, dir, true).iterator().hasNext());
+
+ RawFileStatus subDir = listDir(fsOps, dir, false).iterator().next();
+ assertFalse(subDir.isFile());
+ assertEquals("subDir", subDir.getPath().getName());
+
+ ObjectInfo fileObj = touchFile(new Path(dir, "subFile"), TestUtility.rand(8));
+ RawFileStatus subFile = listFiles(fsOps, dir, false).iterator().next();
+ assertArrayEquals(fileObj.checksum(), subFile.checksum());
+ assertTrue(subFile.isFile());
+
+ assertFalse(fsOps.isEmptyDirectory(dir));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRecursiveList(ObjectStorage store, FsOps fsOps) {
+ setStorage(store);
+ Path root = path("root");
+ Path file1 = path("root", "file1");
+ Path file2 = path("root", "afile2");
+ Path dir1 = path("root", "dir1");
+ Path file3 = path("root", "dir1", "file3");
+
+ mkdir(root);
+ mkdir(dir1);
+ touchFile(file1, TestUtility.rand(8));
+ touchFile(file2, TestUtility.rand(8));
+ touchFile(file3, TestUtility.rand(8));
+
+ // List result is in sorted lexicographical order if recursive is false
+ Assertions.assertThat(listDir(fsOps, root, false))
+ .hasSize(3)
+ .extracting(f -> f.getPath().getName())
+ .contains("afile2", "dir1", "file1");
+
+ // List result is in sorted lexicographical order if recursive is false
+ Assertions.assertThat(listFiles(fsOps, root, false))
+ .hasSize(2)
+ .extracting(f -> f.getPath().getName())
+ .contains("afile2", "file1");
+
+ // listDir with recursive=true doesn't guarantee the return result in a sorted order
+ Assertions.assertThat(listDir(fsOps, root, true))
+ .hasSize(4)
+ .extracting(f -> f.getPath().getName())
+ .containsExactlyInAnyOrder("afile2", "dir1", "file1", "file3");
+
+ // listFiles with recursive=true doesn't guarantee the return result in a sorted order
+ Assertions.assertThat(listFiles(fsOps, root, true))
+ .hasSize(3)
+ .extracting(f -> f.getPath().getName())
+ .containsExactlyInAnyOrder("afile2", "file1", "file3");
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRenameFile(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+
+ int dataSize = 1024 * 1024;
+ String filename = String.format("%sMB.txt", dataSize >> 20);
+ Path srcFile = new Path(renameSrc, filename);
+ byte[] data = writeData(srcFile, dataSize);
+ Path dstFile = new Path(renameDest, filename);
+
+ // The dest file and dest parent don't exist.
+ assertFileExist(srcFile);
+ assertDirDoesNotExist(renameDest);
+ assertFileDoesNotExist(dstFile);
+
+ fsOps.renameFile(srcFile, dstFile, data.length);
+ assertFileDoesNotExist(srcFile);
+ assertDirExist(renameSrc);
+ assertFileExist(dstFile);
+
+ try (InputStream in = store.get(ObjectUtils.pathToKey(dstFile)).stream()) {
+ assertArrayEquals(data, IOUtils.toByteArray(in));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRenameDir(ObjectStorage store, FsOps fsOps) throws IOException {
+ setStorage(store);
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+
+ mkdir(renameSrc);
+ int dataSize = 1024 * 1024;
+ String filename = String.format("%sMB.txt", dataSize >> 20);
+ Path srcFile = new Path(renameSrc, filename);
+ Path dstFile = new Path(renameDest, filename);
+ byte[] data = writeData(srcFile, dataSize);
+
+ assertFileExist(srcFile);
+ assertFileDoesNotExist(dstFile);
+ assertDirExist(renameSrc);
+ assertDirDoesNotExist(renameDest);
+
+ fsOps.renameDir(renameSrc, renameDest);
+ assertFileDoesNotExist(srcFile);
+ assertDirDoesNotExist(renameSrc);
+ assertFileExist(dstFile);
+ assertDirExist(renameDest);
+
+ try (InputStream in = store.get(ObjectUtils.pathToKey(dstFile)).stream()) {
+ assertArrayEquals(data, IOUtils.toByteArray(in));
+ }
+ }
+
+ private byte[] writeData(Path path, int size) {
+ byte[] data = TestUtility.rand(size);
+ touchFile(path, data);
+ return data;
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseOps.java
new file mode 100644
index 00000000000..90e31cbc47b
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestBaseOps.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.ops;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public interface TestBaseOps {
+
+ default Path path(String... keys) {
+ return new Path(String.format("/%s", Joiner.on("/").join(keys)));
+ }
+
+ default void assertFileExist(Path file) {
+ assertNotNull(ObjectUtils.pathToKey(file));
+ }
+
+ default void assertFileDoesNotExist(String key) {
+ assertNull(storage().head(key));
+ }
+
+ default void assertFileDoesNotExist(Path file) {
+ assertFileDoesNotExist(ObjectUtils.pathToKey(file));
+ }
+
+ default void assertDirExist(String key) {
+ assertNotNull(storage().head(key));
+ }
+
+ default void assertDirExist(Path path) {
+ assertDirExist(ObjectUtils.pathToKey(path, true));
+ }
+
+ default void assertDirDoesNotExist(String key) {
+ assertNull(storage().head(key));
+ }
+
+ default void assertDirDoesNotExist(Path path) {
+ assertDirDoesNotExist(ObjectUtils.pathToKey(path, true));
+ }
+
+ default void mkdir(Path path) {
+ storage().put(ObjectUtils.pathToKey(path, true), new byte[0]);
+ assertDirExist(path);
+ }
+
+ default ObjectInfo touchFile(Path path, byte[] data) {
+ byte[] checksum = storage().put(ObjectUtils.pathToKey(path), data);
+ ObjectInfo obj = storage().head(ObjectUtils.pathToKey(path));
+ assertArrayEquals(checksum, obj.checksum());
+ return obj;
+ }
+
+ ObjectStorage storage();
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
new file mode 100644
index 00000000000..e587c71c61a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.ops;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.provider.Arguments;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Stream;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestDefaultFsOps extends TestBaseFsOps {
+ private static ExecutorService threadPool;
+
+ static Stream<Arguments> provideArguments() {
+ // Case1: direct rename.
+ List<Arguments> values = new ArrayList<>();
+ Configuration directRenameConf = new Configuration();
+ directRenameConf.setBoolean(ConfKeys.FS_OBJECT_RENAME_ENABLED.key("tos"), true);
+ directRenameConf.setBoolean(ConfKeys.FS_ASYNC_CREATE_MISSED_PARENT.key("tos"), false);
+
+ ObjectStorage storage0 =
+ ObjectStorageFactory.createWithPrefix(String.format("tos-%s/", UUIDUtils.random()),
+ TOS_SCHEME, TestUtility.bucket(), directRenameConf);
+ values.add(Arguments.of(
+ storage0,
+ new DefaultFsOps(storage0, directRenameConf, threadPool, obj -> {
+ long modifiedTime = RawFileSystem.dateToLong(obj.mtime());
+ String path =
+ String.format("%s://%s/%s", storage0.scheme(), storage0.bucket().name(), obj.key());
+ return new RawFileStatus(obj.size(), obj.isDir(), 0, modifiedTime, new Path(path), "fake",
+ obj.checksum());
+ })));
+
+ // Case2: copied rename.
+ Configuration copiedRenameConf = new Configuration();
+ copiedRenameConf.setLong(ConfKeys.FS_MULTIPART_COPY_THRESHOLD.key("tos"), 1L << 20);
+ copiedRenameConf.setBoolean(ConfKeys.FS_ASYNC_CREATE_MISSED_PARENT.key("tos"), false);
+
+ ObjectStorage storage1 =
+ ObjectStorageFactory.createWithPrefix(String.format("tos-%s/", UUIDUtils.random()),
+ TOS_SCHEME, TestUtility.bucket(), copiedRenameConf);
+ values.add(Arguments.of(
+ storage1,
+ new DefaultFsOps(storage1, copiedRenameConf, threadPool, obj -> {
+ long modifiedTime = RawFileSystem.dateToLong(obj.mtime());
+ String path =
+ String.format("%s://%s/%s", storage1.scheme(), storage1.bucket().name(), obj.key());
+ return new RawFileStatus(obj.size(), obj.isDir(), 0, modifiedTime, new Path(path), "fake",
+ obj.checksum());
+ })));
+
+ return values.stream();
+ }
+
+ @BeforeAll
+ public static void beforeClass() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ threadPool = ThreadPools.newWorkerPool("TestDefaultFsHelper-pool");
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ if (!TestEnv.checkTestEnabled()) {
+ return;
+ }
+
+ if (!threadPool.isShutdown()) {
+ threadPool.shutdown();
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDirectoryFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDirectoryFsOps.java
new file mode 100644
index 00000000000..eaac49ef73e
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDirectoryFsOps.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.ops;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.object.DirectoryStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.provider.Arguments;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+// TODO change to directory bucket configuration.
+public class TestDirectoryFsOps extends TestBaseFsOps {
+
+ public static Stream<Arguments> provideArguments() {
+ List<Arguments> values = new ArrayList<>();
+
+ ObjectStorage storage =
+ ObjectStorageFactory.createWithPrefix(String.format("tos-%s/", UUIDUtils.random()),
+ TOS_SCHEME, TestUtility.bucket(), new Configuration());
+ values.add(Arguments.of(storage, new DirectoryFsOps((DirectoryStorage) storage, obj -> {
+ long modifiedTime = RawFileSystem.dateToLong(obj.mtime());
+ String path =
+ String.format("%s://%s/%s", storage.scheme(), storage.bucket().name(), obj.key());
+ return new RawFileStatus(obj.size(), obj.isDir(), 0, modifiedTime, new Path(path), "fake",
+ obj.checksum());
+ })));
+
+ return values.stream();
+ }
+
+ @BeforeAll
+ public static void beforeClass() {
+ assumeTrue(TestEnv.checkTestEnabled());
+ }
+
+ @Override
+ public void testRenameDir(ObjectStorage store, FsOps fsOps) {
+ // Will remove this test case once test environment support
+ assumeTrue(store.bucket().isDirectory());
+ }
+
+ @Override
+ public void testRenameFile(ObjectStorage store, FsOps fsOps) {
+ // Will remove this test case once test environment support
+ assumeTrue(store.bucket().isDirectory());
+ }
+
+ @Override
+ public void testCreateDirRecursive(ObjectStorage store, FsOps fsOps) {
+ // Will remove this test case once test environment support
+ assumeTrue(store.bucket().isDirectory());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestRenameOp.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestRenameOp.java
new file mode 100644
index 00000000000..159e6f87601
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestRenameOp.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.ops;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.apache.hadoop.fs.tosfs.common.ThreadPools;
+import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
+import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
+import org.apache.hadoop.fs.tosfs.object.Part;
+import org.apache.hadoop.fs.tosfs.util.CommonUtils;
+import org.apache.hadoop.fs.tosfs.util.TempFiles;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class TestRenameOp implements TestBaseOps {
+ private static final String FILE_STORE_ROOT = TempFiles.newTempDir("TestRenameOp");
+
+ private ObjectStorage storage;
+ private ExecutorService renamePool;
+
+ static Stream<Arguments> provideArguments() {
+ assumeTrue(TestEnv.checkTestEnabled());
+
+ List<Arguments> values = new ArrayList<>();
+ List<ObjectStorage> storages = TestUtility.createTestObjectStorage(FILE_STORE_ROOT);
+ for (ObjectStorage store : storages) {
+ values.add(Arguments.of(store));
+ }
+ return values.stream();
+ }
+
+ @Override
+ public ObjectStorage storage() {
+ return storage;
+ }
+
+ private void setStorage(ObjectStorage storage) {
+ this.storage = storage;
+ }
+
+ @BeforeEach
+ public void prepare() {
+ this.renamePool = ThreadPools.newWorkerPool("renamePool");
+ }
+
+ @AfterEach
+ public void tearDown() {
+ CommonUtils.runQuietly(() -> storage().deleteAll(""));
+ CommonUtils.runQuietly(renamePool::shutdown);
+ }
+
+ @AfterAll
+ public static void afterClass() {
+ CommonUtils.runQuietly(() -> TempFiles.deleteDir(FILE_STORE_ROOT));
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRenameFileDirectly(ObjectStorage store) throws IOException {
+ setStorage(store);
+
+ Configuration conf = new Configuration();
+ conf.setLong(ConfKeys.FS_MULTIPART_COPY_THRESHOLD.key(store.scheme()), 1L << 20);
+ ExtendedRenameOp operation = new ExtendedRenameOp(conf, store, renamePool);
+
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+
+ int dataSize = 1024 * 1024;
+ String filename = String.format("%sMB.txt", dataSize >> 20);
+ Path srcFile = new Path(renameSrc, filename);
+ Path dstFile = new Path(renameDest, filename);
+ byte[] data = writeData(srcFile, dataSize);
+ mkdir(renameDest);
+
+ assertFileExist(srcFile);
+ assertFileDoesNotExist(dstFile);
+ assertDirExist(renameDest);
+
+ operation.renameFile(srcFile, dstFile, data.length);
+ assertFileDoesNotExist(srcFile);
+ assertFileExist(dstFile);
+ Map<String, List<Part>> uploadInfos = operation.uploadInfos;
+ assertEquals(0, uploadInfos.size(),
+ "use put method when rename file, upload info's size should be 0");
+
+ try (InputStream in = store.get(ObjectUtils.pathToKey(dstFile)).stream()) {
+ assertArrayEquals(data, IOUtils.toByteArray(in));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("provideArguments")
+ public void testRenameFileByUploadParts(ObjectStorage store) throws IOException {
+ setStorage(store);
+
+ assumeFalse(store.bucket().isDirectory());
+ Configuration conf = new Configuration();
+ conf.setLong(ConfKeys.FS_MULTIPART_COPY_THRESHOLD.key(store.scheme()), 1L << 20);
+ ExtendedRenameOp operation = new ExtendedRenameOp(conf, store, renamePool);
+
+ Path renameSrc = path("renameSrc");
+ Path renameDest = path("renameDst");
+
+ int dataSize = 10 * 1024 * 1024;
+ String filename = String.format("%sMB.txt", dataSize >> 20);
+ Path srcFile = new Path(renameSrc, filename);
+ Path dstFile = new Path(renameDest, filename);
+ byte[] data = writeData(srcFile, dataSize);
+ mkdir(renameDest);
+
+ assertFileExist(srcFile);
+ assertFileDoesNotExist(dstFile);
+ assertDirExist(renameDest);
+
+ operation.renameFile(srcFile, dstFile, data.length);
+ assertFileDoesNotExist(srcFile);
+ assertFileExist(dstFile);
+ Map<String, List<Part>> uploadInfos = operation.uploadInfos;
+ assertTrue(uploadInfos.size() != 0,
+ "use upload parts method when rename file, upload info's size should not be 0");
+ List<Part> parts = uploadInfos.get(ObjectUtils.pathToKey(dstFile));
+ assertNotNull(parts,
+ "use upload parts method when rename file, upload info should not be null");
+ assertTrue(parts.size() >= 2, "use upload parts method when rename file,"
+ + " the num of upload parts should be greater than or equal to 2");
+ long fileLength = parts.stream().mapToLong(Part::size).sum();
+ assertEquals(dataSize, fileLength);
+
+ try (InputStream in = store.get(ObjectUtils.pathToKey(dstFile)).stream()) {
+ assertArrayEquals(data, IOUtils.toByteArray(in));
+ }
+ }
+
+ private byte[] writeData(Path path, int size) {
+ byte[] data = TestUtility.rand(size);
+ touchFile(path, data);
+ return data;
+ }
+
+ static class ExtendedRenameOp extends RenameOp {
+ private Map<String, List<Part>> uploadInfos = Maps.newHashMap();
+
+ ExtendedRenameOp(Configuration conf, ObjectStorage storage, ExecutorService pool) {
+ super(conf, storage, pool);
+ }
+
+ @Override
+ protected void finishUpload(String key, String uploadId, List<Part> uploadParts) {
+ super.finishUpload(key, uploadId, uploadParts);
+ if (!uploadInfos.isEmpty()) {
+ uploadInfos.clear();
+ }
+ uploadInfos.put(key, uploadParts);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
new file mode 100644
index 00000000000..d4191de66a7
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.Lists;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+public final class TempFiles implements Closeable {
+ private final List<String> files = Lists.newArrayList();
+ private final List<String> dirs = Lists.newArrayList();
+
+ private TempFiles() {
+ }
+
+ public static TempFiles of() {
+ return new TempFiles();
+ }
+
+ public String newFile() {
+ String p = newTempFile();
+ files.add(p);
+ return p;
+ }
+
+ public String newDir() {
+ return newDir(null);
+ }
+
+ public String newDir(String prefix) {
+ String p = newTempDir(prefix);
+ dirs.add(p);
+ return p;
+ }
+
+ @Override
+ public void close() {
+ files.forEach(file -> CommonUtils.runQuietly(() -> TempFiles.deleteFile(file)));
+ files.clear();
+ dirs.forEach(dir -> CommonUtils.runQuietly(() -> TempFiles.deleteDir(dir)));
+ dirs.clear();
+ }
+
+ public static String newTempFile() {
+ return String.join(File.pathSeparator, newTempDir(), UUIDUtils.random());
+ }
+
+ public static String newTempDir() {
+ return newTempDir(null);
+ }
+
+ public static String newTempDir(String prefix) {
+ try {
+ return Files.createTempDirectory(prefix).toFile().getAbsolutePath();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static void deleteFile(String path) {
+ try {
+ Files.deleteIfExists(Paths.get(path));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static void deleteDir(String path) {
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestFSUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestFSUtils.java
new file mode 100644
index 00000000000..cdf32997777
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestFSUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+public class TestFSUtils {
+ @Test
+ public void testNormalizeURI() throws URISyntaxException {
+ URI uri = new URI("tos://abc/dir/key");
+ URI normalizeURI = FSUtils.normalizeURI(uri, new Configuration());
+ assertEquals("tos", normalizeURI.getScheme());
+ assertEquals("abc", normalizeURI.getAuthority());
+ assertEquals("abc", normalizeURI.getHost());
+ assertEquals("/dir/key", normalizeURI.getPath());
+
+ uri = new URI("/abc/dir/key");
+ normalizeURI = FSUtils.normalizeURI(uri, new Configuration());
+ assertNull(uri.getScheme());
+ assertEquals("file", normalizeURI.getScheme());
+ assertNull(uri.getAuthority());
+ assertNull(normalizeURI.getAuthority());
+ assertEquals("/abc/dir/key", uri.getPath());
+ assertEquals("/", normalizeURI.getPath());
+
+ uri = new URI("tos:///abc/dir/key");
+ normalizeURI = FSUtils.normalizeURI(uri, new Configuration());
+ assertEquals("tos", uri.getScheme());
+ assertNull(uri.getAuthority());
+ assertEquals("/abc/dir/key", uri.getPath());
+ assertEquals("tos", normalizeURI.getScheme());
+ assertNull(normalizeURI.getAuthority());
+ assertEquals("/abc/dir/key", normalizeURI.getPath());
+
+ Configuration conf = new Configuration();
+ conf.set(FS_DEFAULT_NAME_KEY, "tos://bucket/");
+ normalizeURI = FSUtils.normalizeURI(uri, conf);
+ assertEquals("tos", normalizeURI.getScheme());
+ assertEquals("bucket", normalizeURI.getAuthority());
+ assertEquals("/", normalizeURI.getPath());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java
new file mode 100644
index 00000000000..16ba414af07
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestIterables {
+
+ @Test
+ public void testTransform() {
+ List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+ Function<Integer, Integer> transform = i -> i + 10;
+ Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+ for (int i = 0; i < 5; i++) {
+ assertTrue(iter.hasNext());
+ int value = iter.next();
+ assertEquals(10 + i + 1, value);
+ }
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testTransformEmptyIterable() {
+ List<Integer> list = Arrays.asList();
+ Function<Integer, Integer> transform = i -> i + 10;
+ Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testFilter() {
+ // Filter odd elements.
+ List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+ Predicate<Integer> filter = i -> (i % 2) == 0;
+ Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+ for (int i = 0; i < 2; i++) {
+ assertTrue(iter.hasNext());
+ int value = iter.next();
+ assertEquals((i + 1) * 2, value);
+ }
+ assertFalse(iter.hasNext());
+
+ // Ignore all elements.
+ filter = i -> false;
+ iter = Iterables.filter(list, filter).iterator();
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testFilterEmptyIterable() {
+ List<Integer> list = Arrays.asList();
+ Predicate<Integer> filter = i -> (i % 2) == 0;
+ Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+ assertFalse(iter.hasNext());
+ }
+
+ // Full iterators.
+ @Test
+ public void testConcatFullIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<Integer> list = new ArrayList<>();
+ for (int j = 0; j < 10; j++) {
+ list.add(i * 10 + j);
+ expectedList.add(i * 10 + j);
+ }
+ iterList.add(list);
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Empty iterators.
+ @Test
+ public void testConcatEmptyIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ iterList.add(Collections.emptyList());
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Mix full and empty iterators.
+ @Test
+ public void testConcatMixFullAndEmptyIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<Integer> list = new ArrayList<>();
+ for (int j = 0; j < 10; j++) {
+ list.add(i * 10 + j);
+ expectedList.add(i * 10 + j);
+ }
+ iterList.add(list);
+ iterList.add(Collections.emptyList());
+ iterList.add(Collections.emptyList());
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Invalid iterators.
+ @Test
+ public void testConcatNullMetaIterator() {
+ assertThrows(NullPointerException.class, () -> verifyConcat(Collections.emptyIterator(), null),
+ "Expect null verification error.");
+ }
+
+ // Concat null iterators.
+ @Test
+ public void testConcatNullElementIterators() {
+ List<Iterable<Integer>> list = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ list.add(() -> null);
+ }
+ verifyConcat(Collections.emptyIterator(), list);
+ }
+
+ private <T> void verifyConcat(Iterator<T> expectedValues, Iterable<Iterable<T>> metaIter) {
+ Iterator<T> iter = Iterables.concat(metaIter).iterator();
+ while (expectedValues.hasNext()) {
+ assertTrue(iter.hasNext());
+ T v1 = expectedValues.next();
+ T v2 = iter.next();
+ assertEquals(v1, v2);
+ }
+ assertFalse(iter.hasNext());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestLazyReload.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestLazyReload.java
new file mode 100644
index 00000000000..c4c0d9ffb4d
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestLazyReload.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestLazyReload {
+ @Test
+ public void testLoadWithFilterCondition() {
+ LazyReload<Integer> integers = new LazyReload<>(() -> {
+ Iterator<Integer> source = Arrays.asList(1, 3, 5, 2, 4, 6).iterator();
+ return buf -> {
+ if (!source.hasNext()) {
+ return true;
+ }
+
+ int pollCnt = 2;
+ while (source.hasNext() && pollCnt-- > 0) {
+ Integer item = source.next();
+ if (item % 2 == 0) {
+ buf.add(item);
+ }
+ }
+
+ return !source.hasNext();
+ };
+ });
+
+ Iterator<Integer> iterator = integers.iterator();
+ assertTrue(iterator.hasNext());
+ assertEquals(2, (int) iterator.next());
+ assertEquals(4, (int) iterator.next());
+ assertEquals(6, (int) iterator.next());
+ assertFalse(iterator.hasNext());
+ }
+
+ @Test
+ public void testLoadResultIsIdempotent() {
+ LazyReload<Integer> integers = new LazyReload<>(() -> {
+ Iterator<Integer> source = Arrays.asList(1, 3, 5, 2, 4, 6).iterator();
+ return buf -> {
+ if (!source.hasNext()) {
+ return true;
+ }
+
+ int pollCnt = 2;
+ while (source.hasNext() && pollCnt-- > 0) {
+ Integer item = source.next();
+ buf.add(item);
+ }
+
+ return !source.hasNext();
+ };
+ });
+ Iterator<Integer> iterator1 = integers.iterator();
+ Iterator<Integer> iterator2 = integers.iterator();
+
+ assertEquals(1, (int) iterator1.next());
+ assertEquals(1, (int) iterator2.next());
+ assertEquals(3, (int) iterator1.next());
+ assertEquals(3, (int) iterator2.next());
+ assertEquals(5, (int) iterator1.next());
+ assertEquals(5, (int) iterator2.next());
+
+ assertEquals(2, (int) iterator1.next());
+ assertEquals(4, (int) iterator1.next());
+ assertEquals(6, (int) iterator1.next());
+ assertEquals(2, (int) iterator2.next());
+ assertEquals(4, (int) iterator2.next());
+ assertEquals(6, (int) iterator2.next());
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestRange.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestRange.java
new file mode 100644
index 00000000000..08fcae1e20c
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestRange.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestRange {
+
+ @Test
+ public void testInclude() {
+ Object[][] inputs = new Object[][]{
+ new Object[]{Range.of(0, 0), 0L, false},
+ new Object[]{Range.of(0, 1), 0L, true},
+ new Object[]{Range.of(1, 1), 0L, false},
+ new Object[]{Range.of(1, 1), 1L, true},
+ new Object[]{Range.of(1, 1), 2L, false},
+ new Object[]{Range.of(1, 99), 0L, false},
+ new Object[]{Range.of(1, 99), 1L, true},
+ new Object[]{Range.of(1, 99), 99L, true},
+ new Object[]{Range.of(1, 99), 100L, false}
+ };
+
+ for (Object[] input : inputs) {
+ Range r = (Range) input[0];
+ long pos = (long) input[1];
+ boolean expected = (boolean) input[2];
+
+ assertEquals(expected, r.include(pos));
+ }
+ }
+
+ @Test
+ public void testOverlap() {
+ Object[][] inputs = new Object[][]{
+ new Object[]{Range.of(0, 0), Range.of(0, 0), false},
+ new Object[]{Range.of(0, 1), Range.of(0, 1), true},
... 245 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org