You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2022/03/17 11:48:25 UTC
[hadoop] branch branch-3.3 updated (712d9be -> 3238bda)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 712d9be HDFS-16502. Reconfigure Block Invalidate limit (#4064)
new e06ed88 HADOOP-18162. hadoop-common support for MAPREDUCE-7341 Manifest Committer
new 1cc83f0 MAPREDUCE-7341. Add an intermediate manifest committer for Azure and GCS
new 3238bda HADOOP-18163. hadoop-azure support for the Manifest Committer of MAPREDUCE-7341
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/fs/statistics/StoreStatisticNames.java | 6 +
.../fs/statistics/impl/IOStatisticsBinding.java | 29 +-
.../fs/statistics/impl/IOStatisticsStore.java | 11 +
.../impl/PairedDurationTrackerFactory.java | 5 +
.../statistics/impl/StatisticDurationTracker.java | 7 +
.../org/apache/hadoop/util/JsonSerialization.java | 5 +-
.../java/org/apache/hadoop/util/RateLimiting.java | 52 +
.../apache/hadoop/util/RateLimitingFactory.java | 102 ++
.../functional/CloseableTaskPoolSubmitter.java | 74 +
.../apache/hadoop/util/functional/TaskPool.java | 613 +++++++
.../hadoop/util/functional/TestTaskPool.java | 585 +++++++
.../hadoop-mapreduce-client-core/pom.xml | 7 +-
.../lib/output/PathOutputCommitterFactory.java | 10 +-
.../committer/manifest/ManifestCommitter.java | 761 +++++++++
.../manifest/ManifestCommitterConfig.java | 375 ++++
.../manifest/ManifestCommitterConstants.java | 240 +++
.../manifest/ManifestCommitterFactory.java | 57 +
.../manifest/ManifestCommitterStatisticNames.java | 264 +++
.../manifest/files/AbstractManifestData.java | 137 ++
.../committer/manifest/files/DiagnosticKeys.java | 44 +
.../output/committer/manifest/files/DirEntry.java | 202 +++
.../committer/manifest/files/EntryStatus.java | 87 +
.../output/committer/manifest/files/FileEntry.java | 189 ++
.../committer/manifest/files/ManifestPrinter.java | 144 ++
.../manifest/files/ManifestSuccessData.java | 493 ++++++
.../committer/manifest/files/TaskManifest.java | 365 ++++
.../committer/manifest/files/package-info.java | 41 +
.../manifest/impl/AuditingIntegration.java | 94 +
.../committer/manifest/impl/InternalConstants.java | 130 ++
.../manifest/impl/ManifestCommitterSupport.java | 374 ++++
.../manifest/impl/ManifestStoreOperations.java | 291 ++++
.../ManifestStoreOperationsThroughFileSystem.java | 187 ++
.../manifest/impl/OutputValidationException.java | 40 +
.../committer/manifest/impl/package-info.java | 30 +
.../output/committer/manifest/package-info.java | 34 +
.../committer/manifest/stages/AbortTaskStage.java | 63 +
.../manifest/stages/AbstractJobOrTaskStage.java | 942 ++++++++++
.../committer/manifest/stages/CleanupJobStage.java | 511 ++++++
.../committer/manifest/stages/CommitJobStage.java | 245 +++
.../committer/manifest/stages/CommitTaskStage.java | 115 ++
.../stages/CreateOutputDirectoriesStage.java | 423 +++++
.../committer/manifest/stages/JobOrTaskStage.java | 33 +
.../manifest/stages/LoadManifestsStage.java | 293 ++++
.../manifest/stages/RenameFilesStage.java | 173 ++
.../manifest/stages/SaveSuccessFileStage.java | 79 +
.../manifest/stages/SaveTaskManifestStage.java | 81 +
.../committer/manifest/stages/SetupJobStage.java | 63 +
.../committer/manifest/stages/SetupTaskStage.java | 54 +
.../committer/manifest/stages/StageConfig.java | 556 ++++++
.../manifest/stages/StageEventCallbacks.java | 37 +
.../stages/TaskAttemptScanDirectoryStage.java | 214 +++
.../manifest/stages/ValidateRenamedFilesStage.java | 200 +++
.../committer/manifest/stages/package-info.java | 30 +
.../src/main/resources/mapred-default.xml | 19 +
.../src/site/markdown/manifest_committer.md | 605 +++++++
.../markdown/manifest_committer_architecture.md | 335 ++++
.../site/markdown/manifest_committer_protocol.md | 617 +++++++
.../manifest/AbstractManifestCommitterTest.java | 1085 ++++++++++++
.../committer/manifest/CommitterTestBinding.java | 152 ++
.../manifest/ManifestCommitterTestSupport.java | 420 +++++
.../committer/manifest/StubStoreOperations.java | 109 ++
.../committer/manifest/TaggedFileStatus.java | 54 +
.../committer/manifest/TestCleanupStage.java | 142 ++
.../committer/manifest/TestCommitTaskStage.java | 126 ++
.../manifest/TestCreateOutputDirectoriesStage.java | 307 ++++
.../manifest/TestJobThroughManifestCommitter.java | 601 +++++++
.../committer/manifest/TestLoadManifestsStage.java | 141 ++
.../manifest/TestManifestCommitProtocol.java | 1801 ++++++++++++++++++++
.../committer/manifest/TestRenameStageFailure.java | 379 ++++
.../committer/manifest/TestTaskManifestFileIO.java | 185 ++
.../committer/manifest/TextOutputForTests.java | 136 ++
.../committer/manifest/ThreadLeakTracker.java | 83 +
.../impl/UnreliableManifestStoreOperations.java | 380 +++++
.../output/committer/manifest/package-info.java | 29 +
.../src/test/resources/contract/localfs.xml | 138 ++
.../src/test/resources/core-site.xml | 33 +
hadoop-project/pom.xml | 7 +
hadoop-tools/hadoop-azure/pom.xml | 81 +-
.../src/config/checkstyle-suppressions.xml | 3 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 9 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 201 ++-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 34 +-
.../commit/AbfsManifestStoreOperations.java | 130 ++
.../commit/AzureManifestCommitterFactory.java | 58 +
.../azurebfs/commit/ResilientCommitByRename.java | 101 ++
.../hadoop/fs/azurebfs/commit/package-info.java | 28 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 3 +
.../constants/FileSystemConfigurations.java | 5 +
.../hadoop/fs/azurebfs/services/AbfsClient.java | 99 +-
.../fs/azurebfs/services/SimpleKeyProvider.java | 10 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 11 +-
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 13 +-
.../ITestAzureBlobFileSystemDelegationSAS.java | 5 +-
.../fs/azurebfs/ITestCustomerProvidedKey.java | 3 +-
.../fs/azurebfs/commit/AbfsCommitTestHelper.java | 49 +
.../azurebfs/commit/AbstractAbfsClusterITest.java | 260 +++
.../fs/azurebfs/commit/ITestAbfsCleanupStage.java | 54 +
.../azurebfs/commit/ITestAbfsCommitTaskStage.java | 54 +
.../ITestAbfsCreateOutputDirectoriesStage.java | 54 +
.../ITestAbfsJobThroughManifestCommitter.java | 101 ++
.../commit/ITestAbfsLoadManifestsStage.java | 55 +
.../commit/ITestAbfsManifestCommitProtocol.java | 62 +
.../commit/ITestAbfsManifestStoreOperations.java | 175 ++
.../commit/ITestAbfsRenameStageFailure.java | 69 +
.../commit/ITestAbfsTaskManifestFileIO.java | 54 +
.../fs/azurebfs/commit/ITestAbfsTerasort.java | 353 ++++
.../hadoop/fs/azurebfs/commit/package-info.java | 24 +
.../azurebfs/contract/AbfsFileSystemContract.java | 2 +-
.../hadoop-azure/src/test/resources/core-site.xml | 25 +
109 files changed, 20174 insertions(+), 92 deletions(-)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_protocol.md
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/AbstractManifestCommitterTest.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/CommitterTestBinding.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterTestSupport.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/StubStoreOperations.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TaggedFileStatus.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCleanupStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCommitTaskStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestCreateOutputDirectoriesStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestJobThroughManifestCommitter.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestLoadManifestsStage.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestTaskManifestFileIO.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TextOutputForTests.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ThreadLeakTracker.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/UnreliableManifestStoreOperations.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/contract/localfs.xml
create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/core-site.xml
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java
create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
create mode 100644 hadoop-tools/hadoop-azure/src/test/resources/core-site.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: MAPREDUCE-7341. Add an intermediate manifest committer for Azure and GCS
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1cc83f0f4595f2f527721239be92458fb9e4c147
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Mar 16 15:21:54 2022 +0000
MAPREDUCE-7341. Add an intermediate manifest committer for Azure and GCS
This is a mapreduce/spark output committer optimized for
performance and correctness on Azure ADLS Gen 2 storage
(via the abfs connector) and Google Cloud Storage
(via the external gcs connector library).
* It is safe to use with HDFS, however it has not been optimized
for that use.
* It is *not* safe for use with S3, and will fail if an attempt
is made to do so.
Contributed by Steve Loughran
Change-Id: I6f3502e79c578b9fd1a8c1485f826784b5421fca
---
.../hadoop-mapreduce-client-core/pom.xml | 7 +-
.../lib/output/PathOutputCommitterFactory.java | 10 +-
.../committer/manifest/ManifestCommitter.java | 761 +++++++++
.../manifest/ManifestCommitterConfig.java | 375 ++++
.../manifest/ManifestCommitterConstants.java | 240 +++
.../manifest/ManifestCommitterFactory.java | 57 +
.../manifest/ManifestCommitterStatisticNames.java | 264 +++
.../manifest/files/AbstractManifestData.java | 137 ++
.../committer/manifest/files/DiagnosticKeys.java | 44 +
.../output/committer/manifest/files/DirEntry.java | 202 +++
.../committer/manifest/files/EntryStatus.java | 87 +
.../output/committer/manifest/files/FileEntry.java | 189 ++
.../committer/manifest/files/ManifestPrinter.java | 144 ++
.../manifest/files/ManifestSuccessData.java | 493 ++++++
.../committer/manifest/files/TaskManifest.java | 365 ++++
.../committer/manifest/files/package-info.java | 41 +
.../manifest/impl/AuditingIntegration.java | 94 +
.../committer/manifest/impl/InternalConstants.java | 130 ++
.../manifest/impl/ManifestCommitterSupport.java | 374 ++++
.../manifest/impl/ManifestStoreOperations.java | 291 ++++
.../ManifestStoreOperationsThroughFileSystem.java | 187 ++
.../manifest/impl/OutputValidationException.java | 40 +
.../committer/manifest/impl/package-info.java | 30 +
.../output/committer/manifest/package-info.java | 34 +
.../committer/manifest/stages/AbortTaskStage.java | 63 +
.../manifest/stages/AbstractJobOrTaskStage.java | 942 ++++++++++
.../committer/manifest/stages/CleanupJobStage.java | 511 ++++++
.../committer/manifest/stages/CommitJobStage.java | 245 +++
.../committer/manifest/stages/CommitTaskStage.java | 115 ++
.../stages/CreateOutputDirectoriesStage.java | 423 +++++
.../committer/manifest/stages/JobOrTaskStage.java | 33 +
.../manifest/stages/LoadManifestsStage.java | 293 ++++
.../manifest/stages/RenameFilesStage.java | 173 ++
.../manifest/stages/SaveSuccessFileStage.java | 79 +
.../manifest/stages/SaveTaskManifestStage.java | 81 +
.../committer/manifest/stages/SetupJobStage.java | 63 +
.../committer/manifest/stages/SetupTaskStage.java | 54 +
.../committer/manifest/stages/StageConfig.java | 556 ++++++
.../manifest/stages/StageEventCallbacks.java | 37 +
.../stages/TaskAttemptScanDirectoryStage.java | 214 +++
.../manifest/stages/ValidateRenamedFilesStage.java | 200 +++
.../committer/manifest/stages/package-info.java | 30 +
.../src/main/resources/mapred-default.xml | 19 +
.../src/site/markdown/manifest_committer.md | 605 +++++++
.../markdown/manifest_committer_architecture.md | 335 ++++
.../site/markdown/manifest_committer_protocol.md | 617 +++++++
.../manifest/AbstractManifestCommitterTest.java | 1085 ++++++++++++
.../committer/manifest/CommitterTestBinding.java | 152 ++
.../manifest/ManifestCommitterTestSupport.java | 420 +++++
.../committer/manifest/StubStoreOperations.java | 109 ++
.../committer/manifest/TaggedFileStatus.java | 54 +
.../committer/manifest/TestCleanupStage.java | 142 ++
.../committer/manifest/TestCommitTaskStage.java | 126 ++
.../manifest/TestCreateOutputDirectoriesStage.java | 307 ++++
.../manifest/TestJobThroughManifestCommitter.java | 601 +++++++
.../committer/manifest/TestLoadManifestsStage.java | 141 ++
.../manifest/TestManifestCommitProtocol.java | 1801 ++++++++++++++++++++
.../committer/manifest/TestRenameStageFailure.java | 379 ++++
.../committer/manifest/TestTaskManifestFileIO.java | 185 ++
.../committer/manifest/TextOutputForTests.java | 136 ++
.../committer/manifest/ThreadLeakTracker.java | 83 +
.../impl/UnreliableManifestStoreOperations.java | 380 +++++
.../output/committer/manifest/package-info.java | 29 +
.../src/test/resources/contract/localfs.xml | 138 ++
.../src/test/resources/core-site.xml | 33 +
65 files changed, 16579 insertions(+), 6 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 95c8c66..4c40cad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -102,9 +102,14 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <configuration>
+ <configuration>
<systemPropertyVariables>
<log4j.configuration>file:///${project.basedir}/src/test/resources/log4j.properties</log4j.configuration>
+
+ <!-- project directory (target/) on a maven build. -->
+ <project.build.directory>${project.build.directory}</project.build.directory>
+ <!-- this property is used in hadoop-azure with separate paths for parallel runs -->
+ <test.build.dir>${project.build.directory}</test.build.dir>
</systemPropertyVariables>
</configuration>
</plugin>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
index f3f0dab..cf83bf2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
@@ -52,8 +52,8 @@ import org.apache.hadoop.util.ReflectionUtils;
* attempt context and a possibly null path.
*
*/
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class PathOutputCommitterFactory extends Configured {
private static final Logger LOG =
LoggerFactory.getLogger(PathOutputCommitterFactory.class);
@@ -158,7 +158,7 @@ public class PathOutputCommitterFactory extends Configured {
scheme);
if (StringUtils.isNotEmpty(conf.getTrimmed(schemeKey))) {
// it does, so use that key in the classname lookup
- LOG.debug("Using schema-specific factory for {}", outputPath);
+ LOG.info("Using schema-specific factory for {}", outputPath);
key = schemeKey;
} else {
LOG.debug("No scheme-specific factory defined in {}", schemeKey);
@@ -171,7 +171,7 @@ public class PathOutputCommitterFactory extends Configured {
String trimmedValue = conf.getTrimmed(key, "");
if (StringUtils.isEmpty(trimmedValue)) {
// empty/null value, use default
- LOG.debug("No output committer factory defined,"
+ LOG.info("No output committer factory defined,"
+ " defaulting to FileOutputCommitterFactory");
factory = FileOutputCommitterFactory.class;
} else {
@@ -179,7 +179,7 @@ public class PathOutputCommitterFactory extends Configured {
factory = conf.getClass(key,
FileOutputCommitterFactory.class,
PathOutputCommitterFactory.class);
- LOG.debug("Using OutputCommitter factory class {} from key {}",
+ LOG.info("Using OutputCommitter factory class {} from key {}",
factory, key);
}
return ReflectionUtils.newInstance(factory, conf);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java
new file mode 100644
index 0000000..99625e8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java
@@ -0,0 +1,761 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.AbortTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CommitTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupJobStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.SetupTaskStage;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageEventCallbacks;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtDebug;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_DIAGNOSTICS_MANIFEST_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_SUMMARY_REPORT_DIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_COMPLETED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_FAILED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME_RECOVERED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_ABORT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.updateCommonContextOnCommitterExit;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.updateCommonContextOnCommitterEntry;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createIOStatisticsStore;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createJobSummaryFilename;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createManifestOutcome;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestPathForTask;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.CleanupJobStage.cleanupStageOptionsFromConfig;
+
+/**
+ * This is the Intermediate-Manifest committer.
+ * At every entry point it updates the thread's audit context with
+ * the current stage info; this is a placeholder for
+ * adding audit information to stores other than S3A.
+ *
+ * This is tagged as public/stable. This is mandatory
+ * for the classname and PathOutputCommitter implementation
+ * classes.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ManifestCommitter extends PathOutputCommitter implements
+ IOStatisticsSource, StageEventCallbacks {
+
+ public static final Logger LOG = LoggerFactory.getLogger(
+ ManifestCommitter.class);
+
+ /**
+ * Role: task committer.
+ */
+ public static final String TASK_COMMITTER = "task committer";
+
+ /**
+ * Role: job committer.
+ */
+ public static final String JOB_COMMITTER = "job committer";
+
+ /**
+ * Committer Configuration as extracted from
+ * the job/task context and set in the constructor.
+ */
+ private final ManifestCommitterConfig baseConfig;
+
+ /**
+ * Destination of the job.
+ */
+ private final Path destinationDir;
+
+ /**
+ * For tasks, the attempt directory.
+ * Null for jobs.
+ */
+ private final Path taskAttemptDir;
+
+ /**
+ * IOStatistics to update.
+ */
+ private final IOStatisticsStore iostatistics;
+
+ /**
+ * The job Manifest Success data; only valid after a job successfully
+ * commits.
+ */
+ private ManifestSuccessData successReport;
+
+ /**
+ * The active stage; is updated by a callback from within the stages.
+ */
+ private String activeStage;
+
+ /**
+ * The task manifest of the task commit.
+ * Null unless this is a task attempt and the
+ * task has successfully been committed.
+ */
+ private TaskManifest taskAttemptCommittedManifest;
+
+ /**
+ * Create a committer.
+ * @param outputPath output path
+ * @param context job/task context
+ * @throws IOException failure.
+ */
+ public ManifestCommitter(final Path outputPath,
+ final TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ this.destinationDir = resolveDestinationDirectory(outputPath,
+ context.getConfiguration());
+ this.iostatistics = createIOStatisticsStore().build();
+ this.baseConfig = enterCommitter(
+ context.getTaskAttemptID() != null,
+ context);
+
+ this.taskAttemptDir = baseConfig.getTaskAttemptDir();
+ LOG.info("Created ManifestCommitter with JobID {},"
+ + " Task Attempt {} and destination {}",
+ context.getJobID(), context.getTaskAttemptID(), outputPath);
+ }
+
+ /**
+ * Committer method invoked; generates a config for it.
+ * Calls {@code #updateCommonContextOnCommitterEntry()}
+ * to update the audit context.
+ * @param isTask is this a task entry point?
+ * @param context context
+ * @return committer config
+ */
+ private ManifestCommitterConfig enterCommitter(boolean isTask,
+ JobContext context) {
+ ManifestCommitterConfig committerConfig =
+ new ManifestCommitterConfig(
+ getOutputPath(),
+ isTask ? TASK_COMMITTER : JOB_COMMITTER,
+ context,
+ iostatistics,
+ this);
+ updateCommonContextOnCommitterEntry(committerConfig);
+ return committerConfig;
+ }
+
+ /**
+ * Set up a job through a {@link SetupJobStage}.
+ * @param jobContext Context of the job whose output is being written.
+ * @throws IOException IO Failure.
+ */
+ @Override
+ public void setupJob(final JobContext jobContext) throws IOException {
+ ManifestCommitterConfig committerConfig = enterCommitter(false,
+ jobContext);
+ StageConfig stageConfig =
+ committerConfig
+ .createStageConfig()
+ .withOperations(createManifestStoreOperations())
+ .build();
+ // set up the job.
+ new SetupJobStage(stageConfig)
+ .apply(committerConfig.getCreateJobMarker());
+ logCommitterStatisticsAtDebug();
+ }
+
+ /**
+ * Set up a task through a {@link SetupTaskStage}.
+ * Classic FileOutputCommitter is a no-op here, relying
+ * on RecordWriters to create the dir implicitly on file
+ * create().
+ * FileOutputCommitter also uses the existence of that
+ * file as a flag to indicate task commit is needed.
+ * @param context task context.
+ * @throws IOException IO Failure.
+ */
+ @Override
+ public void setupTask(final TaskAttemptContext context)
+ throws IOException {
+ ManifestCommitterConfig committerConfig =
+ enterCommitter(true, context);
+ StageConfig stageConfig =
+ committerConfig
+ .createStageConfig()
+ .withOperations(createManifestStoreOperations())
+ .build();
+ // create task attempt dir; delete if present. Or fail?
+ new SetupTaskStage(stageConfig).apply("");
+ logCommitterStatisticsAtDebug();
+ }
+
+ /**
+ * Always return true.
+ * This way, even if there is no output, stats are collected.
+ * @param context task context.
+ * @return true
+ * @throws IOException IO Failure.
+ */
+ @Override
+ public boolean needsTaskCommit(final TaskAttemptContext context)
+ throws IOException {
+ LOG.info("Probe for needsTaskCommit({})",
+ context.getTaskAttemptID());
+ return true;
+ }
+
+ /**
+ * Failure during Job Commit is not recoverable from.
+ *
+ * @param jobContext
+ * Context of the job whose output is being written.
+ * @return false, always
+ * @throws IOException never
+ */
+ @Override
+ public boolean isCommitJobRepeatable(final JobContext jobContext)
+ throws IOException {
+ LOG.info("Probe for isCommitJobRepeatable({}): returning false",
+ jobContext.getJobID());
+ return false;
+ }
+
+ /**
+ * Declare that task recovery is not supported.
+ * It would be, if someone added the code *and tests*.
+ * @param jobContext
+ * Context of the job whose output is being written.
+ * @return false, always
+ * @throws IOException never
+ */
+ @Override
+ public boolean isRecoverySupported(final JobContext jobContext)
+ throws IOException {
+ LOG.info("Probe for isRecoverySupported({}): returning false",
+ jobContext.getJobID());
+ return false;
+ }
+
+ /**
+ *
+ * @param taskContext Context of the task whose output is being recovered
+ * @throws IOException always
+ */
+ @Override
+ public void recoverTask(final TaskAttemptContext taskContext)
+ throws IOException {
+ LOG.warn("Rejecting recoverTask({}) call", taskContext.getTaskAttemptID());
+ throw new IOException("Cannot recover task "
+ + taskContext.getTaskAttemptID());
+ }
+
+ /**
+ * Commit the task.
+ * This is where the task attempt tree list takes place.
+ * @param context task context.
+ * @throws IOException IO Failure.
+ */
+ @Override
+ public void commitTask(final TaskAttemptContext context)
+ throws IOException {
+ ManifestCommitterConfig committerConfig = enterCommitter(true,
+ context);
+ try {
+ StageConfig stageConfig = committerConfig.createStageConfig()
+ .withOperations(createManifestStoreOperations())
+ .build();
+ taskAttemptCommittedManifest = new CommitTaskStage(stageConfig)
+ .apply(null).getTaskManifest();
+ iostatistics.incrementCounter(COMMITTER_TASKS_COMPLETED_COUNT, 1);
+ } catch (IOException e) {
+ iostatistics.incrementCounter(COMMITTER_TASKS_FAILED_COUNT, 1);
+ throw e;
+ } finally {
+ logCommitterStatisticsAtDebug();
+ updateCommonContextOnCommitterExit();
+ }
+
+ }
+
+ /**
+ * Abort a task.
+ * @param context task context
+ * @throws IOException failure during the delete
+ */
+ @Override
+ public void abortTask(final TaskAttemptContext context)
+ throws IOException {
+ ManifestCommitterConfig committerConfig = enterCommitter(true,
+ context);
+ try {
+ new AbortTaskStage(
+ committerConfig.createStageConfig()
+ .withOperations(createManifestStoreOperations())
+ .build())
+ .apply(false);
+ } finally {
+ logCommitterStatisticsAtDebug();
+ updateCommonContextOnCommitterExit();
+ }
+ }
+
+ /**
+ * Get the manifest success data for this job; creating on demand if needed.
+ * @param committerConfig source config.
+ * @return the current {@link #successReport} value; never null.
+ */
+ private ManifestSuccessData getOrCreateSuccessData(
+ ManifestCommitterConfig committerConfig) {
+ if (successReport == null) {
+ successReport = createManifestOutcome(
+ committerConfig.createStageConfig(), activeStage);
+ }
+ return successReport;
+ }
+
+ /**
+ * This is the big job commit stage.
+ * Load the manifests, prepare the destination, rename
+ * the files then cleanup the job directory.
+ * @param jobContext Context of the job whose output is being written.
+ * @throws IOException failure.
+ */
+ @Override
+ public void commitJob(final JobContext jobContext) throws IOException {
+
+ ManifestCommitterConfig committerConfig = enterCommitter(false, jobContext);
+
+ // create the initial success data.
+ // this is overwritten by that created during the operation sequence,
+ // but if the sequence fails before that happens, it
+ // will be saved to the report directory.
+ ManifestSuccessData marker = getOrCreateSuccessData(committerConfig);
+ IOException failure = null;
+ try (CloseableTaskPoolSubmitter ioProcs =
+ committerConfig.createSubmitter();
+ ManifestStoreOperations storeOperations = createManifestStoreOperations()) {
+ // the stage config will be shared across all stages.
+ StageConfig stageConfig = committerConfig.createStageConfig()
+ .withOperations(storeOperations)
+ .withIOProcessors(ioProcs)
+ .build();
+
+ // commit the job, including any cleanup and validation.
+ final Configuration conf = jobContext.getConfiguration();
+ CommitJobStage.Result result = new CommitJobStage(stageConfig).apply(
+ new CommitJobStage.Arguments(
+ committerConfig.getCreateJobMarker(),
+ committerConfig.getValidateOutput(),
+ conf.getTrimmed(OPT_DIAGNOSTICS_MANIFEST_DIR, ""),
+ cleanupStageOptionsFromConfig(
+ OP_STAGE_JOB_CLEANUP, conf)
+ ));
+ marker = result.getJobSuccessData();
+ // update the cached success with the new report.
+ setSuccessReport(marker);
+
+ } catch (IOException e) {
+ // failure. record it for the summary
+ failure = e;
+ // rethrow
+ throw e;
+ } finally {
+ // save the report summary, even on failure
+ maybeSaveSummary(activeStage,
+ committerConfig,
+ marker,
+ failure,
+ true,
+ true);
+ // print job commit stats
+ LOG.info("{}: Job Commit statistics {}",
+ committerConfig.getName(),
+ ioStatisticsToPrettyString(iostatistics));
+ // and warn of rename problems
+ final Long recoveries = iostatistics.counters().get(OP_COMMIT_FILE_RENAME_RECOVERED);
+ if (recoveries != null && recoveries > 0) {
+ LOG.warn("{}: rename failures were recovered from. Number of recoveries: {}",
+ committerConfig.getName(), recoveries);
+ }
+ updateCommonContextOnCommitterExit();
+ }
+ }
+
+ /**
+ * Abort the job.
+ * Invokes
+ * {@link #executeCleanup(String, JobContext, ManifestCommitterConfig)}
+ * then saves the (ongoing) job report data if reporting is enabled.
+ * @param jobContext Context of the job whose output is being written.
+ * @param state final runstate of the job
+ * @throws IOException failure during cleanup; report failure are swallowed
+ */
+ @Override
+ public void abortJob(final JobContext jobContext,
+ final JobStatus.State state)
+ throws IOException {
+ LOG.info("Aborting Job {} in state {}", jobContext.getJobID(), state);
+ ManifestCommitterConfig committerConfig = enterCommitter(false,
+ jobContext);
+ ManifestSuccessData report = getOrCreateSuccessData(
+ committerConfig);
+ IOException failure = null;
+
+ try {
+ executeCleanup(OP_STAGE_JOB_ABORT, jobContext, committerConfig);
+ } catch (IOException e) {
+ // failure.
+ failure = e;
+ }
+ report.setSuccess(false);
+ // job abort does not overwrite any existing report, so a job commit
+ // failure cause will be preserved.
+ maybeSaveSummary(activeStage, committerConfig, report, failure,
+ true, false);
+ // print job stats
+ LOG.info("Job Abort statistics {}",
+ ioStatisticsToPrettyString(iostatistics));
+ updateCommonContextOnCommitterExit();
+ }
+
+ /**
+ * Execute the {@code CleanupJobStage} to remove the job attempt dir.
+ * This does
+ * @param jobContext Context of the job whose output is being written.
+ * @throws IOException failure during cleanup
+ */
+ @SuppressWarnings("deprecation")
+ @Override
+ public void cleanupJob(final JobContext jobContext) throws IOException {
+ ManifestCommitterConfig committerConfig = enterCommitter(false,
+ jobContext);
+ try {
+ executeCleanup(OP_STAGE_JOB_CLEANUP, jobContext, committerConfig);
+ } finally {
+ logCommitterStatisticsAtDebug();
+ updateCommonContextOnCommitterExit();
+ }
+ }
+
+ /**
+ * Perform the cleanup operation for job cleanup or abort.
+ * @param statisticName statistic/stage name
+ * @param jobContext job context
+ * @param committerConfig committer config
+ * @throws IOException failure
+ * @return the outcome
+ */
+ private CleanupJobStage.Result executeCleanup(
+ final String statisticName,
+ final JobContext jobContext,
+ final ManifestCommitterConfig committerConfig) throws IOException {
+ try (CloseableTaskPoolSubmitter ioProcs =
+ committerConfig.createSubmitter()) {
+
+ return new CleanupJobStage(
+ committerConfig.createStageConfig()
+ .withOperations(createManifestStoreOperations())
+ .withIOProcessors(ioProcs)
+ .build())
+ .apply(cleanupStageOptionsFromConfig(
+ statisticName,
+ jobContext.getConfiguration()));
+ }
+ }
+
+ /**
+ * Output path: destination directory of the job.
+ * @return the overall job destination directory.
+ */
+ @Override
+ public Path getOutputPath() {
+ return getDestinationDir();
+ }
+
+ /**
+ * Work path of the current task attempt.
+ * This is null if the task does not have one.
+ * @return a path.
+ */
+ @Override
+ public Path getWorkPath() {
+ return getTaskAttemptDir();
+ }
+
+ /**
+ * Get the job destination dir.
+ * @return dest dir.
+ */
+ private Path getDestinationDir() {
+ return destinationDir;
+ }
+
+ /**
+ * Get the task attempt dir.
+ * May be null.
+ * @return a path or null.
+ */
+ private Path getTaskAttemptDir() {
+ return taskAttemptDir;
+ }
+
+ /**
+ * Callback on stage entry.
+ * Sets {@link #activeStage} and updates the
+ * common context.
+ * @param stage new stage
+ */
+ @Override
+ public void enterStage(String stage) {
+ activeStage = stage;
+ AuditingIntegration.enterStage(stage);
+ }
+
+ /**
+ * Remove stage from common audit context.
+ * @param stage stage exited.
+ */
+ @Override
+ public void exitStage(String stage) {
+ AuditingIntegration.exitStage();
+ }
+
+ /**
+ * Get the unique ID of this job.
+ * @return job ID (yarn, spark)
+ */
+ public String getJobUniqueId() {
+ return baseConfig.getJobUniqueId();
+ }
+
+ /**
+ * Get the config of the task attempt this instance was constructed
+ * with.
+ * @return a configuration.
+ */
+ public Configuration getConf() {
+ return baseConfig.getConf();
+ }
+
+ /**
+ * Get the manifest Success data; only valid after a job.
+ * @return the job _SUCCESS data, or null.
+ */
+ public ManifestSuccessData getSuccessReport() {
+ return successReport;
+ }
+
+ private void setSuccessReport(ManifestSuccessData successReport) {
+ this.successReport = successReport;
+ }
+
+ /**
+ * Get the manifest of the last committed task.
+ * @return a task manifest or null.
+ */
+ @VisibleForTesting
+ TaskManifest getTaskAttemptCommittedManifest() {
+ return taskAttemptCommittedManifest;
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed.
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ @VisibleForTesting
+ public Path getTaskAttemptPath(TaskAttemptContext context) {
+ return enterCommitter(false, context).getTaskAttemptDir();
+ }
+
+ /**
+ * The path to where the manifest file of a task attempt will be
+ * saved when the task is committed.
+ * This path will be the same for all attempts of the same task.
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ @VisibleForTesting
+ public Path getTaskManifestPath(TaskAttemptContext context) {
+ final Path dir = enterCommitter(false, context).getTaskManifestDir();
+
+ return manifestPathForTask(dir,
+ context.getTaskAttemptID().getTaskID().toString());
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed.
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ @VisibleForTesting
+ public Path getJobAttemptPath(JobContext context) {
+
+ return enterCommitter(false, context).getJobAttemptDir();
+ }
+
+ /**
+ * Get the final output path, including resolving any relative path.
+ * @param outputPath output path
+ * @param conf configuration to create any FS with
+ * @return a resolved path.
+ * @throws IOException failure.
+ */
+ private Path resolveDestinationDirectory(Path outputPath,
+ Configuration conf) throws IOException {
+ return FileSystem.get(outputPath.toUri(), conf).makeQualified(outputPath);
+ }
+
+ /**
+ * Create manifest store operations for the destination store.
+ * This MUST NOT be used for the success report operations, as
+ * they may be to a different filesystem.
+ * This is a point which can be overridden during testing.
+ * @return a new store operations instance bonded to the destination fs.
+ * @throws IOException failure to instantiate.
+ */
+ protected ManifestStoreOperations createManifestStoreOperations() throws IOException {
+ return ManifestCommitterSupport.createManifestStoreOperations(
+ baseConfig.getConf(),
+ baseConfig.getDestinationFileSystem(),
+ baseConfig.getDestinationDir());
+ }
+
+ /**
+ * Log IO Statistics at debug.
+ */
+ private void logCommitterStatisticsAtDebug() {
+ logIOStatisticsAtDebug(LOG, "Committer Statistics", this);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ManifestCommitter{");
+ sb.append(baseConfig);
+ sb.append(", iostatistics=").append(ioStatisticsToPrettyString(iostatistics));
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Save a summary to the report dir if the config option
+ * is set.
+ * The IOStatistics of the summary will be updated to the latest
+ * snapshot of the committer's statistics, so the report is up
+ * to date.
+ * The report will updated with the current active stage,
+ * and if {@code thrown} is non-null, it will be added to the
+ * diagnistics (and the job tagged as a failure).
+ * Static for testability.
+ * @param activeStage active stage
+ * @param config configuration to use.
+ * @param report summary file.
+ * @param thrown any exception indicting failure.
+ * @param quiet should exceptions be swallowed.
+ * @param overwrite should the existing file be overwritten
+ * @return the path of a file, if successfully saved
+ * @throws IOException if a failure occured and quiet==false
+ */
+ private static Path maybeSaveSummary(
+ String activeStage,
+ ManifestCommitterConfig config,
+ ManifestSuccessData report,
+ Throwable thrown,
+ boolean quiet,
+ boolean overwrite) throws IOException {
+ Configuration conf = config.getConf();
+ String reportDir = conf.getTrimmed(OPT_SUMMARY_REPORT_DIR, "");
+ if (reportDir.isEmpty()) {
+ LOG.debug("No summary directory set in " + OPT_SUMMARY_REPORT_DIR);
+ return null;
+ }
+ LOG.debug("Summary directory set in to {}" + OPT_SUMMARY_REPORT_DIR,
+ reportDir);
+
+ // update to the latest statistics
+ report.snapshotIOStatistics(config.getIOStatistics());
+
+ Path reportDirPath = new Path(reportDir);
+ Path path = new Path(reportDirPath,
+ createJobSummaryFilename(config.getJobUniqueId()));
+
+ if (thrown != null) {
+ report.recordJobFailure(thrown);
+ }
+ report.putDiagnostic(STAGE, activeStage);
+ // the store operations here is explicitly created for the FS where
+ // the reports go, which may not be the target FS of the job.
+
+ final FileSystem fs = path.getFileSystem(conf);
+ try (ManifestStoreOperations operations = new ManifestStoreOperationsThroughFileSystem(fs)) {
+ if (!overwrite) {
+ // check for file existence so there is no need to worry about
+ // precisely what exception is raised when overwrite=false and dest file
+ // exists
+ try {
+ FileStatus st = operations.getFileStatus(path);
+ // get here and the file exists
+ LOG.debug("Report already exists: {}", st);
+ return null;
+ } catch (FileNotFoundException ignored) {
+ }
+ }
+ operations.save(report, path, overwrite);
+ LOG.info("Job summary saved to {}", path);
+ return path;
+ } catch (IOException e) {
+ LOG.debug("Failed to save summary to {}", path, e);
+ if (quiet) {
+ return null;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public IOStatisticsStore getIOStatistics() {
+ return iostatistics;
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java
new file mode 100644
index 0000000..40dffee
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConfig.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageEventCallbacks;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter;
+
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.SUCCESSFUL_JOB_OUTPUT_DIR_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.*;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.buildJobUUID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.getAppAttemptId;
+
+/**
+ * The configuration for the committer as built up from the job configuration
+ * and data passed down from the committer factory.
+ * Isolated for ease of dev/test
+ */
+public final class ManifestCommitterConfig implements IOStatisticsSource {
+
+ /**
+ * Final destination of work.
+ * This is <i>unqualified</i>.
+ */
+ private final Path destinationDir;
+
+ /**
+ * Role: used in log/text messages.
+ */
+ private final String role;
+
+ /**
+ * This is the directory for all intermediate work: where the output
+ * format will write data.
+ * Will be null if built from a job context.
+ */
+ private final Path taskAttemptDir;
+
+ /** Configuration of the job. */
+ private final Configuration conf;
+
+ /** The job context. For a task, this can be cast to a TaskContext. */
+ private final JobContext jobContext;
+
+ /** Should a job marker be created? */
+ private final boolean createJobMarker;
+
+ /**
+ * Job ID Or UUID -without any attempt suffix.
+ * This is expected/required to be unique, though
+ * Spark has had "issues" there until recently
+ * with lack of uniqueness of generated MR Job IDs.
+ */
+ private final String jobUniqueId;
+
+ /**
+ * Where did the job Unique ID come from?
+ */
+ private final String jobUniqueIdSource;
+
+ /**
+ * Number of this attempt; starts at zero.
+ */
+ private final int jobAttemptNumber;
+
+ /**
+ * Job ID + AttemptID.
+ */
+ private final String jobAttemptId;
+
+ /**
+ * Task ID: used as the filename of the manifest.
+ * Will be "" if built from a job context.
+ */
+ private final String taskId;
+
+ /**
+ * Task attempt ID. Determines the working
+ * directory for task attempts to write data into,
+ * and for the task committer to scan.
+ * Will be "" if built from a job context.
+ */
+ private final String taskAttemptId;
+
+ /** Any progressable for progress callbacks. */
+ private final Progressable progressable;
+
+ /**
+ * IOStatistics to update.
+ */
+ private final IOStatisticsStore iostatistics;
+
+
+ /** Should the output be validated after the commit? */
+ private final boolean validateOutput;
+
+ /**
+ * Attempt directory management.
+ */
+ private final ManifestCommitterSupport.AttemptDirectories dirs;
+
+ /**
+ * Callback when a stage is entered.
+ */
+ private final StageEventCallbacks stageEventCallbacks;
+
+ /**
+ * Name for logging.
+ */
+ private final String name;
+
+ /**
+ * Delete target paths on commit? Stricter, but
+ * higher IO cost.
+ */
+ private final boolean deleteTargetPaths;
+
+ /**
+ * Constructor.
+ * @param outputPath destination path of the job.
+ * @param role role for log messages.
+ * @param context job/task context
+ * @param iostatistics IO Statistics
+ * @param stageEventCallbacks stage event callbacks.
+ */
+
+ ManifestCommitterConfig(
+ final Path outputPath,
+ final String role,
+ final JobContext context,
+ final IOStatisticsStore iostatistics,
+ final StageEventCallbacks stageEventCallbacks) {
+ this.role = role;
+ this.jobContext = context;
+ this.conf = context.getConfiguration();
+ this.destinationDir = outputPath;
+ this.iostatistics = iostatistics;
+ this.stageEventCallbacks = stageEventCallbacks;
+
+ Pair<String, String> pair = buildJobUUID(conf, context.getJobID());
+ this.jobUniqueId = pair.getLeft();
+ this.jobUniqueIdSource = pair.getRight();
+ this.jobAttemptNumber = getAppAttemptId(context);
+ this.jobAttemptId = this.jobUniqueId + "_" + jobAttemptNumber;
+
+ // build directories
+ this.dirs = new ManifestCommitterSupport.AttemptDirectories(outputPath,
+ this.jobUniqueId, jobAttemptNumber);
+
+ // read in configuration options
+ this.createJobMarker = conf.getBoolean(
+ SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
+ DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER);
+ this.validateOutput = conf.getBoolean(
+ OPT_VALIDATE_OUTPUT,
+ OPT_VALIDATE_OUTPUT_DEFAULT);
+ this.deleteTargetPaths = conf.getBoolean(
+ OPT_DELETE_TARGET_FILES,
+ OPT_DELETE_TARGET_FILES_DEFAULT);
+
+ // if constructed with a task attempt, build the task ID and path.
+ if (context instanceof TaskAttemptContext) {
+ // it's a task
+ final TaskAttemptContext tac = (TaskAttemptContext) context;
+ TaskAttemptID taskAttempt = Objects.requireNonNull(
+ tac.getTaskAttemptID());
+ taskAttemptId = taskAttempt.toString();
+ taskId = taskAttempt.getTaskID().toString();
+ // Task attempt dir; must be different across instances
+ taskAttemptDir = dirs.getTaskAttemptPath(taskAttemptId);
+ // the context is also the progress callback.
+ progressable = tac;
+ name = String.format(InternalConstants.NAME_FORMAT_TASK_ATTEMPT, taskAttemptId);
+
+ } else {
+ // it's a job
+ taskId = "";
+ taskAttemptId = "";
+ taskAttemptDir = null;
+ progressable = null;
+ name = String.format(InternalConstants.NAME_FORMAT_JOB_ATTEMPT, jobAttemptId);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "ManifestCommitterConfig{" +
+ "name=" + name +
+ ", destinationDir=" + destinationDir +
+ ", role='" + role + '\'' +
+ ", taskAttemptDir=" + taskAttemptDir +
+ ", createJobMarker=" + createJobMarker +
+ ", jobUniqueId='" + jobUniqueId + '\'' +
+ ", jobUniqueIdSource='" + jobUniqueIdSource + '\'' +
+ ", jobAttemptNumber=" + jobAttemptNumber +
+ ", jobAttemptId='" + jobAttemptId + '\'' +
+ ", taskId='" + taskId + '\'' +
+ ", taskAttemptId='" + taskAttemptId + '\'' +
+ '}';
+ }
+
+ /**
+ * Get the destination filesystem.
+ * @return destination FS.
+ * @throws IOException Problems binding to the destination FS.
+ */
+ FileSystem getDestinationFileSystem() throws IOException {
+ return FileSystem.get(destinationDir.toUri(), conf);
+ }
+
+ /**
+ * Create the stage config from the committer
+ * configuration.
+ * This does not bind the store operations
+ * or processors.
+ * @return a stage config with configuration options passed in.
+ */
+ StageConfig createStageConfig() {
+ StageConfig stageConfig = new StageConfig();
+ stageConfig
+ .withIOStatistics(iostatistics)
+ .withJobAttemptNumber(jobAttemptNumber)
+ .withJobDirectories(dirs)
+ .withJobId(jobUniqueId)
+ .withJobIdSource(jobUniqueIdSource)
+ .withName(name)
+ .withProgressable(progressable)
+ .withStageEventCallbacks(stageEventCallbacks)
+ .withTaskAttemptDir(taskAttemptDir)
+ .withTaskAttemptId(taskAttemptId)
+ .withTaskId(taskId)
+ .withDeleteTargetPaths(deleteTargetPaths);
+
+ return stageConfig;
+ }
+
+ public Path getDestinationDir() {
+ return destinationDir;
+ }
+
+ public String getRole() {
+ return role;
+ }
+
+ public Path getTaskAttemptDir() {
+ return taskAttemptDir;
+ }
+
+ public Path getJobAttemptDir() {
+ return dirs.getJobAttemptDir();
+ }
+
+ public Path getTaskManifestDir() {
+ return dirs.getTaskManifestDir();
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public JobContext getJobContext() {
+ return jobContext;
+ }
+
+ public boolean getCreateJobMarker() {
+ return createJobMarker;
+ }
+
+ public String getJobAttemptId() {
+ return jobAttemptId;
+ }
+
+ public String getTaskAttemptId() {
+ return taskAttemptId;
+ }
+
+ public String getTaskId() {
+ return taskId;
+ }
+
+ public String getJobUniqueId() {
+ return jobUniqueId;
+ }
+
+ public boolean getValidateOutput() {
+ return validateOutput;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public IOStatisticsStore getIOStatistics() {
+ return iostatistics;
+ }
+
+ /**
+ * Create a new submitter task pool from the
+ * {@link ManifestCommitterConstants#OPT_IO_PROCESSORS}
+ * settings.
+ * @return a new thread pool.
+ */
+ public CloseableTaskPoolSubmitter createSubmitter() {
+ return createSubmitter(
+ OPT_IO_PROCESSORS, OPT_IO_PROCESSORS_DEFAULT);
+ }
+
+ /**
+ * Create a new submitter task pool.
+ * @param key config key with pool size.
+ * @param defVal default value.
+ * @return a new task pool.
+ */
+ public CloseableTaskPoolSubmitter createSubmitter(String key, int defVal) {
+ int numThreads = conf.getInt(key, defVal);
+ if (numThreads <= 0) {
+ // ignore the setting if it is too invalid.
+ numThreads = defVal;
+ }
+ return createCloseableTaskSubmitter(numThreads, getJobAttemptId());
+ }
+
+ /**
+ * Create a new submitter task pool.
+ *
+ * @param numThreads thread count.
+ * @param jobAttemptId job ID
+ * @return a new task pool.
+ */
+ public static CloseableTaskPoolSubmitter createCloseableTaskSubmitter(
+ final int numThreads,
+ final String jobAttemptId) {
+ return new CloseableTaskPoolSubmitter(
+ HadoopExecutors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("manifest-committer-" + jobAttemptId + "-%d")
+ .build()));
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java
new file mode 100644
index 0000000..eb344e8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+
+/**
+ * Public constants for the manifest committer.
+ * This includes all configuration options and their default values.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class ManifestCommitterConstants {
+
+ /**
+ * Suffix to use in manifest files in the manifest subdir.
+ * Value: {@value}.
+ */
+ public static final String MANIFEST_SUFFIX = "-manifest.json";
+
+ /**
+ * Prefix for summary files in the report dir. Call
+ */
+ public static final String SUMMARY_FILENAME_PREFIX = "summary-";
+
+ /**
+ * Format string used to build a summary file from a Job ID.
+ */
+ public static final String SUMMARY_FILENAME_FORMAT =
+ SUMMARY_FILENAME_PREFIX + "%s.json";
+
+ /**
+ * Suffix to use for temp files before renaming them.
+ * Value: {@value}.
+ */
+ public static final String TMP_SUFFIX = ".tmp";
+
+ /**
+ * Initial number of all app attempts.
+ * This is fixed in YARN; for Spark jobs the
+ * same number "0" is used.
+ */
+ public static final int INITIAL_APP_ATTEMPT_ID = 0;
+
+ /**
+ * Format string for building a job dir.
+ * Value: {@value}.
+ */
+ public static final String JOB_DIR_FORMAT_STR = "%s";
+
+ /**
+ * Format string for building a job attempt dir.
+ * This uses the job attempt number so previous versions
+ * can be found trivially.
+ * Value: {@value}.
+ */
+ public static final String JOB_ATTEMPT_DIR_FORMAT_STR = "%02d";
+
+ /**
+ * Name of directory under job attempt dir for manifests.
+ */
+ public static final String JOB_TASK_MANIFEST_SUBDIR = "manifests";
+
+ /**
+ * Name of directory under job attempt dir for task attempts.
+ */
+ public static final String JOB_TASK_ATTEMPT_SUBDIR = "tasks";
+
+
+ /**
+ * Committer classname as recorded in the committer _SUCCESS file.
+ */
+ public static final String MANIFEST_COMMITTER_CLASSNAME =
+ ManifestCommitter.class.getName();
+
+ /**
+ * Marker file to create on success: {@value}.
+ */
+ public static final String SUCCESS_MARKER = "_SUCCESS";
+
+ /** Default job marker option: {@value}. */
+ public static final boolean DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER = true;
+
+ /**
+ * The limit to the number of committed objects tracked during
+ * job commits and saved to the _SUCCESS file.
+ * Value: {@value}.
+ */
+ public static final int SUCCESS_MARKER_FILE_LIMIT = 100;
+
+ /**
+ * The UUID for jobs: {@value}.
+ * This was historically created in Spark 1.x's SQL queries,
+ * but "went away".
+ * It has been restored in recent spark releases.
+ * If found: it is used instead of the MR job attempt ID.
+ */
+ public static final String SPARK_WRITE_UUID = "spark.sql.sources.writeJobUUID";
+
+ /**
+ * String to use as source of the job ID.
+ * This SHOULD be kept in sync with that of
+ * {@code AbstractS3ACommitter.JobUUIDSource}.
+ * Value: {@value}.
+ */
+ public static final String JOB_ID_SOURCE_MAPREDUCE = "JobID";
+
+ /**
+ * Prefix to use for config options: {@value}.
+ */
+ public static final String OPT_PREFIX = "mapreduce.manifest.committer.";
+
+ /**
+ * Should dir cleanup do parallel deletion of task attempt dirs
+ * before trying to delete the toplevel dirs.
+ * For GCS this may deliver speedup, while on ABFS it may avoid
+ * timeouts in certain deployments.
+ * Value: {@value}.
+ */
+ public static final String OPT_CLEANUP_PARALLEL_DELETE =
+ OPT_PREFIX + "cleanup.parallel.delete";
+
+ /**
+ * Default value: {@value}.
+ */
+ public static final boolean OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT = true;
+
+ /**
+ * Threads to use for IO.
+ */
+ public static final String OPT_IO_PROCESSORS = OPT_PREFIX + "io.threads";
+
+ /**
+ * Default value: {@value}.
+ */
+ public static final int OPT_IO_PROCESSORS_DEFAULT = 64;
+
+ /**
+ * Directory for saving job summary reports.
+ * These are the _SUCCESS files, but are saved even on
+ * job failures.
+ * Value: {@value}.
+ */
+ public static final String OPT_SUMMARY_REPORT_DIR =
+ OPT_PREFIX + "summary.report.directory";
+
+ /**
+ * Directory for moving manifests under for diagnostics.
+ * Value: {@value}.
+ */
+ public static final String OPT_DIAGNOSTICS_MANIFEST_DIR =
+ OPT_PREFIX + "diagnostics.manifest.directory";
+
+ /**
+ * Should the output be validated?
+ * This will check expected vs actual file lengths, and,
+ * if etags can be obtained, etags.
+ * Value: {@value}.
+ */
+ public static final String OPT_VALIDATE_OUTPUT = OPT_PREFIX + "validate.output";
+
+ /**
+ * Default value: {@value}.
+ */
+ public static final boolean OPT_VALIDATE_OUTPUT_DEFAULT = false;
+
+ /**
+ * Should job commit delete for files/directories at the targets
+ * of renames, and, if found, deleting them?
+ *
+ * This is part of the effective behavior of the FileOutputCommitter,
+ * however it adds an extra delete call per file being committed.
+ *
+ * If a job is writing to a directory which has only just been created
+ * or were unique filenames are being used, there is no need to perform
+ * this preparation.
+ * The recognition of newly created dirs is automatic.
+ *
+ * Value: {@value}.
+ */
+ public static final String OPT_DELETE_TARGET_FILES =
+ OPT_PREFIX + "delete.target.files";
+
+ /**
+ * Default value: {@value}.
+ */
+ public static final boolean OPT_DELETE_TARGET_FILES_DEFAULT = false;
+
+ /**
+ * Name of the factory: {@value}.
+ */
+ public static final String MANIFEST_COMMITTER_FACTORY =
+ ManifestCommitterFactory.class.getName();
+
+ /**
+ * Classname of the store operations; filesystems and tests
+ * may override.
+ * Value: {@value}.
+ */
+ public static final String OPT_STORE_OPERATIONS_CLASS = OPT_PREFIX + "store.operations.classname";
+
+ /**
+ * Default classname of the store operations.
+ * Value: {@value}.
+ */
+ public static final String STORE_OPERATIONS_CLASS_DEFAULT =
+ ManifestStoreOperationsThroughFileSystem.class.getName();
+
+ /**
+ * Stage attribute in audit context: {@value}.
+ */
+ public static final String CONTEXT_ATTR_STAGE = "st";
+
+ /**
+ * Task ID attribute in audit context: {@value}.
+ */
+ public static final String CONTEXT_ATTR_TASK_ATTEMPT_ID = "ta";
+
+ private ManifestCommitterConstants() {
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java
new file mode 100644
index 0000000..b07182c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.UNSUPPORTED_FS_SCHEMAS;
+
+/**
+ * This is the committer factory to register as the source of committers
+ * for the job/filesystem schema.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class ManifestCommitterFactory extends PathOutputCommitterFactory {
+
+ /**
+ * Name of this factory.
+ */
+ public static final String NAME = ManifestCommitterFactory.class.getName();
+
+ @Override
+ public ManifestCommitter createOutputCommitter(final Path outputPath,
+ final TaskAttemptContext context) throws IOException {
+ // safety check. S3A does not support this, so fail fast.
+ final String scheme = outputPath.toUri().getScheme();
+ if (UNSUPPORTED_FS_SCHEMAS.contains(scheme)) {
+ throw new PathIOException(outputPath.toString(),
+ "This committer does not work with the filesystem of type " + scheme);
+ }
+ return new ManifestCommitter(outputPath, context);
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java
new file mode 100644
index 0000000..243fd60
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterStatisticNames.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.statistics.StoreStatisticNames;
+
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_RENAME;
+
+/**
+ * Statistic names for committers.
+ * Please keep in sync with org.apache.hadoop.fs.s3a.Statistic
+ * so that S3A and manifest committers are in sync.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class ManifestCommitterStatisticNames {
+
+
+ /** Amount of data committed: {@value}. */
+ public static final String COMMITTER_BYTES_COMMITTED_COUNT =
+ "committer_bytes_committed";
+
+ /** Duration Tracking of time to commit an entire job: {@value}. */
+ public static final String COMMITTER_COMMIT_JOB =
+ "committer_commit_job";
+
+ /** Number of files committed: {@value}. */
+ public static final String COMMITTER_FILES_COMMITTED_COUNT =
+ "committer_files_committed";
+
+ /** "Count of successful tasks:: {@value}. */
+ public static final String COMMITTER_TASKS_COMPLETED_COUNT =
+ "committer_tasks_completed";
+
+ /** Count of failed tasks: {@value}. */
+ public static final String COMMITTER_TASKS_FAILED_COUNT =
+ "committer_tasks_failed";
+
+ /** Count of commits aborted: {@value}. */
+ public static final String COMMITTER_COMMITS_ABORTED_COUNT =
+ "committer_commits_aborted";
+
+ /** Count of commits reverted: {@value}. */
+ public static final String COMMITTER_COMMITS_REVERTED_COUNT =
+ "committer_commits_reverted";
+
+ /** Count of commits failed: {@value}. */
+ public static final String COMMITTER_COMMITS_FAILED =
+ "committer_commits" + StoreStatisticNames.SUFFIX_FAILURES;
+
+ /**
+ * The number of files in a task. This will be a MeanStatistic.
+ */
+ public static final String COMMITTER_FILE_COUNT_MEAN =
+ "committer_task_file_count";
+
+ /**
+ * File Size.
+ */
+ public static final String COMMITTER_FILE_SIZE_MEAN =
+ "committer_task_file_size";
+
+ /**
+ * What is a task attempt's directory count.
+ */
+ public static final String COMMITTER_TASK_DIRECTORY_COUNT_MEAN =
+ "committer_task_directory_count";
+
+ /**
+ * What is the depth of a task attempt's directory tree.
+ */
+ public static final String COMMITTER_TASK_DIRECTORY_DEPTH_MEAN =
+ "committer_task_directory_depth";
+
+ /**
+ * The number of files in a task. This will be a MeanStatistic.
+ */
+ public static final String COMMITTER_TASK_FILE_COUNT_MEAN =
+ "committer_task_file_count";
+
+ /**
+ * The number of files in a task. This will be a MeanStatistic.
+ */
+ public static final String COMMITTER_TASK_FILE_SIZE_MEAN =
+ "committer_task_file_size";
+
+ /**
+ * The size of manifest files. This will be a MeanStatistic.
+ * Useful to highlight whether this size is too large and
+ * effort to optimise for file IO and memory consumption
+ * justified.
+ */
+ public static final String COMMITTER_TASK_MANIFEST_FILE_SIZE =
+ "committer_task_manifest_file_size";
+
+ /**
+ * A file renamed during the commit operation {@value}.
+ */
+ public static final String OP_COMMIT_FILE_RENAME =
+ "commit_file_rename";
+
+ /**
+ * A file rename during the commit operation was recovered from a failure {@value}.
+ */
+ public static final String OP_COMMIT_FILE_RENAME_RECOVERED =
+ "commit_file_rename_recovered";
+
+ /** Directory creation {@value}. */
+ public static final String OP_CREATE_DIRECTORIES = "op_create_directories";
+
+ /** Creating a single directory {@value}. */
+ public static final String OP_CREATE_ONE_DIRECTORY =
+ "op_create_one_directory";
+
+ /**
+ * A file in the destination directory tree has been deleted.
+ * {@value}.
+ */
+ public static final String OP_DELETE_FILE_UNDER_DESTINATION =
+ "op_delete_file_under_destination";
+
+ /** Directory scan {@value}. */
+ public static final String OP_DIRECTORY_SCAN = "op_directory_scan";
+
+ /**
+ * Overall job commit {@value}.
+ */
+ public static final String OP_STAGE_JOB_COMMIT = COMMITTER_COMMIT_JOB;
+
+ /** {@value}. */
+ public static final String OP_LOAD_ALL_MANIFESTS = "op_load_all_manifests";
+
+ /**
+ * Load a task manifest: {@value}.
+ */
+ public static final String OP_LOAD_MANIFEST = "op_load_manifest";
+
+ /**
+ * mkdir failure statistic: {@value}.
+ * This is incremented whenever mkdir() returns false, e.g because
+ * there was a file at the end of the path.
+ */
+ public static final String OP_MKDIRS_RETURNED_FALSE = "op_mkdir_returned_false";
+
+ /**
+ * msync statistic: {@value}.
+ * This should be the same as StoreStatisticNames.OP_MSYNC; it's
+ * duplicated just to isolate this committer into a single JAR
+ * for ease of testing.
+ */
+ public static final String OP_MSYNC = "op_msync";
+
+ /**
+ * Preparing ancestor dirs: {@value}.
+ * Probing for paths being files, and if true: deleting them.
+ */
+ public static final String OP_PREPARE_DIR_ANCESTORS = "op_prepare_dir_ancestors";
+
+ /** Rename a dir: {@value}. */
+ public static final String OP_RENAME_DIR = OP_RENAME;
+
+
+ /** Rename a file: {@value}. */
+ public static final String OP_RENAME_FILE = OP_RENAME;
+
+ /**
+ * Save a task manifest: {@value}.
+ */
+ public static final String OP_SAVE_TASK_MANIFEST =
+ "task_stage_save_task_manifest";
+
+ /**
+ * Task abort: {@value}.
+ */
+ public static final String OP_STAGE_TASK_ABORT_TASK
+ = "task_stage_abort_task";
+
+ /**
+ * Job abort: {@value}.
+ */
+ public static final String OP_STAGE_JOB_ABORT = "job_stage_abort";
+
+ /**
+ * Job cleanup: {@value}.
+ */
+ public static final String OP_STAGE_JOB_CLEANUP = "job_stage_cleanup";
+
+ /**
+ * Prepare Directories Stage: {@value}.
+ */
+ public static final String OP_STAGE_JOB_CREATE_TARGET_DIRS =
+ "job_stage_create_target_dirs";
+
+ /**
+ * Load Manifest Stage: {@value}.
+ */
+ public static final String OP_STAGE_JOB_LOAD_MANIFESTS =
+ "job_stage_load_manifests";
+
+ /**
+ * Rename files stage duration: {@value}.
+ */
+ public static final String OP_STAGE_JOB_RENAME_FILES =
+ "job_stage_rename_files";
+
+
+ /**
+ * Job Setup Stage: {@value}.
+ */
+ public static final String OP_STAGE_JOB_SETUP = "job_stage_setup";
+
+ /**
+ * Job saving _SUCCESS marker Stage: {@value}.
+ */
+ public static final String OP_STAGE_JOB_SAVE_SUCCESS =
+ "job_stage_save_success_marker";
+
+ /**
+ * Output Validation (within job commit) Stage: {@value}.
+ */
+ public static final String OP_STAGE_JOB_VALIDATE_OUTPUT =
+ "job_stage_optional_validate_output";
+
+ /**
+ * Task saving manifest file Stage: {@value}.
+ */
+ public static final String OP_STAGE_TASK_SAVE_MANIFEST =
+ "task_stage_save_manifest";
+
+ /**
+ * Task Setup Stage: {@value}.
+ */
+ public static final String OP_STAGE_TASK_SETUP = "task_stage_setup";
+
+ /**
+ * Task Commit Stage: {@value}.
+ */
+ public static final String OP_STAGE_TASK_COMMIT = "task_stage_commit";
+
+ /** Task Scan directory Stage: {@value}. */
+ public static final String OP_STAGE_TASK_SCAN_DIRECTORY
+ = "task_stage_scan_directory";
+
+ private ManifestCommitterStatisticNames() {
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java
new file mode 100644
index 0000000..7020d5c
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Class for single/multiple commit data structures.
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public abstract class AbstractManifestData<T extends AbstractManifestData>
+ implements Serializable, IOStatisticsSource {
+
+
+ /**
+ * Convert a path to a string which can be included in the JSON.
+ * @param path path
+ * @return a string value, or, if path==null, null.
+ */
+ public static String marshallPath(@Nullable Path path) {
+ return path != null
+ ? path.toUri().toString()
+ : null;
+ }
+
+ /**
+ * Convert a string path to Path type, by way of a URI.
+ * @param path path as a string
+ * @return path value
+ * @throws RuntimeException marshalling failure.
+ */
+ public static Path unmarshallPath(String path) {
+ try {
+ return new Path(new URI(requireNonNull(path, "No path")));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(
+ "Failed to parse \"" + path + "\" : " + e,
+ e);
+
+ }
+ }
+
+ /**
+ * Validate the data: those fields which must be non empty, must be set.
+ * @return the validated instance.
+ * @throws IOException if the data is invalid
+ */
+ public abstract T validate() throws IOException;
+
+ /**
+ * Serialize to JSON and then to a byte array, after performing a
+ * preflight validation of the data to be saved.
+ * @return the data in a persistable form.
+ * @throws IOException serialization problem or validation failure.
+ */
+ public abstract byte[] toBytes() throws IOException;
+
+ /**
+ * Save to a hadoop filesystem.
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public abstract void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException;
+
+ /**
+ * Get a (usually shared) JSON serializer.
+ * @return a serializer. Call
+ */
+ public abstract JsonSerialization<T> createSerializer();
+
+ /**
+ * Verify that all instances in a collection are of the given class.
+ * @param it iterator
+ * @param classname classname to require
+ * @throws IOException on a failure
+ */
+ void validateCollectionClass(Iterable it, Class classname)
+ throws IOException {
+ for (Object o : it) {
+ verify(o.getClass().equals(classname),
+ "Collection element is not a %s: %s", classname, o.getClass());
+ }
+ }
+
+ /**
+ * Verify that a condition holds.
+ * @param expression expression which must be true
+ * @param message message to raise on a failure
+ * @param args arguments for the message formatting
+ * @throws IOException on a failure
+ */
+
+ static void verify(boolean expression,
+ String message,
+ Object... args) throws IOException {
+ if (!expression) {
+ throw new IOException(String.format(message, args));
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
new file mode 100644
index 0000000..cb67338
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DiagnosticKeys.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Diagnostic keys in the manifests.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class DiagnosticKeys {
+ /**
+ * Attribute added to diagnostics in _SUCCESS file.
+ */
+ public static final String PRINCIPAL = "principal";
+ public static final String STAGE = "stage";
+ public static final String EXCEPTION = "exception";
+ public static final String STACKTRACE = "stacktrace";
+
+
+ /** Directory where manifests were renamed: {@value}. */
+ public static final String MANIFESTS = "manifests";
+
+ private DiagnosticKeys() {
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java
new file mode 100644
index 0000000..15e8cac
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/DirEntry.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.marshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.unmarshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.verify;
+
+/**
+ * A directory entry in the task manifest.
+ * Uses shorter field names for smaller files.
+ * Hash and equals are on dir name only; there's no real expectation
+ * that those operations are needed.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class DirEntry implements Serializable {
+
+ private static final long serialVersionUID = 5658520530209859765L;
+
+ /**
+ * Destination directory.
+ */
+ @JsonProperty("d")
+ private String dir;
+
+ /**
+ * Type of dest entry as found when probed for in task commit.
+ */
+ @JsonProperty("t")
+ private int type;
+
+ /**
+ * Level in the treewalk.
+ */
+ @JsonProperty("l")
+ private int level;
+
+ /**
+ * Constructor only for use by jackson.
+ * Do Not Delete.
+ */
+ private DirEntry() {
+ }
+
+ /**
+ * Construct an entry.
+ *
+ * @param dir destination path.
+ * @param type type of dest entry
+ * @param level Level in the treewalk.
+ *
+ */
+ public DirEntry(
+ final String dir,
+ final int type,
+ final int level) {
+ this.dir = requireNonNull(dir);
+ this.type = type;
+ this.level = level;
+ }
+
+ /**
+ * Construct an entry.
+ *
+ * @param dir destination path.
+ * @param type type of dest entry
+ * @param level Level in the treewalk.
+ *
+ */
+ public DirEntry(
+ final Path dir,
+ final int type,
+ final int level) {
+ this(marshallPath(dir), type, level);
+ }
+
+ public void setDir(final String dir) {
+ this.dir = dir;
+ }
+
+ public String getDir() {
+ return dir;
+ }
+
+ @JsonIgnore
+ public Path getDestPath() {
+ return unmarshallPath(dir);
+ }
+
+ public int getType() {
+ return type;
+ }
+
+ public void setType(final int type) {
+ this.type = type;
+ }
+
+ public void setLevel(final int level) {
+ this.level = level;
+ }
+
+ public int getLevel() {
+ return level;
+ }
+
+ @JsonIgnore
+ public EntryStatus getStatus() {
+ return EntryStatus.toEntryStatus(type);
+ }
+
+ @JsonIgnore
+ public void setStatus(EntryStatus status) {
+ setType(status.ordinal());
+ }
+ public void validate() throws IOException {
+ final String s = toString();
+ verify(dir != null && dir.length() > 0,
+ "destination path is missing from " + s);
+ verify(type >= 0,
+ "Invalid type in " + s);
+ verify(level >= 0,
+ "Invalid level in " + s);
+ }
+
+ @Override
+ public String toString() {
+ return "DirEntry{" +
+ "dir='" + dir + '\'' +
+ ", type=" + type +
+ ", level=" + level +
+ '}';
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DirEntry dirEntry = (DirEntry) o;
+ return dir.equals(dirEntry.dir);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dir);
+ }
+
+ /**
+ * A directory entry.
+ * @param dest destination path.
+ * @param type type
+ * @param level Level in the treewalk.
+ * @return an entry
+ */
+ public static DirEntry dirEntry(Path dest, int type, int level) {
+ return new DirEntry(dest, type, level);
+ }
+
+ /**
+ * A directory entry.
+ * @param dest destination path.
+ * @param type type
+ * @param level Level in the treewalk.
+ * @return an entry
+ */
+ public static DirEntry dirEntry(Path dest, EntryStatus type, int level) {
+ return dirEntry(dest, type.ordinal(), level);
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java
new file mode 100644
index 0000000..73ac5d7
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/EntryStatus.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+
+/**
+ * Status of a file or dir entry, designed to be marshalled as
+ * an integer -the ordinal value of the enum is the
+ * wire value.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum EntryStatus {
+
+ unknown,
+ not_found,
+ file,
+ dir,
+ created_dir;
+
+ /**
+ * Go from a marshalled type to a status value.
+ * Any out of range value is converted to unknown.
+ * @param type type
+ * @return the status value.
+ */
+ public static EntryStatus toEntryStatus(int type) {
+ switch (type) {
+ case 1:
+ return not_found;
+ case 2:
+ return file;
+ case 3:
+ return dir;
+ case 4:
+ return created_dir;
+ case 0:
+ default:
+ return unknown;
+ }
+ }
+
+
+ /**
+ * Go from the result of a getFileStatus call or
+ * listing entry to a status.
+ * A null argument is mapped to {@link #not_found}
+ * @param st file status
+ * @return the status enum.
+ */
+ public static EntryStatus toEntryStatus(@Nullable FileStatus st) {
+
+ if (st == null) {
+ return not_found;
+ }
+ if (st.isDirectory()) {
+ return dir;
+ }
+ if (st.isFile()) {
+ return file;
+ }
+ return unknown;
+ }
+
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java
new file mode 100644
index 0000000..bc6cdd9
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/FileEntry.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.marshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.unmarshallPath;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData.verify;
+
+/**
+ * A File entry in the task manifest.
+ * Uses shorter field names for smaller files.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public final class FileEntry implements Serializable {
+
+ private static final long serialVersionUID = -550288489009777867L;
+
+ @JsonProperty("s")
+ private String source;
+
+ @JsonProperty("d")
+ private String dest;
+
+ @JsonProperty("z")
+ private long size;
+
+ /**
+ * Etag value if we can extract this.
+ */
+ @JsonProperty("e")
+ private String etag;
+
+ /**
+ * Constructor only for use by jackson.
+ * Do Not Delete.
+ */
+ private FileEntry() {
+ }
+
+ /**
+ * Construct an entry.
+ * @param source source path.
+ * @param dest destination path.
+ * @param size file size.
+ * @param etag optional etag
+ */
+ public FileEntry(
+ final String source,
+ final String dest,
+ final long size,
+ final String etag) {
+ this.source = source;
+ this.dest = dest;
+ this.size = size;
+ this.etag = etag;
+ }
+
+
+ /**
+ * Construct an entry.
+ * @param source source path.
+ * @param dest destination path.
+ * @param size file size.
+ * @param etag optional etag
+ */
+ public FileEntry(
+ final Path source,
+ final Path dest,
+ final long size,
+ final String etag) {
+ this(marshallPath(source), marshallPath(dest), size, etag);
+ }
+
+
+ public void setSource(final String source) {
+ this.source = source;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ @JsonIgnore
+ public Path getSourcePath() {
+ return unmarshallPath(source);
+ }
+
+ public void setDest(final String dest) {
+ this.dest = dest;
+ }
+
+ public String getDest() {
+ return dest;
+ }
+
+ @JsonIgnore
+ public Path getDestPath() {
+ return unmarshallPath(dest);
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(final long size) {
+ this.size = size;
+ }
+
+ public String getEtag() {
+ return etag;
+ }
+
+ public void setEtag(final String etag) {
+ this.etag = etag;
+ }
+
+ public void validate() throws IOException {
+ final String s = toString();
+ verify(source != null && source.length() > 0,
+ "Source is missing from " + s);
+ verify(dest != null && dest.length() > 0,
+ "Source is missing from " + s);
+ verify(size >= 0,
+ "Invalid size in " + s);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "FileOrDirEntry{");
+ sb.append("source='").append(source).append('\'');
+ sb.append(", dest='").append(dest).append('\'');
+ sb.append(", size=").append(size);
+ sb.append(", etag='").append(etag).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ FileEntry that = (FileEntry) o;
+ return size == that.size && source.equals(that.source) && dest.equals(
+ that.dest) &&
+ Objects.equals(etag, that.etag);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(source, dest);
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java
new file mode 100644
index 0000000..c95ec7b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestPrinter.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+
+/**
+ * Tool to print a manifest.
+ */
+public class ManifestPrinter extends Configured implements Tool {
+
+ private static final String USAGE = "ManifestPrinter <success-file>";
+
+ /**
+ * Output for printing.
+ */
+ private final PrintStream out;
+
+ /**
+ * Print to System.out.
+ */
+ public ManifestPrinter() {
+ this(null, System.out);
+ }
+
+ /**
+ * Print to the supplied stream.
+ * @param conf configuration
+ * @param out output
+ */
+ public ManifestPrinter(Configuration conf, PrintStream out) {
+ super(conf);
+ this.out = out;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length != 1) {
+ printUsage();
+ return -1;
+ }
+ Path path = new Path(args[0]);
+ loadAndPrintManifest(path.getFileSystem(getConf()), path);
+ return 0;
+ }
+
+ /**
+ * Load and print a manifest.
+ * @param fs filesystem.
+ * @param path path
+ * @throws IOException failure to load
+ * @return the manifest
+ */
+ public ManifestSuccessData loadAndPrintManifest(FileSystem fs, Path path)
+ throws IOException {
+ // load the manifest
+ println("Manifest file: %s", path);
+ final ManifestSuccessData success = ManifestSuccessData.load(fs, path);
+
+ printManifest(success);
+ return success;
+ }
+
+ private void printManifest(ManifestSuccessData success) {
+ field("succeeded", success.getSuccess());
+ field("created", success.getDate());
+ field("committer", success.getCommitter());
+ field("hostname", success.getHostname());
+ field("description", success.getDescription());
+ field("jobId", success.getJobId());
+ field("jobIdSource", success.getJobIdSource());
+ field("stage", success.getStage());
+ println("Diagnostics\n%s",
+ success.dumpDiagnostics(" ", " = ", "\n"));
+ println("Statistics:\n%s",
+ ioStatisticsToPrettyString(success.getIOStatistics()));
+ out.flush();
+ }
+
+ private void printUsage() {
+ println(USAGE);
+ }
+
+ /**
+ * Print a line to the output stream.
+ * @param format format string
+ * @param args arguments.
+ */
+ private void println(String format, Object... args) {
+ out.format(format, args);
+ out.println();
+ }
+
+ /**
+ * Print a field, if non-null.
+ * @param name field name.
+ * @param value value.
+ */
+ private void field(String name, Object value) {
+ if (value != null) {
+ println("%s: %s", name, value);
+ }
+ }
+
+ /**
+ * Entry point.
+ */
+ public static void main(String[] argv) throws Exception {
+
+ try {
+ int res = ToolRunner.run(new ManifestPrinter(), argv);
+ System.exit(res);
+ } catch (ExitUtil.ExitException e) {
+ ExitUtil.terminate(e);
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java
new file mode 100644
index 0000000..73e7307
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/ManifestSuccessData.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSupport;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Summary data saved into a {@code _SUCCESS} marker file.
+ *
+ * This is a copy of the S3A committer success data format, with
+ * a goal of being/remaining compatible.
+ * This makes it easier for tests in downstream modules to
+ * be able to parse the success files from any of the committers.
+ *
+ * This should be considered public; it is based on the S3A
+ * format, which has proven stable over time.
+ *
+ * The JSON format SHOULD be considered public and evolving
+ * with compatibility across versions.
+ *
+ * All the Java serialization data is different and may change
+ * across versions with no stability guarantees other than
+ * "manifest summaries MAY be serialized between processes with
+ * the exact same version of this binary on their classpaths."
+ * That is sufficient for testing in Spark.
+ *
+ * To aid with Java serialization, the maps and lists are
+ * exclusively those which serialize well.
+ * IOStatisticsSnapshot has a lot of complexity in marshalling
+ * there; this class doesn't worry about concurrent access
+ * so is simpler.
+ *
+ */
+@SuppressWarnings({"unused", "CollectionDeclaredAsConcreteClass"})
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ManifestSuccessData
+ extends AbstractManifestData<ManifestSuccessData> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ManifestSuccessData.class);
+
+ /**
+ * Supported version value: {@value}.
+ * If this is changed the value of {@link #serialVersionUID} will change,
+ * to avoid deserialization problems.
+ */
+ public static final int VERSION = 1;
+
+ /**
+ * Serialization ID: {@value}.
+ */
+ private static final long serialVersionUID = 4755993198698104084L + VERSION;
+
+ /**
+ * Name to include in persisted data, so as to differentiate from
+ * any other manifests: {@value}.
+ */
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.commit.files.SuccessData/" + VERSION;
+
+ /**
+ * Name of file; includes version marker.
+ */
+ private String name;
+
+ /** Timestamp of creation. */
+ private long timestamp;
+
+ /**
+ * Did this succeed?
+ * It is implicitly true in a _SUCCESS file, but if the file
+ * is also saved to a log dir, then it depends on the outcome
+ */
+ private boolean success = true;
+
+ /** Timestamp as date string; no expectation of parseability. */
+ private String date;
+
+ /**
+ * Host which created the file (implicitly: committed the work).
+ */
+ private String hostname;
+
+ /**
+ * Committer name.
+ */
+ private String committer;
+
+ /**
+ * Description text.
+ */
+ private String description;
+
+ /** Job ID, if known. */
+ private String jobId = "";
+
+ /**
+ * Source of the job ID.
+ */
+ private String jobIdSource = "";
+
+ /**
+ * Metrics.
+ * Uses a treemap for serialization.
+ */
+ private TreeMap<String, Long> metrics = new TreeMap<>();
+
+ /**
+ * Diagnostics information.
+ * Uses a treemap for serialization.
+ */
+ private TreeMap<String, String> diagnostics = new TreeMap<>();
+
+ /**
+ * Filenames in the commit.
+ */
+ private ArrayList<String> filenames = new ArrayList<>(0);
+
+ /**
+ * IOStatistics.
+ */
+ @JsonProperty("iostatistics")
+ private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot();
+
+ /**
+ * State (committed, aborted).
+ */
+ private String state;
+
+ /**
+ * Stage: last stage executed.
+ */
+ private String stage;
+
+ @Override
+ public ManifestSuccessData validate() throws IOException {
+ verify(name != null,
+ "Incompatible file format: no 'name' field");
+ verify(NAME.equals(name),
+ "Incompatible file format: " + name);
+ return this;
+ }
+
+ @Override
+ public JsonSerialization<ManifestSuccessData> createSerializer() {
+ return serializer();
+ }
+
+ @Override
+ public byte[] toBytes() throws IOException {
+ return serializer().toBytes(this);
+ }
+
+ /**
+ * To JSON.
+ * @return json string value.
+ * @throws IOException failure
+ */
+ public String toJson() throws IOException {
+ return serializer().toJson(this);
+ }
+
+ @Override
+ public void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException {
+ // always set the name field before being saved.
+ name = NAME;
+ serializer().save(fs, path, this, overwrite);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ManifestSuccessData{");
+ sb.append("committer='").append(committer).append('\'');
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append(", description='").append(description).append('\'');
+ sb.append(", date='").append(date).append('\'');
+ sb.append(", filenames=[").append(
+ StringUtils.join(filenames, ", "))
+ .append("]");
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Dump the metrics (if any) to a string.
+ * The metrics are sorted for ease of viewing.
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return the dumped string
+ */
+ public String dumpMetrics(String prefix, String middle, String suffix) {
+ return joinMap(metrics, prefix, middle, suffix);
+ }
+
+ /**
+ * Dump the diagnostics (if any) to a string.
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return the dumped string
+ */
+ public String dumpDiagnostics(String prefix, String middle, String suffix) {
+ return joinMap(diagnostics, prefix, middle, suffix);
+ }
+
+ /**
+ * Join any map of string to value into a string, sorting the keys first.
+ * @param map map to join
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return a string for reporting.
+ */
+ protected static String joinMap(Map<String, ?> map,
+ String prefix,
+ String middle, String suffix) {
+ if (map == null) {
+ return "";
+ }
+ List<String> list = new ArrayList<>(map.keySet());
+ Collections.sort(list);
+ StringBuilder sb = new StringBuilder(list.size() * 32);
+ for (String k : list) {
+ sb.append(prefix)
+ .append(k)
+ .append(middle)
+ .append(map.get(k))
+ .append(suffix);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * @param fs filesystem
+ * @param path path
+ * @return the loaded instance
+ * @throws IOException IO failure
+ */
+ public static ManifestSuccessData load(FileSystem fs, Path path)
+ throws IOException {
+ LOG.debug("Reading success data from {}", path);
+ ManifestSuccessData instance = serializer().load(fs, path);
+ instance.validate();
+ return instance;
+ }
+
+ /**
+ * Get a JSON serializer for this class.
+ * @return a serializer.
+ */
+ public static JsonSerialization<ManifestSuccessData> serializer() {
+ return new JsonSerialization<>(ManifestSuccessData.class, false, true);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /** @return timestamp of creation. */
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ /** @return timestamp as date; no expectation of parseability. */
+ public String getDate() {
+ return date;
+ }
+
+ public void setDate(String date) {
+ this.date = date;
+ }
+
+ /**
+ * @return host which created the file (implicitly: committed the work).
+ */
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * @return committer name.
+ */
+ public String getCommitter() {
+ return committer;
+ }
+
+ public void setCommitter(String committer) {
+ this.committer = committer;
+ }
+
+ /**
+ * @return any description text.
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ /**
+ * @return any metrics.
+ */
+ public Map<String, Long> getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(TreeMap<String, Long> metrics) {
+ this.metrics = metrics;
+ }
+
+ /**
+ * @return a list of filenames in the commit.
+ */
+ public List<String> getFilenames() {
+ return filenames;
+ }
+
+ /**
+ * Get the list of filenames as paths.
+ * @return the paths.
+ */
+ @JsonIgnore
+ public List<Path> getFilenamePaths() {
+ return getFilenames().stream()
+ .map(AbstractManifestData::unmarshallPath)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Set the list of filename paths.
+ */
+ @JsonIgnore
+ public void setFilenamePaths(List<Path> paths) {
+ setFilenames(new ArrayList<>(
+ paths.stream()
+ .map(AbstractManifestData::marshallPath)
+ .collect(Collectors.toList())));
+ }
+
+ public void setFilenames(ArrayList<String> filenames) {
+ this.filenames = filenames;
+ }
+
+ public Map<String, String> getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(TreeMap<String, String> diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Add a diagnostics entry.
+ * @param key name
+ * @param value value
+ */
+ public void putDiagnostic(String key, String value) {
+ diagnostics.put(key, value);
+ }
+
+ /** @return Job ID, if known. */
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getJobIdSource() {
+ return jobIdSource;
+ }
+
+ public void setJobIdSource(final String jobIdSource) {
+ this.jobIdSource = jobIdSource;
+ }
+
+ @Override
+ public IOStatisticsSnapshot getIOStatistics() {
+ return iostatistics;
+ }
+
+ public void setIOStatistics(final IOStatisticsSnapshot ioStatistics) {
+ this.iostatistics = ioStatistics;
+ }
+
+ /**
+ * Set the IOStatistics to a snapshot of the source.
+ * @param iostats. Statistics; may be null.
+ */
+ public void snapshotIOStatistics(IOStatistics iostats) {
+ setIOStatistics(IOStatisticsSupport.snapshotIOStatistics(iostats));
+ }
+
+ /**
+ * Set the success flag.
+ * @param success did the job succeed?
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ }
+
+ /**
+ * Get the success flag.
+ * @return did the job succeed?
+ */
+ public boolean getSuccess() {
+ return success;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public String getStage() {
+ return stage;
+ }
+
+ /**
+ * Note a failure by setting success flag to false,
+ * then add the exception to the diagnostics.
+ * @param thrown throwable
+ */
+ public void recordJobFailure(Throwable thrown) {
+ setSuccess(false);
+ String stacktrace = ExceptionUtils.getStackTrace(thrown);
+ diagnostics.put(DiagnosticKeys.EXCEPTION, thrown.toString());
+ diagnostics.put(DiagnosticKeys.STACKTRACE, stacktrace);
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java
new file mode 100644
index 0000000..a06b837
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/TaskManifest.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * This is the manifest of files which were created by
+ * this task attempt.
+ *
+ * Versioning:
+ * In a rolling cluster update, MR or Spark workers deployed on a newer
+ * node (and/or with a newer version of artifacts in a cluster-FS hosted
+ * tar.gz file) may be a later version of this class than that of
+ * job committer.
+ * If any changes are made to the manifest which are backwards compatible,
+ * this new manifest can still be loaded from JSON and processed.
+ *
+ * If the manifest is no longer compatible, the job output may
+ * be invalid.
+ *
+ * It is CRITICAL that the {@link #VERSION} constant is updated whenever
+ * such an incompatible change is made.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class TaskManifest extends AbstractManifestData<TaskManifest> {
+
+ /**
+ * Supported version value: {@value}.
+ * If this is changed the value of {@code serialVersionUID} will change,
+ * to avoid deserialization problems.
+ */
+ public static final int VERSION = 1;
+
+ /**
+ * Manifest type.
+ */
+ public static final String TYPE =
+ "org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest/"
+ + VERSION;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskManifest.class);
+
+ /**
+ * Serialization version.
+ */
+ private static final long serialVersionUID = 7090285511966046094L + VERSION;
+
+ /**
+ * Manifest type.
+ */
+ @JsonProperty("type")
+ private String type = TYPE;
+
+ /** Version marker. */
+ @JsonProperty("version")
+ private int version = VERSION;
+
+ /**
+ * Job ID; constant over multiple attempts.
+ */
+ @JsonProperty("jobId")
+ private String jobId;
+
+ /**
+ * Number of the job attempt; starts at zero.
+ */
+ @JsonProperty("jobAttemptNumber")
+ private int jobAttemptNumber;
+
+ /**
+ * Task Attempt ID.
+ */
+ @JsonProperty("taskID")
+ private String taskID;
+
+ /**
+ * Task Attempt ID.
+ */
+ @JsonProperty("taskAttemptID")
+ private String taskAttemptID;
+
+ /**
+ * The task attempt directory.
+ */
+ @JsonProperty("taskAttemptDir")
+ private String taskAttemptDir;
+
+ /**
+ * The list of files to commit from this task attempt, including
+ * precalculated destination and size.
+ */
+ @JsonProperty("files")
+ private final List<FileEntry> filesToCommit = new ArrayList<>();
+
+ /**
+ * The list of directories needed by this task attempt, both
+ * source and destination.
+ * All these directories must exist in the destination before any of
+ * the files can be renamed there.
+ */
+ @JsonProperty("directories")
+ private final List<DirEntry> destDirectories = new ArrayList<>();
+
+ /**
+ * Any custom extra data committers may choose to add.
+ */
+ private final Map<String, String> extraData = new HashMap<>(0);
+
+ /**
+ * IOStatistics.
+ */
+ @JsonProperty("iostatistics")
+ private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot();
+
+ /**
+ * Empty constructor; will be used by jackson as well as in application
+ * code.
+ */
+ public TaskManifest() {
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ @Override
+ public IOStatisticsSnapshot getIOStatistics() {
+ return iostatistics;
+ }
+
+ public void setIOStatistics(
+ @Nullable final IOStatisticsSnapshot ioStatistics) {
+ this.iostatistics = ioStatistics;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(final String jobId) {
+ this.jobId = jobId;
+ }
+
+ public int getJobAttemptNumber() {
+ return jobAttemptNumber;
+ }
+
+ public void setJobAttemptNumber(final int jobAttemptNumber) {
+ this.jobAttemptNumber = jobAttemptNumber;
+ }
+
+ public String getTaskID() {
+ return taskID;
+ }
+
+ public void setTaskID(final String taskID) {
+ this.taskID = taskID;
+ }
+
+ public String getTaskAttemptID() {
+ return taskAttemptID;
+ }
+
+ public void setTaskAttemptID(final String taskAttemptID) {
+ this.taskAttemptID = taskAttemptID;
+ }
+
+ public String getTaskAttemptDir() {
+ return taskAttemptDir;
+ }
+
+ public void setTaskAttemptDir(final String taskAttemptDir) {
+ this.taskAttemptDir = taskAttemptDir;
+ }
+
+ /**
+ * Add a file to the list of files to commit.
+ * @param entry entry to add
+ */
+ public void addFileToCommit(FileEntry entry) {
+ filesToCommit.add(entry);
+ }
+
+ public List<FileEntry> getFilesToCommit() {
+ return filesToCommit;
+ }
+
+ /**
+ * Calculate the total amount of data which will be committed.
+ * @return the sum of sizes of all files to commit.
+ */
+ @JsonIgnore
+ public long getTotalFileSize() {
+ return filesToCommit.stream().mapToLong(FileEntry::getSize).sum();
+ }
+
+ /**
+ * All the destination directories.
+ * @return directory list.
+ */
+ public List<DirEntry> getDestDirectories() {
+ return destDirectories;
+ }
+
+ /**
+ * Add a directory to the list of directories to create.
+ * @param entry entry to add
+ */
+ public void addDirectory(DirEntry entry) {
+ destDirectories.add(entry);
+ }
+
+ public Map<String, String> getExtraData() {
+ return extraData;
+ }
+
+ @Override
+ public byte[] toBytes() throws IOException {
+ return serializer().toBytes(this);
+ }
+
+ /**
+ * To JSON.
+ * @return json string value.
+ * @throws IOException failure
+ */
+ public String toJson() throws IOException {
+ return serializer().toJson(this);
+ }
+
+ @Override
+ public void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException {
+ serializer().save(fs, path, this, overwrite);
+ }
+
+ /**
+ * Validate the data: those fields which must be non empty, must be set.
+ * @throws IOException if the data is invalid
+ * @return
+ */
+ public TaskManifest validate() throws IOException {
+ verify(TYPE.equals(type), "Wrong type: %s", type);
+ verify(version == VERSION, "Wrong version: %s", version);
+ validateCollectionClass(extraData.keySet(), String.class);
+ validateCollectionClass(extraData.values(), String.class);
+ Set<String> destinations = new HashSet<>(filesToCommit.size());
+ validateCollectionClass(filesToCommit, FileEntry.class);
+ for (FileEntry c : filesToCommit) {
+ c.validate();
+ verify(!destinations.contains(c.getDest()),
+ "Destination %s is written to by more than one pending commit",
+ c.getDest());
+ destinations.add(c.getDest());
+ }
+ return this;
+ }
+
+ /**
+ * Get a JSON serializer for this class.
+ * @return a serializer.
+ */
+ @Override
+ public JsonSerialization<TaskManifest> createSerializer() {
+ return serializer();
+ }
+
+ /**
+ * Create a JSON serializer for this class.
+ * @return a serializer.
+ */
+ public static JsonSerialization<TaskManifest> serializer() {
+ return new JsonSerialization<>(TaskManifest.class, false, true);
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * @param fs filesystem
+ * @param path path
+ * @return the loaded instance
+ * @throws IOException IO failure/the data is invalid
+ */
+ public static TaskManifest load(FileSystem fs, Path path)
+ throws IOException {
+ LOG.debug("Reading Manifest in file {}", path);
+ return serializer().load(fs, path).validate();
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * If loading through a listing; use this API so that filestatus
+ * hints can be used.
+ * @param serializer serializer.
+ * @param fs filesystem
+ * @param path path to load from
+ * @param status status of file to load
+ * @return the loaded instance
+ * @throws IOException IO failure/the data is invalid
+ */
+ public static TaskManifest load(
+ JsonSerialization<TaskManifest> serializer,
+ FileSystem fs,
+ Path path,
+ FileStatus status)
+ throws IOException {
+ LOG.debug("Reading Manifest in file {}", path);
+ return serializer.load(fs, path, status)
+ .validate();
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java
new file mode 100644
index 0000000..e1fbb4a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/package-info.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Persistence formats.
+ * These are the persistence formats used for passing data from tasks
+ * to the job committer
+ * {@link org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest},
+ * and for a {@code _SUCCESS} file, which is in
+ * {@link org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData}.
+ * The {@code _SUCCESS} file is a copy of the S3A Committer
+ * {@code org.apache.hadoop.fs.s3a.commit.files.ManifestSuccessData},
+ * the intent being that at the JSON-level they are compatible.
+ * This is to aid testing/validation and support calls, with one single
+ * format to load.
+ *
+ * Consult the individual formats for their declarations of access;
+ * the _SUCCESS file is one which tests may use.
+ *
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.files;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java
new file mode 100644
index 0000000..de67a2a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/AuditingIntegration.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.audit.CommonAuditContext;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConfig;
+
+import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_JOB_ID;
+import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CONTEXT_ATTR_STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CONTEXT_ATTR_TASK_ATTEMPT_ID;
+
+/**
+ * Helper class to support integration with Hadoop 3.3.2+ Auditing.
+ * This MUST BE the sole place where fs.audit methods are used, so can be replaced
+ * by a stub class on any backport.
+ */
+@InterfaceAudience.Private
+public final class AuditingIntegration {
+ private AuditingIntegration() {
+ }
+
+ /**
+ * Add jobID to current context; also
+ * task attempt ID if set.
+ */
+ public static void updateCommonContextOnCommitterEntry(
+ ManifestCommitterConfig committerConfig) {
+ CommonAuditContext context = currentAuditContext();
+ context.put(PARAM_JOB_ID,
+ committerConfig.getJobUniqueId());
+ // maybe the task attempt ID.
+ if (!committerConfig.getTaskAttemptId().isEmpty()) {
+ context.put(CONTEXT_ATTR_TASK_ATTEMPT_ID,
+ committerConfig.getTaskAttemptId());
+ }
+ }
+
+ /**
+ * Callback on stage entry.
+ * Sets the activeStage and updates the
+ * common context.
+ * @param stage new stage
+ */
+ public static void enterStage(String stage) {
+ currentAuditContext().put(CONTEXT_ATTR_STAGE, stage);
+ }
+
+ /**
+ * Remove stage from common audit context.
+ */
+ public static void exitStage() {
+ currentAuditContext().remove(CONTEXT_ATTR_STAGE);
+ }
+
+ /**
+ * Remove commit info at the end of the task or job.
+ */
+ public static void updateCommonContextOnCommitterExit() {
+ currentAuditContext().remove(PARAM_JOB_ID);
+ currentAuditContext().remove(CONTEXT_ATTR_TASK_ATTEMPT_ID);
+ }
+
+ /**
+ * Update the thread context with the stage name and
+ * job ID.
+ * This MUST be invoked at the start of methods invoked in helper threads,
+ * to ensure that they are all annotated with job and stage.
+ * @param jobId job ID.
+ * @param stage stage name.
+ */
+ public static void enterStageWorker(String jobId, String stage) {
+ CommonAuditContext context = currentAuditContext();
+ context.put(PARAM_JOB_ID, jobId);
+ context.put(CONTEXT_ATTR_STAGE, stage);
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java
new file mode 100644
index 0000000..15f9899
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/InternalConstants.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.util.Set;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_DIRECTORY;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_FILE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_LIST_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_MKDIRS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.STORE_IO_RATE_LIMITED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.*;
+
+/**
+ * Constants internal to the manifest committer.
+ */
+@InterfaceAudience.Private
+public final class InternalConstants {
+ private InternalConstants() {
+ }
+
+ /**
+ * Durations.
+ */
+ public static final String[] DURATION_STATISTICS = {
+
+ /* Job stages. */
+ OP_STAGE_JOB_ABORT,
+ OP_STAGE_JOB_CLEANUP,
+ OP_STAGE_JOB_COMMIT,
+ OP_STAGE_JOB_CREATE_TARGET_DIRS,
+ OP_STAGE_JOB_LOAD_MANIFESTS,
+ OP_STAGE_JOB_RENAME_FILES,
+ OP_STAGE_JOB_SAVE_SUCCESS,
+ OP_STAGE_JOB_SETUP,
+ OP_STAGE_JOB_VALIDATE_OUTPUT,
+
+ /* Task stages. */
+
+ OP_STAGE_TASK_ABORT_TASK,
+ OP_STAGE_TASK_COMMIT,
+ OP_STAGE_TASK_SAVE_MANIFEST,
+ OP_STAGE_TASK_SCAN_DIRECTORY,
+ OP_STAGE_TASK_SETUP,
+
+ /* Lower level store/fs operations. */
+ OP_COMMIT_FILE_RENAME,
+ OP_CREATE_DIRECTORIES,
+ OP_CREATE_ONE_DIRECTORY,
+ OP_DIRECTORY_SCAN,
+ OP_DELETE,
+ OP_DELETE_FILE_UNDER_DESTINATION,
+ OP_GET_FILE_STATUS,
+ OP_IS_DIRECTORY,
+ OP_IS_FILE,
+ OP_LIST_STATUS,
+ OP_LOAD_MANIFEST,
+ OP_LOAD_ALL_MANIFESTS,
+ OP_MKDIRS,
+ OP_MKDIRS_RETURNED_FALSE,
+ OP_MSYNC,
+ OP_PREPARE_DIR_ANCESTORS,
+ OP_RENAME_FILE,
+ OP_SAVE_TASK_MANIFEST,
+
+ OBJECT_LIST_REQUEST,
+ OBJECT_CONTINUE_LIST_REQUEST,
+
+ STORE_IO_RATE_LIMITED
+ };
+
+ /**
+ * Counters.
+ */
+ public static final String[] COUNTER_STATISTICS = {
+ COMMITTER_BYTES_COMMITTED_COUNT,
+ COMMITTER_FILES_COMMITTED_COUNT,
+ COMMITTER_TASKS_COMPLETED_COUNT,
+ COMMITTER_TASKS_FAILED_COUNT,
+ COMMITTER_TASK_DIRECTORY_COUNT_MEAN,
+ COMMITTER_TASK_DIRECTORY_DEPTH_MEAN,
+ COMMITTER_TASK_FILE_COUNT_MEAN,
+ COMMITTER_TASK_FILE_SIZE_MEAN,
+ COMMITTER_TASK_MANIFEST_FILE_SIZE,
+ OP_COMMIT_FILE_RENAME_RECOVERED,
+ };
+
+ /**
+ * Error string from ABFS connector on timeout.
+ */
+ public static final String OPERATION_TIMED_OUT = "OperationTimedOut";
+
+ /**
+ * Format string for task attempt names.
+ */
+ public static final String NAME_FORMAT_TASK_ATTEMPT = "[Task-Attempt %s]";
+
+ /**
+ * Format string for job attempt names.
+ */
+ public static final String NAME_FORMAT_JOB_ATTEMPT = "[Job-Attempt %s]";
+
+ /** Schemas of filesystems we know to not work with this committer. */
+ public static final Set<String> UNSUPPORTED_FS_SCHEMAS =
+ ImmutableSet.of("s3a", "wasb");
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java
new file mode 100644
index 0000000..f6edde5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestCommitterSupport.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.io.IOException;
+import java.time.ZonedDateTime;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.EtagSource;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.statistics.IOStatisticsAggregator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStoreBuilder;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages.StageConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.PENDING_DIR_NAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.INITIAL_APP_ATTEMPT_ID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ATTEMPT_DIR_FORMAT_STR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_DIR_FORMAT_STR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_ID_SOURCE_MAPREDUCE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_TASK_ATTEMPT_SUBDIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.JOB_TASK_MANIFEST_SUBDIR;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_COMMITTER_CLASSNAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SPARK_WRITE_UUID;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUMMARY_FILENAME_FORMAT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.TMP_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.PRINCIPAL;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.COUNTER_STATISTICS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.InternalConstants.DURATION_STATISTICS;
+
+/**
+ * Class for manifest committer support util methods.
+ */
+
+@InterfaceAudience.Private
+public final class ManifestCommitterSupport {
+
+ private ManifestCommitterSupport() {
+ }
+
+ /**
+ * Create an IOStatistics Store with the standard statistics
+ * set up.
+ * @return a store builder preconfigured with the standard stats.
+ */
+ public static IOStatisticsStoreBuilder createIOStatisticsStore() {
+
+ final IOStatisticsStoreBuilder store
+ = iostatisticsStore();
+
+ store.withCounters(COUNTER_STATISTICS);
+ store.withMaximums(COUNTER_STATISTICS);
+ store.withMinimums(COUNTER_STATISTICS);
+ store.withMeanStatistics(COUNTER_STATISTICS);
+ store.withDurationTracking(DURATION_STATISTICS);
+ return store;
+ }
+
+ /**
+ * If the object is an IOStatisticsSource, get and add
+ * its IOStatistics.
+ * @param o source object.
+ */
+ public static void maybeAddIOStatistics(IOStatisticsAggregator ios,
+ Object o) {
+ if (o instanceof IOStatisticsSource) {
+ ios.aggregate(((IOStatisticsSource) o).getIOStatistics());
+ }
+ }
+
+ /**
+ * Build a Job UUID from the job conf (if it is
+ * {@link ManifestCommitterConstants#SPARK_WRITE_UUID}
+ * or the MR job ID.
+ * @param conf job/task configuration
+ * @param jobId job ID from YARN or spark.
+ * @return (a job ID, source)
+ */
+ public static Pair<String, String> buildJobUUID(Configuration conf,
+ JobID jobId) {
+ String jobUUID = conf.getTrimmed(SPARK_WRITE_UUID, "");
+ if (jobUUID.isEmpty()) {
+ jobUUID = jobId.toString();
+ return Pair.of(jobUUID, JOB_ID_SOURCE_MAPREDUCE);
+ } else {
+ return Pair.of(jobUUID, SPARK_WRITE_UUID);
+ }
+ }
+
+ /**
+ * Get the location of pending job attempts.
+ * @param out the base output directory.
+ * @return the location of pending job attempts.
+ */
+ public static Path getPendingJobAttemptsPath(Path out) {
+ return new Path(out, PENDING_DIR_NAME);
+ }
+
+ /**
+ * Get the Application Attempt Id for this job.
+ * @param context the context to look in
+ * @return the Application Attempt Id for a given job.
+ */
+ public static int getAppAttemptId(JobContext context) {
+ return getAppAttemptId(context.getConfiguration());
+ }
+
+ /**
+ * Get the Application Attempt Id for this job
+ * by looking for {@link MRJobConfig#APPLICATION_ATTEMPT_ID}
+ * in the configuration, falling back to 0 if unset.
+ * For spark it will always be 0, for MR it will be set in the AM
+ * to the {@code ApplicationAttemptId} the AM is launched with.
+ * @param conf job configuration.
+ * @return the Application Attempt Id for the job.
+ */
+ public static int getAppAttemptId(Configuration conf) {
+ return conf.getInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+ INITIAL_APP_ATTEMPT_ID);
+ }
+
+ /**
+ * Get the path in the job attempt dir for a manifest for a task.
+ * @param manifestDir manifest directory
+ * @param taskId taskID.
+ * @return the final path to rename the manifest file to
+ */
+ public static Path manifestPathForTask(Path manifestDir, String taskId) {
+
+ return new Path(manifestDir, taskId + MANIFEST_SUFFIX);
+ }
+
+ /**
+ * Get the path in the manifest subdir for the temp path to save a
+ * task attempt's manifest before renaming it to the
+ * path defined by {@link #manifestPathForTask(Path, String)}.
+ * @param manifestDir manifest directory
+ * @param taskAttemptId task attempt ID.
+ * @return the path to save/load the manifest.
+ */
+ public static Path manifestTempPathForTaskAttempt(Path manifestDir,
+ String taskAttemptId) {
+ return new Path(manifestDir,
+ taskAttemptId + MANIFEST_SUFFIX + TMP_SUFFIX);
+ }
+
+ /**
+ * Create a task attempt dir; stage config must be for a task attempt.
+ * @param stageConfig state config.
+ * @return a manifest with job and task attempt info set up.
+ */
+ public static TaskManifest createTaskManifest(StageConfig stageConfig) {
+ final TaskManifest manifest = new TaskManifest();
+ manifest.setTaskAttemptID(stageConfig.getTaskAttemptId());
+ manifest.setTaskID(stageConfig.getTaskId());
+ manifest.setJobId(stageConfig.getJobId());
+ manifest.setJobAttemptNumber(stageConfig.getJobAttemptNumber());
+ manifest.setTaskAttemptDir(
+ stageConfig.getTaskAttemptDir().toUri().toString());
+ return manifest;
+ }
+
+ /**
+ * Create success/outcome data.
+ * @param stageConfig configuration.
+ * @param stage
+ * @return a _SUCCESS object with some diagnostics.
+ */
+ public static ManifestSuccessData createManifestOutcome(
+ StageConfig stageConfig, String stage) {
+ final ManifestSuccessData outcome = new ManifestSuccessData();
+ outcome.setJobId(stageConfig.getJobId());
+ outcome.setJobIdSource(stageConfig.getJobIdSource());
+ outcome.setCommitter(MANIFEST_COMMITTER_CLASSNAME);
+ // real timestamp
+ outcome.setTimestamp(System.currentTimeMillis());
+ final ZonedDateTime now = ZonedDateTime.now();
+ outcome.setDate(now.toString());
+ outcome.setHostname(NetUtils.getLocalHostname());
+ // add some extra diagnostics which can still be parsed by older
+ // builds of test applications.
+ // Audit Span information can go in here too, in future.
+ try {
+ outcome.putDiagnostic(PRINCIPAL,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ } catch (IOException ignored) {
+ // don't know who we are? exclude from the diagnostics.
+ }
+ outcome.putDiagnostic(STAGE, stage);
+ return outcome;
+ }
+
+ /**
+ * Create the filename for a report from the jobID.
+ * @param jobId jobId
+ * @return filename for a report.
+ */
+ public static String createJobSummaryFilename(String jobId) {
+ return String.format(SUMMARY_FILENAME_FORMAT, jobId);
+ }
+
+ /**
+ * Get an etag from a FileStatus which MUST BE
+ * an implementation of EtagSource and
+ * whose etag MUST NOT BE null/empty.
+ * @param status the status; may be null.
+ * @return the etag or null if not provided
+ */
+ public static String getEtag(FileStatus status) {
+ if (status instanceof EtagSource) {
+ return ((EtagSource) status).getEtag();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Create the manifest store operations for the given FS.
+ * This supports binding to custom filesystem handlers.
+ * @param conf configuration.
+ * @param filesystem fs.
+ * @param path path under FS.
+ * @return a bonded store operations.
+ * @throws IOException on binding/init problems.
+ */
+ public static ManifestStoreOperations createManifestStoreOperations(
+ final Configuration conf,
+ final FileSystem filesystem,
+ final Path path) throws IOException {
+ try {
+ final Class<? extends ManifestStoreOperations> storeClass = conf.getClass(
+ OPT_STORE_OPERATIONS_CLASS,
+ ManifestStoreOperationsThroughFileSystem.class,
+ ManifestStoreOperations.class);
+ final ManifestStoreOperations operations = storeClass.
+ getDeclaredConstructor().newInstance();
+ operations.bindToFileSystem(filesystem, path);
+ return operations;
+ } catch (Exception e) {
+ throw new PathIOException(path.toString(),
+ "Failed to create Store Operations from configuration option "
+ + OPT_STORE_OPERATIONS_CLASS
+ + ":" + e, e);
+ }
+ }
+
+ /**
+ * Logic to create directory names from job and attempt.
+ * This is self-contained it so it can be used in tests
+ * as well as in the committer.
+ */
+ public static class AttemptDirectories {
+
+ /**
+ * Job output path.
+ */
+ private final Path outputPath;
+
+ /**
+ * Path for the job attempt.
+ */
+ private final Path jobAttemptDir;
+
+ /**
+ * Path for the job.
+ */
+ private final Path jobPath;
+
+ /**
+ * Subdir under the job attempt dir where task
+ * attempts will have subdirectories.
+ */
+ private final Path jobAttemptTaskSubDir;
+
+ /**
+ * temp directory under job dest dir.
+ */
+ private final Path outputTempSubDir;
+
+ /**
+ * Directory to save manifests into.
+ */
+ private final Path taskManifestDir;
+
+ /**
+ * Build the attempt directories.
+ * @param outputPath output path
+ * @param jobUniqueId job ID/UUID
+ * @param jobAttemptNumber job attempt number
+ */
+ public AttemptDirectories(
+ Path outputPath,
+ String jobUniqueId,
+ int jobAttemptNumber) {
+ this.outputPath = requireNonNull(outputPath, "Output path");
+
+ this.outputTempSubDir = new Path(outputPath, PENDING_DIR_NAME);
+ // build the path for the job
+ this.jobPath = new Path(outputTempSubDir,
+ String.format(JOB_DIR_FORMAT_STR, jobUniqueId));
+
+ // then the specific path underneath that for the attempt.
+ this.jobAttemptDir = new Path(jobPath,
+ String.format(JOB_ATTEMPT_DIR_FORMAT_STR, jobAttemptNumber));
+
+ // subdir for task attempts.
+ this.jobAttemptTaskSubDir = new Path(jobAttemptDir, JOB_TASK_ATTEMPT_SUBDIR);
+
+ this.taskManifestDir = new Path(jobAttemptDir, JOB_TASK_MANIFEST_SUBDIR);
+ }
+
+ public Path getOutputPath() {
+ return outputPath;
+ }
+
+ public Path getJobAttemptDir() {
+ return jobAttemptDir;
+ }
+
+ public Path getJobPath() {
+ return jobPath;
+ }
+
+ public Path getJobAttemptTaskSubDir() {
+ return jobAttemptTaskSubDir;
+ }
+
+ public Path getTaskAttemptPath(String taskAttemptId) {
+ return new Path(jobAttemptTaskSubDir, taskAttemptId);
+ }
+
+ public Path getOutputTempSubDir() {
+ return outputTempSubDir;
+ }
+
+ public Path getTaskManifestDir() {
+ return taskManifestDir;
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java
new file mode 100644
index 0000000..b81fa9d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperations.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * FileSystem operations which are needed to generate the task manifest.
+ * The specific choice of which implementation to use is configurable.
+ * Object store implementations MAY subclass if they
+ * need to implement resilient commit operations.
+ * However, the actual API MUST NOT be used outside
+ * the manifest committer and its tests.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce, object-stores")
+@InterfaceStability.Unstable
+public abstract class ManifestStoreOperations implements Closeable {
+
+ /**
+ * Bind to the filesystem.
+ * This is called by the manifest committer after the operations
+ * have been instantiated.
+ * @param fileSystem target FS
+ * @param path actual path under FS.
+ * @throws IOException if there are binding problems.
+ */
+ public void bindToFileSystem(FileSystem fileSystem, Path path) throws IOException {
+
+ }
+
+ /**
+ * Forward to {@link FileSystem#getFileStatus(Path)}.
+ * @param path path
+ * @return status
+ * @throws IOException failure.
+ */
+ public abstract FileStatus getFileStatus(Path path) throws IOException;
+
+ /**
+ * Is a path a file? Used during directory creation.
+ * The is a copy & paste of FileSystem.isFile();
+ * {@code StoreOperationsThroughFileSystem} calls into
+ * the FS direct so that stores which optimize their probes
+ * can save on IO.
+ * @param path path to probe
+ * @return true if the path exists and resolves to a file
+ * @throws IOException failure other than FileNotFoundException
+ */
+ public boolean isFile(Path path) throws IOException {
+ try {
+ return getFileStatus(path).isFile();
+ } catch (FileNotFoundException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Forward to {@link FileSystem#delete(Path, boolean)}.
+ * If it returns without an error: there is nothing at
+ * the end of the path.
+ * @param path path
+ * @param recursive recursive delete.
+ * @return true if the path was deleted.
+ * @throws IOException failure.
+ */
+ public abstract boolean delete(Path path, boolean recursive)
+ throws IOException;
+
+ /**
+ * Forward to {@link FileSystem#mkdirs(Path)}.
+ * Usual "what does 'false' mean" ambiguity.
+ * @param path path
+ * @return true if the directory was created.
+ * @throws IOException failure.
+ */
+ public abstract boolean mkdirs(Path path) throws IOException;
+
+ /**
+ * Forward to {@link FileSystem#rename(Path, Path)}.
+ * Usual "what does 'false' mean" ambiguity.
+ * @param source source file
+ * @param dest destination path -which must not exist.
+ * @return the return value of the rename
+ * @throws IOException failure.
+ */
+ public abstract boolean renameFile(Path source, Path dest)
+ throws IOException;
+
+ /**
+ * Rename a dir; defaults to invoking
+ * Forward to {@link #renameFile(Path, Path)}.
+ * Usual "what does 'false' mean?" ambiguity.
+ * @param source source file
+ * @param dest destination path -which must not exist.
+ * @return true if the directory was created.
+ * @throws IOException failure.
+ */
+ public boolean renameDir(Path source, Path dest)
+ throws IOException {
+ return renameFile(source, dest);
+ }
+
+ /**
+ * List the directory.
+ * @param path path to list.
+ * @return an iterator over the results.
+ * @throws IOException any immediate failure.
+ */
+ public abstract RemoteIterator<FileStatus> listStatusIterator(Path path)
+ throws IOException;
+
+ /**
+ * Load a task manifest from the store.
+ * with a real FS, this is done with
+ * {@link TaskManifest#load(JsonSerialization, FileSystem, Path, FileStatus)}
+ *
+ * @param serializer serializer.
+ * @param st status with the path and other data.
+ * @return the manifest
+ * @throws IOException failure to load/parse
+ */
+ public abstract TaskManifest loadTaskManifest(
+ JsonSerialization<TaskManifest> serializer,
+ FileStatus st) throws IOException;
+
+ /**
+ * Save a task manifest by {@code FileSystem.create(path)}.
+ * there's no attempt at renaming anything here.
+ * @param manifestData the manifest/success file
+ * @param path temp path for the initial save
+ * @param overwrite should create(overwrite=true) be used?
+ * @throws IOException failure to load/parse
+ */
+ public abstract <T extends AbstractManifestData<T>> void save(
+ T manifestData,
+ Path path,
+ boolean overwrite) throws IOException;
+
+ /**
+ * Make an msync() call; swallow when unsupported.
+ * @param path path
+ * @throws IOException IO failure
+ */
+ public void msync(Path path) throws IOException {
+
+ }
+
+
+ /**
+ * Extract an etag from a status if the conditions are met.
+ * If the conditions are not met, return null or ""; they will
+ * both be treated as "no etags available"
+ * <pre>
+ * 1. The status is of a type which the implementation recognizes
+ * as containing an etag.
+ * 2. After casting the etag field can be retrieved
+ * 3. and that value is non-null/non-empty.
+ * </pre>
+ * @param status status, which may be null of any subclass of FileStatus.
+ * @return either a valid etag, or null or "".
+ */
+ public String getEtag(FileStatus status) {
+ return ManifestCommitterSupport.getEtag(status);
+ }
+
+ /**
+ * Does the store preserve etags through renames.
+ * If true, and if the source listing entry has an etag,
+ * it will be used to attempt to validate a failed rename.
+ * @param path path to probe.
+ * @return true if etag comparison is a valid strategy.
+ */
+ public boolean storePreservesEtagsThroughRenames(Path path) {
+ return false;
+ }
+
+ /**
+ * Does the store provide rename resilience through an
+ * implementation of {@link #commitFile(FileEntry)}?
+ * If true then that method will be invoked to commit work
+ * @return true if resilient commit support is available.
+ */
+ public boolean storeSupportsResilientCommit() {
+ return false;
+ }
+
+ /**
+ * Commit one file through any resilient API.
+ * This operation MUST rename source to destination,
+ * else raise an exception.
+ * The result indicates whether or not some
+ * form of recovery took place.
+ *
+ * If etags were collected during task commit, these will be
+ * in the entries passed in here.
+ *
+ * The base implementation always raises
+ * {@code UnsupportedOperationException}
+ * @param entry entry to commit
+ * @return the result of the commit
+ * @throws IOException failure.
+ * @throws UnsupportedOperationException if not available.
+ *
+ */
+ public CommitFileResult commitFile(FileEntry entry) throws IOException {
+ throw new UnsupportedOperationException("Resilient commit not supported");
+ }
+
+ /**
+ * Outcome from the operation {@link #commitFile(FileEntry)}.
+ * As a rename failure MUST raise an exception, this result
+ * only declares whether or not some form of recovery took place.
+ */
+ public static final class CommitFileResult {
+
+ /** Did recovery take place? */
+ private final boolean recovered;
+
+ /** Time waiting for IO capacity, may be null. */
+ @Nullable
+ private final Duration waitTime;
+
+ /**
+ * Full commit result.
+ * @param recovered Did recovery take place?
+ * @param waitTime any time spent waiting for IO capacity.
+ */
+ public static CommitFileResult fromResilientCommit(
+ final boolean recovered,
+ final Duration waitTime) {
+ return new CommitFileResult(recovered, waitTime);
+ }
+
+ /**
+ * Full commit result.
+ * @param recovered Did recovery take place?
+ * @param waitTime any time spent waiting for IO capacity.
+ */
+ public CommitFileResult(final boolean recovered,
+ @Nullable final Duration waitTime) {
+
+ this.recovered = recovered;
+ this.waitTime = waitTime;
+ }
+
+ /**
+ * Did some form of recovery take place?
+ * @return true if the commit succeeded through some form of (etag-based) recovery
+ */
+ public boolean recovered() {
+ return recovered;
+ }
+
+ @Nullable
+ public Duration getWaitTime() {
+ return waitTime;
+ }
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java
new file mode 100644
index 0000000..9a0b972
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/ManifestStoreOperationsThroughFileSystem.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonPathCapabilities;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Implementation of manifest store operations through the filesystem API.
+ * This class is subclassed in the ABFS module, which does add the resilient
+ * commit method.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce, object-stores")
+@InterfaceStability.Unstable
+public class ManifestStoreOperationsThroughFileSystem extends ManifestStoreOperations {
+
+ /**
+ * Filesystem; set in {@link #bindToFileSystem(FileSystem, Path)}.
+ */
+ private FileSystem fileSystem;
+
+ /**
+ * Has a call to FileSystem.msync() failed as unsupported?
+ * If so, no new attempts will be made when
+ * (@link {@link #msync(Path)} is invoked.
+ */
+ private boolean msyncUnsupported = false;
+
+ /**
+ * Direct Constructor.
+ * @param fileSystem filesystem to write through.
+ */
+ public ManifestStoreOperationsThroughFileSystem(final FileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ }
+
+ /**
+ * Constructor used for introspection-based binding.
+ */
+ public ManifestStoreOperationsThroughFileSystem() {
+ }
+
+ @Override
+ public void close() throws IOException {
+ /* no-op; FS is assumed to be shared. */
+
+ }
+
+ /**
+ * Get the filesystem.
+ * @return the filesystem; null until bound.
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public void bindToFileSystem(FileSystem filesystem, Path path) throws IOException {
+ fileSystem = filesystem;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ return fileSystem.getFileStatus(path);
+ }
+
+ /**
+ * Using FileSystem.isFile to offer stores the option to optimize their probes.
+ * @param path path to probe
+ * @return true if the path resolves to a file.
+ * @throws IOException IO failure.
+ */
+ @SuppressWarnings("deprecation")
+ @Override
+ public boolean isFile(Path path) throws IOException {
+ return fileSystem.isFile(path);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive)
+ throws IOException {
+ return fileSystem.delete(path, recursive);
+ }
+
+ @Override
+ public boolean mkdirs(Path path)
+ throws IOException {
+ return fileSystem.mkdirs(path);
+ }
+
+ @Override
+ public boolean renameFile(Path source, Path dest)
+ throws IOException {
+ return fileSystem.rename(source, dest);
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(Path path)
+ throws IOException {
+ return fileSystem.listStatusIterator(path);
+ }
+
+ @Override
+ public TaskManifest loadTaskManifest(
+ JsonSerialization<TaskManifest> serializer,
+ FileStatus st) throws IOException {
+ return TaskManifest.load(serializer, fileSystem, st.getPath(), st);
+ }
+
+ @Override
+ public <T extends AbstractManifestData<T>> void save(
+ final T manifestData,
+ final Path path,
+ final boolean overwrite) throws IOException {
+ manifestData.save(fileSystem, path, overwrite);
+ }
+
+ /**
+ * Probe filesystem capabilities.
+ * @param path path to probe.
+ * @return true if the FS declares its renames work.
+ */
+ @Override
+ public boolean storePreservesEtagsThroughRenames(Path path) {
+ try {
+ return fileSystem.hasPathCapability(path,
+ CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME);
+ } catch (IOException ignored) {
+ return false;
+ }
+ }
+
+ /**
+ * Invokes FileSystem msync(); swallows UnsupportedOperationExceptions.
+ * This ensures client metadata caches are in sync in an HDFS-HA deployment.
+ * No other filesystems support this; in the absence of a hasPathCapability()
+ * probe, after the operation is rejected, an atomic boolean is set
+ * to stop further attempts from even trying.
+ * @param path path
+ * @throws IOException failure to synchronize.
+ */
+ @Override
+ public void msync(Path path) throws IOException {
+ // there's need for atomicity here, as the sole cost of
+ // multiple failures
+ if (msyncUnsupported) {
+ return;
+ }
+ // qualify so we can be confident that the FS being synced
+ // is the one we expect.
+ fileSystem.makeQualified(path);
+ try {
+ fileSystem.msync();
+ } catch (UnsupportedOperationException ignored) {
+ // this exception is the default.
+ // set the unsupported flag so no future attempts are made.
+ msyncUnsupported = true;
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java
new file mode 100644
index 0000000..f1dacc2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/OutputValidationException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Exception raised during validation.
+ * This can be treated differently from other outcomes.
+ */
+@InterfaceAudience.Private
+public class OutputValidationException extends PathIOException {
+ public OutputValidationException(Path path, String error) {
+ super(path.toUri().toString(), error);
+ }
+
+ public OutputValidationException(Path path,
+ String error,
+ Throwable cause) {
+ super(path.toUri().toString(), error, cause);
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
new file mode 100644
index 0000000..eb7dda6
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/impl/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation classes for the manifest committer.
+ * Nothing outside this module should be using these classes,
+ * except where explicitly stated.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
new file mode 100644
index 0000000..c00ae5a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/package-info.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Intermediate manifest committer.
+ *
+ * Optimized for object stores where listing is slow, directory renames may not
+ * be atomic, and the output is a deep tree of files intermixed with
+ * the output of (many) other task attempts.
+ *
+ * All classes in this module are private/unstable, except where stated.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java
new file mode 100644
index 0000000..c2b44c2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbortTaskStage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_ABORT_TASK;
+
+/**
+ * Abort a task.
+ *
+ * This is done by deleting the task directory.
+ * Exceptions may/may not be suppressed.
+ */
+public class AbortTaskStage extends
+ AbstractJobOrTaskStage<Boolean, Path> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AbortTaskStage.class);
+
+ public AbortTaskStage(final StageConfig stageConfig) {
+ super(true, stageConfig, OP_STAGE_TASK_ABORT_TASK, false);
+ }
+
+ /**
+ * Delete the task attempt directory.
+ * @param suppressExceptions should exceptions be ignored?
+ * @return the directory
+ * @throws IOException failure when exceptions were not suppressed
+ */
+ @Override
+ protected Path executeStage(final Boolean suppressExceptions)
+ throws IOException {
+ final Path dir = getTaskAttemptDir();
+ if (dir != null) {
+ LOG.info("{}: Deleting task attempt directory {}", getName(), dir);
+ deleteDir(dir, suppressExceptions);
+ }
+ return dir;
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java
new file mode 100644
index 0000000..05ee7a5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/AbstractJobOrTaskStage.java
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.DurationTracker;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.AbstractManifestData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.util.OperationDuration;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.functional.CallableRaisingIOE;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_IS_FILE;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_LIST_STATUS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_MKDIRS;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.STORE_IO_RATE_LIMITED;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.createTracker;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.MANIFEST_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_COMMIT_FILE_RENAME_RECOVERED;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_LOAD_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_MSYNC;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_RENAME_FILE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_SAVE_TASK_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.AuditingIntegration.enterStageWorker;
+
+/**
+ * A Stage in Task/Job Commit.
+ * A stage can be executed once only, creating the return value of the
+ * {@link #apply(Object)} method, and, potentially, updating the state of the
+ * store via {@link ManifestStoreOperations}.
+ * IOStatistics will also be updated.
+ * Stages are expected to be combined to form the commit protocol.
+ * @param <IN> Type of arguments to the stage.
+ * @param <OUT> Type of result.
+ */
+public abstract class AbstractJobOrTaskStage<IN, OUT>
+ implements JobOrTaskStage<IN, OUT> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AbstractJobOrTaskStage.class);
+
+ /**
+ * Error text on rename failure: {@value}.
+ */
+ public static final String FAILED_TO_RENAME_PREFIX = "Failed to ";
+
+ /**
+ * Is this a task stage? If so, toString() includes task
+ * info..
+ */
+ private final boolean isTaskStage;
+
+ /**
+ * Configuration of all the stages in the ongoing committer
+ * operation.
+ */
+ private final StageConfig stageConfig;
+
+ /**
+ * Name of the stage for statistics and logging.
+ */
+ private final String stageStatisticName;
+
+ /**
+ * Callbacks to update store.
+ * This is not made visible to the stages; they must
+ * go through the wrapper classes in this class, which
+ * add statistics and logging.
+ */
+ private final ManifestStoreOperations operations;
+
+ /**
+ * Submitter for doing IO against the store.
+ */
+ private final TaskPool.Submitter ioProcessors;
+
+ /**
+ * Used to stop any re-entrancy of the rename.
+ * This is an execute-once operation.
+ */
+ private final AtomicBoolean executed = new AtomicBoolean(false);
+
+ /**
+ * Tracker of the duration of the execution of the stage.
+ * set after {@link #executeStage(Object)} completes.
+ */
+ private DurationTracker stageExecutionTracker;
+
+ /**
+ * Name for logging.
+ */
+ private final String name;
+
+ /**
+ * Constructor.
+ * @param isTaskStage Is this a task stage?
+ * @param stageConfig stage-independent configuration.
+ * @param stageStatisticName name of the stage for statistics/logging
+ * @param requireIOProcessors are the IO processors required?
+ */
+ protected AbstractJobOrTaskStage(
+ final boolean isTaskStage,
+ final StageConfig stageConfig,
+ final String stageStatisticName,
+ final boolean requireIOProcessors) {
+ this.isTaskStage = isTaskStage;
+ this.stageStatisticName = stageStatisticName;
+ this.stageConfig = stageConfig;
+ requireNonNull(stageConfig.getDestinationDir(), "Destination Directory");
+ requireNonNull(stageConfig.getJobId(), "Job ID");
+ requireNonNull(stageConfig.getJobAttemptDir(), "Job attempt directory");
+ this.operations = requireNonNull(stageConfig.getOperations(),
+ "Operations callbacks");
+ // and the processors of work if required.
+ this.ioProcessors = bindProcessor(
+ requireIOProcessors,
+ stageConfig.getIoProcessors());
+ String stageName;
+ if (isTaskStage) {
+ // force fast failure.
+ getRequiredTaskId();
+ getRequiredTaskAttemptId();
+ getRequiredTaskAttemptDir();
+ stageName = String.format("[Task-Attempt %s]", getRequiredTaskAttemptId());
+ } else {
+ stageName = String.format("[Job-Attempt %s/%02d]",
+ stageConfig.getJobId(),
+ stageConfig.getJobAttemptNumber());
+ }
+ name = stageName;
+ }
+
+ /**
+ * Bind to the processor if it is required.
+ * @param required is the processor required?
+ * @param processor processor
+ * @return the processor binding
+ * @throws NullPointerException if required == true and processor is null.
+ */
+ private TaskPool.Submitter bindProcessor(
+ final boolean required,
+ final TaskPool.Submitter processor) {
+ return required
+ ? requireNonNull(processor, "required IO processor is null")
+ : null;
+ }
+
+ /**
+ * Stage entry point.
+ * Verifies that this is the first and only time the stage is invoked,
+ * then calls {@link #executeStage(Object)} for the subclass
+ * to perform its part of the commit protocol.
+ * The duration of the stage is collected as a statistic, and its
+ * entry/exit logged at INFO.
+ * @param arguments arguments to the function.
+ * @return the result.
+ * @throws IOException failures.
+ */
+ @Override
+ public final OUT apply(final IN arguments) throws IOException {
+ executeOnlyOnce();
+ progress();
+ String stageName = getStageName(arguments);
+ getStageConfig().enterStage(stageName);
+ String statisticName = getStageStatisticName(arguments);
+ // The tracker here
+ LOG.info("{}: Executing Stage {}", getName(), stageName);
+ stageExecutionTracker = createTracker(getIOStatistics(), statisticName);
+ try {
+ // exec the input function and return its value
+ final OUT out = executeStage(arguments);
+ LOG.info("{}: Stage {} completed after {}",
+ getName(),
+ stageName,
+ OperationDuration.humanTime(
+ stageExecutionTracker.asDuration().toMillis()));
+ return out;
+ } catch (IOException | RuntimeException e) {
+ LOG.error("{}: Stage {} failed: after {}: {}",
+ getName(),
+ stageName,
+ OperationDuration.humanTime(
+ stageExecutionTracker.asDuration().toMillis()),
+ e.toString());
+ LOG.debug("{}: Stage failure:", getName(), e);
+ // input function failed: note it
+ stageExecutionTracker.failed();
+ // and rethrow
+ throw e;
+ } finally {
+ // update the tracker.
+ // this is called after the catch() call would have
+ // set the failed flag.
+ stageExecutionTracker.close();
+ progress();
+ getStageConfig().exitStage(stageName);
+ }
+ }
+
+ /**
+ * The work of a stage.
+ * Executed exactly once.
+ * @param arguments arguments to the function.
+ * @return the result.
+ * @throws IOException failures.
+ */
+ protected abstract OUT executeStage(IN arguments) throws IOException;
+
+ /**
+ * Check that the operation has not been invoked twice.
+ * This is an atomic check.
+ * @throws IllegalStateException on a second invocation.
+ */
+ private void executeOnlyOnce() {
+ Preconditions.checkState(
+ !executed.getAndSet(true),
+ "Stage attempted twice");
+ }
+
+ /**
+ * The stage statistic name.
+ * @param arguments args to the invocation.
+ * @return stage name.
+ */
+ protected String getStageStatisticName(IN arguments) {
+ return stageStatisticName;
+ }
+
+ /**
+ * Stage name for reporting; defaults to
+ * call {@link #getStageStatisticName(IN)}.
+ * @param arguments args to the invocation.
+ * @return name used in updating reports.
+ */
+ protected String getStageName(IN arguments) {
+ return getStageStatisticName(arguments);
+ }
+
+ /**
+ * Get the execution tracker; non-null
+ * after stage execution.
+ * @return a tracker or null.
+ */
+ public DurationTracker getStageExecutionTracker() {
+ return stageExecutionTracker;
+ }
+
+ /**
+ * Adds the duration of the job to an IOStatistics store
+ * (such as the manifest to be saved).
+ * @param iostats store
+ * @param statistic statistic name.
+ */
+ public void addExecutionDurationToStatistics(IOStatisticsStore iostats,
+ String statistic) {
+ iostats.addTimedOperation(
+ statistic,
+ getStageExecutionTracker().asDuration());
+ }
+
+ /**
+ * Note any rate limiting to the given timing statistic.
+ * If the wait was 0, no statistics are updated.
+ * @param statistic statistic key.
+ * @param wait wait duration.
+ */
+ private void noteAnyRateLimiting(String statistic, Duration wait) {
+ if (!wait.isZero()) {
+ // rate limiting took place
+ getIOStatistics().addTimedOperation(
+ statistic,
+ wait.toMillis());
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "AbstractJobOrTaskStage{");
+ sb.append(isTaskStage ? "Task Stage" : "Job Stage");
+ sb.append(" name='").append(name).append('\'');
+ sb.append(" stage='").append(stageStatisticName).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * The stage configuration.
+ * @return the stage configuration used by this stage.
+ */
+ protected StageConfig getStageConfig() {
+ return stageConfig;
+ }
+
+ /**
+ * Update the thread context with the stage name and
+ * job ID.
+ * This MUST be invoked at the start of methods invoked in helper threads,
+ * to ensure that they are all annotated with job and stage.
+ * @param stage stage name.
+ */
+ protected void updateAuditContext(final String stage) {
+ enterStageWorker(stageConfig.getJobId(), stage);
+ }
+
+ /**
+ * The IOStatistics are shared across all uses of the
+ * StageConfig.
+ * @return the (possibly shared) IOStatistics.
+ */
+ @Override
+ public final IOStatisticsStore getIOStatistics() {
+ return stageConfig.getIOStatistics();
+ }
+
+ /**
+ * Call progress() on any Progressable passed in.
+ */
+ protected final void progress() {
+ if (stageConfig.getProgressable() != null) {
+ stageConfig.getProgressable().progress();
+ }
+ }
+
+ /**
+ * Get a file status value or, if the path doesn't exist, return null.
+ * @param path path
+ * @return status or null
+ * @throws IOException IO Failure.
+ */
+ protected final FileStatus getFileStatusOrNull(
+ final Path path)
+ throws IOException {
+ try {
+ return getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Get a file status value or, if the path doesn't exist, return null.
+ * @param path path
+ * @return status or null
+ * @throws IOException IO Failure.
+ */
+ protected final FileStatus getFileStatus(
+ final Path path)
+ throws IOException {
+ LOG.trace("{}: getFileStatus('{}')", getName(), path);
+ requireNonNull(path,
+ () -> String.format("%s: Null path for getFileStatus() call", getName()));
+ return trackDuration(getIOStatistics(), OP_GET_FILE_STATUS, () ->
+ operations.getFileStatus(path));
+ }
+
+ /**
+ * Get a file status value or, if the path doesn't exist, return null.
+ * @param path path
+ * @return true if the path resolves to a file
+ * @throws IOException IO Failure.
+ */
+ protected final boolean isFile(
+ final Path path)
+ throws IOException {
+ LOG.trace("{}: isFile('{}')", getName(), path);
+ return trackDuration(getIOStatistics(), OP_IS_FILE, () -> {
+ return operations.isFile(path);
+ });
+ }
+
+ /**
+ * Delete a path.
+ * @param path path
+ * @param recursive recursive delete.
+ * @return status or null
+ * @throws IOException IO Failure.
+ */
+ protected final boolean delete(
+ final Path path,
+ final boolean recursive)
+ throws IOException {
+ LOG.trace("{}: delete('{}, {}')", getName(), path, recursive);
+ return delete(path, recursive, OP_DELETE);
+ }
+
+ /**
+ * Delete a path.
+ * @param path path
+ * @param recursive recursive delete.
+ * @param statistic statistic to update
+ * @return status or null
+ * @throws IOException IO Failure.
+ */
+ protected Boolean delete(
+ final Path path,
+ final boolean recursive,
+ final String statistic)
+ throws IOException {
+ return trackDuration(getIOStatistics(), statistic, () -> {
+ return operations.delete(path, recursive);
+ });
+ }
+
+ /**
+ * Create a directory.
+ * @param path path
+ * @param escalateFailure escalate "false" to PathIOE
+ * @return true if the directory was created/exists.
+ * @throws IOException IO Failure.
+ */
+ protected final boolean mkdirs(
+ final Path path,
+ final boolean escalateFailure)
+ throws IOException {
+ LOG.trace("{}: mkdirs('{}')", getName(), path);
+ return trackDuration(getIOStatistics(), OP_MKDIRS, () -> {
+ boolean success = operations.mkdirs(path);
+ if (!success && escalateFailure) {
+ throw new PathIOException(path.toUri().toString(),
+ stageStatisticName + ": mkdirs() returned false");
+ }
+ return success;
+ });
+
+ }
+
+ /**
+ * List all directly files under a path.
+ * Async implementations may under-report their durations.
+ * @param path path
+ * @return iterator over the results.
+ * @throws IOException IO Failure.
+ */
+ protected final RemoteIterator<FileStatus> listStatusIterator(
+ final Path path)
+ throws IOException {
+ LOG.trace("{}: listStatusIterator('{}')", getName(), path);
+ return trackDuration(getIOStatistics(), OP_LIST_STATUS, () ->
+ operations.listStatusIterator(path));
+ }
+
+ /**
+ * Load a manifest file.
+ * @param status source.
+ * @return the manifest.
+ * @throws IOException IO Failure.
+ */
+ protected final TaskManifest loadManifest(
+ final FileStatus status)
+ throws IOException {
+ LOG.trace("{}: loadManifest('{}')", getName(), status);
+ return trackDuration(getIOStatistics(), OP_LOAD_MANIFEST, () ->
+ operations.loadTaskManifest(
+ stageConfig.currentManifestSerializer(),
+ status));
+ }
+
+ /**
+ * List all the manifests in the task manifest dir.
+ * @return a iterator of manifests.
+ * @throws IOException IO Failure.
+ */
+ protected final RemoteIterator<FileStatus> listManifests()
+ throws IOException {
+ return RemoteIterators.filteringRemoteIterator(
+ listStatusIterator(getTaskManifestDir()),
+ st -> st.getPath().toUri().toString().endsWith(MANIFEST_SUFFIX));
+ }
+
+ /**
+ * Make an msync() call; swallow when unsupported.
+ * @param path path
+ * @throws IOException IO failure
+ */
+ protected final void msync(Path path) throws IOException {
+ LOG.trace("{}: msync('{}')", getName(), path);
+ trackDurationOfInvocation(getIOStatistics(), OP_MSYNC, () ->
+ operations.msync(path));
+ }
+
+ /**
+ * Create a directory -failing if it exists or if
+ * mkdirs() failed.
+ * @param operation operation for error reporting.
+ * @param path path path to create.
+ * @return the path.
+ * @throws IOException failure
+ * @throws PathIOException mkdirs failed.
+ * @throws FileAlreadyExistsException destination exists.
+ */
+ protected final Path createNewDirectory(
+ final String operation,
+ final Path path) throws IOException {
+ LOG.trace("{}: {} createNewDirectory('{}')", getName(), operation, path);
+ requireNonNull(path,
+ () -> String.format("%s: Null path for operation %s", getName(), operation));
+ // check for dir existence before trying to create.
+ try {
+ final FileStatus status = getFileStatus(path);
+ // no exception, so the path exists.
+ throw new FileAlreadyExistsException(operation
+ + ": path " + path
+ + " already exists and has status " + status);
+ } catch (FileNotFoundException e) {
+ // the path does not exist, so create it.
+ mkdirs(path, true);
+ return path;
+ }
+ }
+
+ /**
+ * Assert that a path is a directory which must exist.
+ * @param operation operation for error reporting.
+ * @param path path path to create.
+ * @return the path
+ * @throws IOException failure
+ * @throws PathIOException mkdirs failed.
+ * @throws FileAlreadyExistsException destination exists.
+ */
+ protected final Path directoryMustExist(
+ final String operation,
+ final Path path) throws IOException {
+ final FileStatus status = getFileStatus(path);
+ if (!status.isDirectory()) {
+ throw new PathIOException(path.toString(),
+ operation
+ + ": Path is not a directory; its status is :" + status);
+ }
+ return path;
+ }
+
+ /**
+ * Save a task manifest or summary. This will be done by
+ * writing to a temp path and then renaming.
+ * If the destination path exists: Delete it.
+ * @param manifestData the manifest/success file
+ * @param tempPath temp path for the initial save
+ * @param finalPath final path for rename.
+ * @throws IOException failure to load/parse
+ */
+ @SuppressWarnings("unchecked")
+ protected final <T extends AbstractManifestData> void save(T manifestData,
+ final Path tempPath,
+ final Path finalPath) throws IOException {
+ LOG.trace("{}: save('{}, {}, {}')", getName(), manifestData, tempPath, finalPath);
+ trackDurationOfInvocation(getIOStatistics(), OP_SAVE_TASK_MANIFEST, () ->
+ operations.save(manifestData, tempPath, true));
+ renameFile(tempPath, finalPath);
+ }
+
+ /**
+ * Get an etag from a FileStatus which MUST BE
+ * an implementation of EtagSource and
+ * whose etag MUST NOT BE null/empty.
+ * @param status the status; may be null.
+ * @return the etag or null if not provided
+ */
+ public String getEtag(FileStatus status) {
+ return operations.getEtag(status);
+ }
+
+ /**
+ * Rename a file from source to dest; if the underlying FS API call
+ * returned false that's escalated to an IOE.
+ * @param source source file.
+ * @param dest dest file
+ * @throws IOException failure
+ * @throws PathIOException if the rename() call returned false.
+ */
+ protected final void renameFile(final Path source, final Path dest)
+ throws IOException {
+ maybeDeleteDest(true, dest);
+ executeRenamingOperation("renameFile", source, dest,
+ OP_RENAME_FILE, () ->
+ operations.renameFile(source, dest));
+ }
+
+ /**
+ * Rename a file from source to dest; if the underlying FS API call
+ * returned false that's escalated to an IOE.
+ * @param source source file.
+ * @param dest dest file
+ * @throws IOException failure
+ * @throws PathIOException if the rename() call returned false.
+ */
+ protected final void renameDir(final Path source, final Path dest)
+ throws IOException {
+
+ maybeDeleteDest(true, dest);
+ executeRenamingOperation("renameDir", source, dest,
+ OP_RENAME_FILE, () ->
+ operations.renameDir(source, dest)
+ );
+ }
+
+ /**
+ * Commit a file from the manifest using rename or, if available, resilient renaming.
+ * @param entry entry from manifest
+ * @throws PathIOException if the rename() call returned false and was uprated.
+ * @throws IOException failure
+ */
+ protected final CommitOutcome commitFile(FileEntry entry,
+ boolean deleteDest)
+ throws IOException {
+
+ final Path source = entry.getSourcePath();
+ final Path dest = entry.getDestPath();
+
+ maybeDeleteDest(deleteDest, dest);
+ if (storeSupportsResilientCommit()) {
+ // get the commit permits
+ final ManifestStoreOperations.CommitFileResult result = trackDuration(getIOStatistics(),
+ OP_COMMIT_FILE_RENAME, () ->
+ operations.commitFile(entry));
+ if (result.recovered()) {
+ // recovery took place.
+ getIOStatistics().incrementCounter(OP_COMMIT_FILE_RENAME_RECOVERED);
+ }
+ if (result.getWaitTime() != null) {
+ // note any delay which took place
+ noteAnyRateLimiting(STORE_IO_RATE_LIMITED, result.getWaitTime());
+ }
+ } else {
+ // commit with a simple rename; failures will be escalated.
+ executeRenamingOperation("renameFile", source, dest,
+ OP_COMMIT_FILE_RENAME, () ->
+ operations.renameFile(source, dest));
+ }
+ return new CommitOutcome();
+ }
+
+ /**
+ * Does this store support resilient commit.
+ * @return true if resilient commit operations are available.
+ */
+ protected boolean storeSupportsResilientCommit() {
+ return operations.storeSupportsResilientCommit();
+ }
+
+ private void maybeDeleteDest(final boolean deleteDest, final Path dest) throws IOException {
+ if (deleteDest) {
+ // delete the destination, always, knowing that it's a no-op if
+ // the data isn't there. Skipping the change saves one round trip
+ // to actually look for the file/object
+ boolean deleted = delete(dest, true);
+ // log the outcome in case of emergency diagnostics traces
+ // being needed.
+ LOG.debug("{}: delete('{}') returned {}'", getName(), dest, deleted);
+ }
+ }
+
+ /**
+ * Execute an operation to rename a file/dir, commit a manifest entry.
+ * The statistic is tracked; returning false from the operation is considered
+ * a failure from the statistics perspective.
+ * @param operation operation name
+ * @param source source path
+ * @param dest dest path
+ * @param statistic statistic to track
+ * @param action callable of the operation
+ * @throws IOException on any failure
+ */
+ private void executeRenamingOperation(String operation,
+ Path source,
+ Path dest,
+ String statistic,
+ CallableRaisingIOE<Boolean> action) throws IOException {
+
+ LOG.debug("{}: {} '{}' to '{}')", getName(), operation, source, dest);
+ requireNonNull(source, "Null source");
+ requireNonNull(dest, "Null dest");
+
+ // duration tracking is a bit convoluted as it
+ // ensures that rename failures as well as IOEs are
+ // treated as failures from a statistics perspective.
+
+ DurationTracker tracker = createTracker(getIOStatistics(), statistic);
+ boolean success;
+ try {
+ success = action.apply();
+ if (!success) {
+ // record failure in the tracker before closing it
+ tracker.failed();
+ }
+ } catch (IOException | RuntimeException e) {
+ LOG.info("{}: {} raised an exception: {}", getName(), operation, e.toString());
+ LOG.debug("{}: {} stack trace", getName(), operation, e);
+ tracker.failed();
+ throw e;
+ } finally {
+ // success
+ // update the tracker.
+ tracker.close();
+ }
+ // escalate the failure; this is done out of the duration tracker
+ // so its file status probes aren't included.
+ if (!success) {
+ throw escalateRenameFailure(operation, source, dest);
+ }
+ }
+
+ /**
+ * Escalate a rename failure to an exception.
+ * Returns an error exception to throw if one was not
+ * triggered when probing for the source.
+ * @param operation operation name
+ * @param source source path
+ * @param dest dest path
+ * @return an exception to throw
+ * @throws IOException raised probing for source or dest
+ */
+ private PathIOException escalateRenameFailure(String operation,
+ Path source, Path dest) throws IOException {
+ // rename just returned false.
+ // collect information for a meaningful error message
+ // and include in an exception raised.
+
+ // get the source status; this will implicitly raise a FNFE.
+ final FileStatus sourceStatus = getFileStatus(source);
+
+ // and look to see if there is anything at the destination
+ final FileStatus destStatus = getFileStatusOrNull(dest);
+
+ LOG.error("{}: failure to {} {} to {} with" +
+ " source status {} " +
+ " and destination status {}",
+ getName(), operation, source, dest,
+ sourceStatus, destStatus);
+
+ return new PathIOException(source.toString(),
+ FAILED_TO_RENAME_PREFIX + operation + " to " + dest);
+ }
+
+ /**
+ * Outcome from the commit.
+ */
+ public static final class CommitOutcome {
+
+ }
+
+ /**
+ * Job ID: never null.
+ */
+ protected final String getJobId() {
+ return stageConfig.getJobId();
+ }
+
+ /**
+ * Job attempt number.
+ */
+ protected final int getJobAttemptNumber() {
+ return stageConfig.getJobAttemptNumber();
+ }
+
+ /**
+ * ID of the task.
+ */
+ protected final String getTaskId() {
+ return stageConfig.getTaskId();
+ }
+
+ /**
+ * Get the task ID; raise an NPE
+ * if it is null.
+ * @return a non-null task ID.
+ */
+ protected final String getRequiredTaskId() {
+ return requireNonNull(getTaskId(),
+ "No Task ID in stage config");
+ }
+
+ /**
+ * ID of this specific attempt at a task.
+ */
+ protected final String getTaskAttemptId() {
+ return stageConfig.getTaskAttemptId();
+ }
+
+ /**
+ * Get the task attempt ID; raise an NPE
+ * if it is null.
+ * @return a non-null task attempt ID.
+ */
+ protected final String getRequiredTaskAttemptId() {
+ return requireNonNull(getTaskAttemptId(),
+ "No Task Attempt ID in stage config");
+ }
+
+ /**
+ * Job attempt dir.
+ */
+ protected final Path getJobAttemptDir() {
+ return stageConfig.getJobAttemptDir();
+ }
+
+ /**
+ * Directory to put task manifests into.
+ * @return a path under the job attempt dir.
+ */
+ protected final Path getTaskManifestDir() {
+ return stageConfig.getTaskManifestDir();
+ }
+
+
+ /**
+ * Task attempt dir.
+ */
+ protected final Path getTaskAttemptDir() {
+ return stageConfig.getTaskAttemptDir();
+ }
+
+ /**
+ * Get the task attemptDir; raise an NPE
+ * if it is null.
+ * @return a non-null task attempt dir.
+ */
+ protected final Path getRequiredTaskAttemptDir() {
+ return requireNonNull(getTaskAttemptDir(),
+ "No Task Attempt Dir");
+ }
+
+ /**
+ * Destination of job.
+ */
+ protected final Path getDestinationDir() {
+ return stageConfig.getDestinationDir();
+ }
+
+ /**
+ * Stage confog name, for logging.
+ * @return name.
+ */
+ public final String getName() {
+ return name;
+ }
+
+ /**
+ * Submitter for doing IO against the store other than
+ * manifest processing.
+ */
+ protected final TaskPool.Submitter getIOProcessors() {
+ return ioProcessors;
+ }
+
+ /**
+ * Submitter for doing IO against the store other than
+ * manifest processing.
+ * The size parameter is used to select between sequential
+ * and parallel runners.
+ * no data, or one entry: serial.
+ * everything else, parallel.
+ * @param size number of items.
+ * @return a submitter or null
+ */
+ protected final TaskPool.Submitter getIOProcessors(int size) {
+ return size > 1
+ ? getIOProcessors()
+ : null;
+ }
+
+ /**
+ * Delete a directory, possibly suppressing exceptions.
+ * @param dir directory.
+ * @param suppressExceptions should exceptions be suppressed?
+ * @throws IOException exceptions raised in delete if not suppressed.
+ * @return any exception caught and suppressed
+ */
+ protected IOException deleteDir(
+ final Path dir,
+ final Boolean suppressExceptions)
+ throws IOException {
+ try {
+ delete(dir, true);
+ return null;
+ } catch (IOException ex) {
+ LOG.info("Error deleting {}: {}", dir, ex.toString());
+ if (!suppressExceptions) {
+ throw ex;
+ } else {
+ return ex;
+ }
+ }
+ }
+
+ /**
+ * Create an entry for a file to rename under the destination.
+ * If the store operations supports extracting etags from file status
+ * entries, that is included in the entry
+ * @param status source file
+ * @param destDir destination directory
+ * @return an entry which includes the rename path
+ */
+ protected FileEntry fileEntry(FileStatus status, Path destDir) {
+ // generate a new path under the dest dir
+ Path dest = new Path(destDir, status.getPath().getName());
+ return new FileEntry(status.getPath(),
+ dest,
+ status.getLen(),
+ getEtag(status));
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java
new file mode 100644
index 0000000..4a5971a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CleanupJobStage.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.functional.RemoteIterators;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED;
+import static org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_CLEANUP_PARALLEL_DELETE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CLEANUP;
+
+/**
+ * Clean up a job's temporary directory through parallel delete,
+ * base _temporary delete.
+ * Returns: the outcome of the overall operation
+ * The result is detailed purely for the benefit of tests, which need
+ * to make assertions about error handling and fallbacks.
+ *
+ * There's a few known issues with the azure and GCS stores which
+ * this stage tries to address.
+ * - Google GCS directory deletion is O(entries), so is slower for big jobs.
+ * - Azure storage directory delete, when using OAuth authentication or
+ * when not the store owner triggers a scan down the tree to verify the
+ * caller has the permission to delete each subdir.
+ * If this scan takes over 90s, the operation can time out.
+ *
+ * The main solution for both of these is that task attempts are
+ * deleted in parallel, in different threads.
+ * This will speed up GCS cleanup and reduce the risk of
+ * abfs related timeouts.
+ * Exceptions during cleanup can be suppressed,
+ * so that these do not cause the job to fail.
+ *
+ * Also, some users want to be able to run multiple independent jobs
+ * targeting the same output directory simultaneously.
+ * If one job deletes the directory `__temporary` all the others
+ * will fail.
+ *
+ * This can be addressed by disabling cleanup entirely.
+ *
+ */
+public class CleanupJobStage extends
+ AbstractJobOrTaskStage<
+ CleanupJobStage.Arguments,
+ CleanupJobStage.Result> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CleanupJobStage.class);
+
+ /**
+ * Count of deleted directories.
+ */
+ private final AtomicInteger deleteDirCount = new AtomicInteger();
+
+ /**
+ * Count of delete failures.
+ */
+ private final AtomicInteger deleteFailureCount = new AtomicInteger();
+
+ /**
+ * Last delete exception; non null if deleteFailureCount is not zero.
+ */
+ private IOException lastDeleteException;
+
+ /**
+ * Stage name as passed in from arguments.
+ */
+ private String stageName = OP_STAGE_JOB_CLEANUP;
+
+ public CleanupJobStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_CLEANUP, true);
+ }
+
+ /**
+ * Statistic name is extracted from the arguments.
+ * @param arguments args to the invocation.
+ * @return stage name.
+ */
+ @Override
+ protected String getStageStatisticName(Arguments arguments) {
+ return arguments.statisticName;
+ }
+
+ /**
+ * Clean up the job attempt directory tree.
+ * @param args arguments built up.
+ * @return the result.
+ * @throws IOException failure was raised an exceptions weren't surpressed.
+ */
+ @Override
+ protected Result executeStage(
+ final Arguments args)
+ throws IOException {
+ stageName = getStageName(args);
+ // this is $dest/_temporary
+ final Path baseDir = requireNonNull(getStageConfig().getOutputTempSubDir());
+ LOG.debug("{}: Cleaup of directory {} with {}", getName(), baseDir, args);
+ if (!args.enabled) {
+ LOG.info("{}: Cleanup of {} disabled", getName(), baseDir);
+ return new Result(Outcome.DISABLED, baseDir,
+ 0, null);
+ }
+ // shortcut of a single existence check before anything else
+ if (getFileStatusOrNull(baseDir) == null) {
+ return new Result(Outcome.NOTHING_TO_CLEAN_UP,
+ baseDir,
+ 0, null);
+ }
+
+ Outcome outcome = null;
+ IOException exception;
+
+
+ // to delete.
+ LOG.info("{}: Deleting job directory {}", getName(), baseDir);
+
+ if (args.deleteTaskAttemptDirsInParallel) {
+ // Attempt to do a parallel delete of task attempt dirs;
+ // don't overreact if a delete fails, but stop trying
+ // to delete the others, and fall back to deleting the
+ // job dir.
+ Path taskSubDir
+ = getStageConfig().getJobAttemptTaskSubDir();
+ try (DurationInfo info = new DurationInfo(LOG,
+ "parallel deletion of task attempts in %s",
+ taskSubDir)) {
+ RemoteIterator<FileStatus> dirs =
+ RemoteIterators.filteringRemoteIterator(
+ listStatusIterator(taskSubDir),
+ FileStatus::isDirectory);
+ TaskPool.foreach(dirs)
+ .executeWith(getIOProcessors())
+ .stopOnFailure()
+ .suppressExceptions(false)
+ .run(this::rmTaskAttemptDir);
+ getIOStatistics().aggregate((retrieveIOStatistics(dirs)));
+
+ if (getLastDeleteException() != null) {
+ // one of the task attempts failed.
+ throw getLastDeleteException();
+ }
+ // success: record this as the outcome.
+ outcome = Outcome.PARALLEL_DELETE;
+ } catch (FileNotFoundException ex) {
+ // not a problem if there's no dir to list.
+ LOG.debug("{}: Task attempt dir {} not found", getName(), taskSubDir);
+ outcome = Outcome.DELETED;
+ } catch (IOException ex) {
+ // failure. Log and continue
+ LOG.info(
+ "{}: Exception while listing/deleting task attempts under {}; continuing",
+ getName(),
+ taskSubDir, ex);
+ // not overreacting here as the base delete will still get executing
+ outcome = Outcome.DELETED;
+ }
+ }
+ // Now the top-level deletion; exception gets saved
+ exception = deleteOneDir(baseDir);
+ if (exception != null) {
+ // failure, report and continue
+ // assume failure.
+ outcome = Outcome.FAILURE;
+ } else {
+ // if the outcome isn't already recorded as parallel delete,
+ // mark is a simple delete.
+ if (outcome == null) {
+ outcome = Outcome.DELETED;
+ }
+ }
+
+ Result result = new Result(
+ outcome,
+ baseDir,
+ deleteDirCount.get(),
+ exception);
+ if (!result.succeeded() && !args.suppressExceptions) {
+ result.maybeRethrowException();
+ }
+
+ return result;
+ }
+
+ /**
+ * Delete a single TA dir in a parallel task.
+ * Updates the audit context.
+ * Exceptions are swallowed so that attempts are still made
+ * to delete the others, but the first exception
+ * caught is saved in a field which can be retrieved
+ * via {@link #getLastDeleteException()}.
+ *
+ * @param status dir to be deleted.
+ * @throws IOException delete failure.
+ */
+ private void rmTaskAttemptDir(FileStatus status) throws IOException {
+ // stage name in audit context is the one set in the arguments.
+ updateAuditContext(stageName);
+ // update the progress callback in case delete is really slow.
+ progress();
+ deleteOneDir(status.getPath());
+ }
+
+ /**
+ * Delete a directory.
+ * The {@link #deleteFailureCount} counter.
+ * is incremented on every failure.
+ * @param dir directory
+ * @throws IOException if an IOE was raised
+ * @return any IOE raised.
+ */
+ private IOException deleteOneDir(final Path dir)
+ throws IOException {
+
+ deleteDirCount.incrementAndGet();
+ IOException ex = deleteDir(dir, true);
+ if (ex != null) {
+ deleteFailure(ex);
+ }
+ return ex;
+ }
+
+ /**
+ * Note a failure.
+ * @param ex exception
+ */
+ private synchronized void deleteFailure(IOException ex) {
+ // excaption: add the count
+ deleteFailureCount.incrementAndGet();
+ lastDeleteException = ex;
+ }
+
+ /**
+ * Get the last delete exception; synchronized.
+ * @return the last delete exception or null.
+ */
+ public synchronized IOException getLastDeleteException() {
+ return lastDeleteException;
+ }
+
+ /**
+ * Options to pass down to the cleanup stage.
+ */
+ public static final class Arguments {
+
+ /**
+ * Statistic to update.
+ */
+ private final String statisticName;
+
+ /** Delete is enabled? */
+ private final boolean enabled;
+
+ /** Attempt parallel delete of task attempt dirs? */
+ private final boolean deleteTaskAttemptDirsInParallel;
+
+ /** Ignore failures? */
+ private final boolean suppressExceptions;
+
+ /**
+ * Arguments to the stage.
+ * @param statisticName stage name to report
+ * @param enabled is the stage enabled?
+ * @param deleteTaskAttemptDirsInParallel delete task attempt dirs in
+ * parallel?
+ * @param suppressExceptions suppress exceptions?
+ */
+ public Arguments(
+ final String statisticName,
+ final boolean enabled,
+ final boolean deleteTaskAttemptDirsInParallel,
+ final boolean suppressExceptions) {
+ this.statisticName = statisticName;
+ this.enabled = enabled;
+ this.deleteTaskAttemptDirsInParallel = deleteTaskAttemptDirsInParallel;
+ this.suppressExceptions = suppressExceptions;
+ }
+
+ public String getStatisticName() {
+ return statisticName;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public boolean isDeleteTaskAttemptDirsInParallel() {
+ return deleteTaskAttemptDirsInParallel;
+ }
+
+ public boolean isSuppressExceptions() {
+ return suppressExceptions;
+ }
+
+ @Override
+ public String toString() {
+ return "Arguments{" +
+ "statisticName='" + statisticName + '\''
+ + ", enabled=" + enabled
+ + ", deleteTaskAttemptDirsInParallel="
+ + deleteTaskAttemptDirsInParallel
+ + ", suppressExceptions=" + suppressExceptions
+ + '}';
+ }
+ }
+
+ /**
+ * Static disabled arguments.
+ */
+ public static final Arguments DISABLED = new Arguments(OP_STAGE_JOB_CLEANUP,
+ false,
+ false,
+ false
+ );
+
+ /**
+ * Build an options argument from a configuration, using the
+ * settings from FileOutputCommitter and manifest committer.
+ * @param statisticName statistic name to use in duration tracking.
+ * @param conf configuration to use.
+ * @return the options to process
+ */
+ public static Arguments cleanupStageOptionsFromConfig(
+ String statisticName, Configuration conf) {
+
+ boolean enabled = !conf.getBoolean(FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED,
+ FILEOUTPUTCOMMITTER_CLEANUP_SKIPPED_DEFAULT);
+ boolean suppressExceptions = conf.getBoolean(
+ FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED,
+ FILEOUTPUTCOMMITTER_CLEANUP_FAILURES_IGNORED_DEFAULT);
+ boolean deleteTaskAttemptDirsInParallel = conf.getBoolean(
+ OPT_CLEANUP_PARALLEL_DELETE,
+ OPT_CLEANUP_PARALLEL_DELETE_DIRS_DEFAULT);
+ return new Arguments(
+ statisticName,
+ enabled,
+ deleteTaskAttemptDirsInParallel,
+ suppressExceptions
+ );
+ }
+
+ /**
+ * Enum of outcomes.
+ */
+ public enum Outcome {
+ DISABLED("Disabled", false),
+ NOTHING_TO_CLEAN_UP("Nothing to clean up", true),
+ PARALLEL_DELETE("Parallel Delete of Task Attempt Directories", true),
+ DELETED("Delete of job directory", true),
+ FAILURE("Delete failed", false);
+
+ private final String description;
+
+ private final boolean success;
+
+ Outcome(String description, boolean success) {
+ this.description = description;
+ this.success = success;
+ }
+
+ @Override
+ public String toString() {
+ return "Outcome{" + name() +
+ " '" + description + '\'' +
+ "}";
+ }
+
+ /**
+ * description.
+ * @return text for logging
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * Was this a success?
+ * @return true if this outcome is good.
+ */
+ public boolean isSuccess() {
+ return success;
+ }
+ }
+
+ /**
+ * Result of the cleanup.
+ * If the outcome == FAILURE but exceptions were suppressed
+ * (which they are implicitly if an instance of this object
+ * is created and returned), then the exception
+ * MUST NOT be null.
+ */
+ public static final class Result {
+
+ /** Outcome. */
+ private final Outcome outcome;
+
+ /** Directory cleaned up. */
+ private final Path directory;
+
+ /**
+ * Number of delete calls made across all threads.
+ */
+ private final int deleteCalls;
+
+ /**
+ * Any IOE raised.
+ */
+ private final IOException exception;
+
+ public Result(
+ final Outcome outcome,
+ final Path directory,
+ final int deleteCalls,
+ IOException exception) {
+ this.outcome = requireNonNull(outcome, "outcome");
+ this.directory = directory;
+ this.deleteCalls = deleteCalls;
+ this.exception = exception;
+ if (outcome == Outcome.FAILURE) {
+ requireNonNull(exception, "No exception in failure result");
+ }
+ }
+
+ public Path getDirectory() {
+ return directory;
+ }
+
+ public boolean wasExecuted() {
+ return outcome != Outcome.DISABLED;
+ }
+
+ /**
+ * Was the outcome a success?
+ * That is: either the dir wasn't there or through
+ * delete/rename it is no longer there.
+ * @return true if the temporary dir no longer exists.
+ */
+ public boolean succeeded() {
+ return outcome.isSuccess();
+ }
+
+ public Outcome getOutcome() {
+ return outcome;
+ }
+
+ public int getDeleteCalls() {
+ return deleteCalls;
+ }
+
+ public IOException getException() {
+ return exception;
+ }
+
+ /**
+ * If there was an IOE caught, throw it.
+ * For ease of use in (meaningful) lambda expressions
+ * in tests, returns the string value if there
+ * was no exception to throw (for use in tests)
+ * @throws IOException exception.
+ */
+ public String maybeRethrowException() throws IOException {
+ if (exception != null) {
+ throw exception;
+ }
+ return toString();
+ }
+
+ @Override
+ public String toString() {
+ return "CleanupResult{" +
+ "outcome=" + outcome +
+ ", directory=" + directory +
+ ", deleteCalls=" + deleteCalls +
+ ", exception=" + exception +
+ '}';
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java
new file mode 100644
index 0000000..a754f2a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitJobStage.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_BYTES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_FILES_COMMITTED_COUNT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.MANIFESTS;
+
+/**
+ * Commit the Job.
+ * Arguments (save manifest, validate output)
+ * Inputs: saveMarker: boolean, validateOutput: boolean
+ * Outputs: SuccessData
+ */
+public class CommitJobStage extends
+ AbstractJobOrTaskStage<
+ CommitJobStage.Arguments,
+ CommitJobStage.Result> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CommitJobStage.class);
+
+ public CommitJobStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_COMMIT, true);
+ }
+
+ @Override
+ protected CommitJobStage.Result executeStage(
+ final CommitJobStage.Arguments arguments) throws IOException {
+
+ LOG.info("{}: Committing job \"{}\". resilient commit supported = {}",
+ getName(),
+ getJobId(),
+ storeSupportsResilientCommit());
+
+ boolean createMarker = arguments.isCreateMarker();
+
+ // load the manifests
+ final StageConfig stageConfig = getStageConfig();
+ LoadManifestsStage.Result result
+ = new LoadManifestsStage(stageConfig).apply(true);
+ List<TaskManifest> manifests = result.getManifests();
+ LoadManifestsStage.SummaryInfo summary = result.getSummary();
+
+ LOG.debug("{}: Job Summary {}", getName(), summary);
+ LOG.info("{}: Committing job with file count: {}; total size {} bytes",
+ getName(),
+ summary.getFileCount(),
+ byteCountToDisplaySize(summary.getTotalFileSize()));
+
+
+ // add in the manifest statistics to our local IOStatistics for
+ // reporting.
+ IOStatisticsStore iostats = getIOStatistics();
+ iostats.aggregate(summary.getIOStatistics());
+
+ // prepare destination directories.
+ final CreateOutputDirectoriesStage.Result dirStageResults =
+ new CreateOutputDirectoriesStage(stageConfig)
+ .apply(manifests);
+
+ // commit all the tasks.
+ // The success data includes a snapshot of the IO Statistics
+ // and hence all aggregate stats from the tasks.
+ ManifestSuccessData successData;
+ successData = new RenameFilesStage(stageConfig).apply(
+ Pair.of(manifests, dirStageResults.getCreatedDirectories()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: _SUCCESS file summary {}", getName(), successData.toJson());
+ }
+ // update the counter of bytes committed and files.
+ // use setCounter so as to ignore any values accumulated when
+ // aggregating tasks.
+ iostats.setCounter(
+ COMMITTER_FILES_COMMITTED_COUNT,
+ summary.getFileCount());
+ iostats.setCounter(
+ COMMITTER_BYTES_COMMITTED_COUNT,
+ summary.getTotalFileSize());
+ successData.snapshotIOStatistics(iostats);
+
+
+ // rename manifests. Only warn on failure here.
+ final String manifestRenameDir = arguments.getManifestRenameDir();
+ if (isNotBlank(manifestRenameDir)) {
+ Path manifestRenamePath = new Path(
+ new Path(manifestRenameDir),
+ getJobId());
+ LOG.info("{}: Renaming manifests to {}", getName(), manifestRenamePath);
+ try {
+ renameDir(getTaskManifestDir(), manifestRenamePath);
+
+ // save this path in the summary diagnostics
+ successData.getDiagnostics().put(MANIFESTS, manifestRenamePath.toUri().toString());
+ } catch (IOException | IllegalArgumentException e) {
+ // rename failure, including path for wrong filesystem
+ LOG.warn("{}: Failed to rename manifests to {}", getName(), manifestRenamePath, e);
+ }
+ }
+
+ // save the _SUCCESS if the option is enabled.
+ Path successPath = null;
+ if (createMarker) {
+ // save a snapshot of the IO Statistics
+
+ successPath = new SaveSuccessFileStage(stageConfig)
+ .apply(successData);
+ LOG.debug("{}: Saving _SUCCESS file to {}", getName(), successPath);
+ }
+
+ // optional cleanup
+ new CleanupJobStage(stageConfig).apply(arguments.getCleanupArguments());
+
+ // and then, after everything else: optionally validate.
+ if (arguments.isValidateOutput()) {
+ // cache and restore the active stage field
+ LOG.info("{}: Validating output.", getName());
+ new ValidateRenamedFilesStage(stageConfig)
+ .apply(result.getManifests());
+ }
+
+ // restore the active stage so that when the report is saved
+ // it is declared as job commit, not cleanup or validate.
+ stageConfig.enterStage(getStageName(arguments));
+
+ // the result
+ return new CommitJobStage.Result(successPath, successData);
+ }
+
+ /**
+ * Arguments for job commit.
+ */
+ public static final class Arguments {
+
+ /** create the _SUCCESS marker? */
+ private final boolean createMarker;
+
+ /** perform validation checks on the files? */
+ private final boolean validateOutput;
+
+ /** optional directory to rename the task manifests to. */
+ private final String manifestRenameDir;
+
+ /** cleanup arguments.. */
+ private final CleanupJobStage.Arguments cleanupArguments;
+
+ /**
+ *
+ * @param createMarker create the _SUCCESS marker?
+ * @param validateOutput perform validation checks on the files?
+ * @param manifestRenameDir optional directory to rename the task manifests to
+ * @param cleanupArguments cleanup arguments.
+ */
+ public Arguments(
+ boolean createMarker,
+ boolean validateOutput,
+ @Nullable String manifestRenameDir,
+ CleanupJobStage.Arguments cleanupArguments) {
+
+ this.createMarker = createMarker;
+ this.validateOutput = validateOutput;
+ this.manifestRenameDir = manifestRenameDir;
+ this.cleanupArguments = requireNonNull(cleanupArguments);
+ }
+
+ public boolean isCreateMarker() {
+ return createMarker;
+ }
+
+ public boolean isValidateOutput() {
+ return validateOutput;
+ }
+
+ public String getManifestRenameDir() {
+ return manifestRenameDir;
+ }
+
+ public CleanupJobStage.Arguments getCleanupArguments() {
+ return cleanupArguments;
+ }
+ }
+
+ /**
+ * Result of the stage.
+ */
+ public static final class Result {
+ /**
+ * Manifest success data.
+ */
+ private final ManifestSuccessData jobSuccessData;
+
+ /**
+ * Success file path. null if not saved.
+ */
+ private final Path successPath;
+
+ public Result(final Path successPath,
+ ManifestSuccessData jobSuccessData) {
+ this.successPath = successPath;
+ this.jobSuccessData = jobSuccessData;
+ }
+
+ public ManifestSuccessData getJobSuccessData() {
+ return jobSuccessData;
+ }
+
+ public Path getSuccessPath() {
+ return successPath;
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java
new file mode 100644
index 0000000..bf5ba27
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CommitTaskStage.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_COMMIT;
+
+/**
+ * Commit a task attempt.
+ * Scan the task attempt directories through
+ * {@link TaskAttemptScanDirectoryStage}
+ * and then save to the task manifest path at
+ * {@link SaveTaskManifestStage}.
+ */
+
+public class CommitTaskStage extends
+ AbstractJobOrTaskStage<Void, CommitTaskStage.Result> {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CommitTaskStage.class);
+
+ public CommitTaskStage(final StageConfig stageConfig) {
+ super(true, stageConfig, OP_STAGE_TASK_COMMIT, false);
+ }
+
+ /**
+ * Scan the task attempt dir then save the manifest.
+ * A snapshot of the IOStats will be included in the manifest;
+ * this includes the scan time.
+ * @param arguments arguments to the function.
+ * @return the path the manifest was saved to, and the manifest.
+ * @throws IOException IO failure.
+ */
+ @Override
+ protected CommitTaskStage.Result executeStage(final Void arguments)
+ throws IOException {
+ LOG.info("{}: Committing task \"{}\"", getName(), getTaskAttemptId());
+
+ // execute the scan
+ final TaskAttemptScanDirectoryStage scanStage =
+ new TaskAttemptScanDirectoryStage(getStageConfig());
+ TaskManifest manifest = scanStage.apply(arguments);
+
+ // add the scan as task commit. It's not quite, as it doesn't include
+ // the saving, but ...
+ scanStage.addExecutionDurationToStatistics(getIOStatistics(), OP_STAGE_TASK_COMMIT);
+
+ // save a snapshot of the IO Statistics
+ final IOStatisticsSnapshot manifestStats = snapshotIOStatistics();
+ manifestStats.aggregate(getIOStatistics());
+ manifest.setIOStatistics(manifestStats);
+
+ // Now save with rename
+ Path manifestPath = new SaveTaskManifestStage(getStageConfig())
+ .apply(manifest);
+ return new CommitTaskStage.Result(manifestPath, manifest);
+ }
+
+ /**
+ * Result of the stage.
+ */
+ public static final class Result {
+ /** The path the manifest was saved to. */
+ private final Path path;
+ /** The manifest. */
+ private final TaskManifest taskManifest;
+
+ public Result(Path path,
+ TaskManifest taskManifest) {
+ this.path = path;
+ this.taskManifest = taskManifest;
+ }
+
+ /**
+ * Get the manifest path.
+ * @return The path the manifest was saved to.
+ */
+ public Path getPath() {
+ return path;
+ }
+
+ /**
+ * Get the manifest.
+ * @return The manifest.
+ */
+ public TaskManifest getTaskManifest() {
+ return taskManifest;
+ }
+
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java
new file mode 100644
index 0000000..927004e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/CreateOutputDirectoriesStage.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.EntryStatus;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_DELETE;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.measureDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_CREATE_DIRECTORIES;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_DELETE_FILE_UNDER_DESTINATION;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_MKDIRS_RETURNED_FALSE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_PREPARE_DIR_ANCESTORS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_CREATE_TARGET_DIRS;
+import static org.apache.hadoop.util.OperationDuration.humanTime;
+
+/**
+ * Prepare the destination directory tree, as efficiently as possible.
+ * possible -and doing those IO operations in the thread pool.
+ *
+ * The classic FileOutputCommitter does a recursive treewalk and
+ * deletes any files found at paths where directories are to be created.
+ *
+ * Each task manifest's directories are combined with those of the other tasks
+ * to build a set of all directories which are needed, without duplicates.
+ *
+ * This stage requires the aggregate set of manifests to contain
+ * all directories to create, including level,
+ * and expects them to have been probed for existence/state.
+ *
+ * For each level, all dirs are processed in parallel to
+ * be created or, if files, deleted.
+ *
+ * The stage returns the list of directories created, and for testing,
+ * the map of paths to outcomes.
+ *
+ * Directory creation can be surprisingly slow against object stores,
+ * do use benchmarks from real test runs when tuning this algorithm.
+ */
+public class CreateOutputDirectoriesStage extends
+ AbstractJobOrTaskStage<
+ List<TaskManifest>,
+ CreateOutputDirectoriesStage.Result> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CreateOutputDirectoriesStage.class);
+
+ /**
+ * Directories as a map of (path, path).
+ * Using a map rather than any set for efficient concurrency; the
+ * concurrent sets don't do lookups so fast.
+ */
+ private final Map<Path, DirMapState> dirMap = new ConcurrentHashMap<>();
+
+ /**
+ * A list of created paths for the results.
+ */
+ private final List<Path> createdDirectories = new ArrayList<>();
+
+ public CreateOutputDirectoriesStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_CREATE_TARGET_DIRS, true);
+ // add the dest dir to the dir map as we expect the job setup to create it.
+ dirMap.put(getDestinationDir(), DirMapState.dirWasCreated);
+ }
+
+ @Override
+ protected Result executeStage(
+ final List<TaskManifest> taskManifests)
+ throws IOException {
+
+ final List<Path> directories = createAllDirectories(taskManifests);
+ LOG.debug("{}: Created {} directories", getName(), directories.size());
+ return new Result(new HashSet<>(directories), dirMap);
+ }
+
+ /**
+ * For each task, build the list of directories it wants.
+ * @param taskManifests task manifests
+ * @return the list of paths which have been created.
+ */
+ private List<Path> createAllDirectories(final List<TaskManifest> taskManifests)
+ throws IOException {
+
+ // all directories which need to exist across all
+ // tasks.
+ // leaf directories
+ final Map<Path, DirEntry> leaves = new HashMap<>();
+ // parent directories. these do not need to be
+ // explicitly created.
+ final Map<Path, DirEntry> parents = new HashMap<>();
+ // the files which must be deleted as a directory
+ // will be created at that path.
+ final Set<Path> filesToDelete = new HashSet<>();
+
+ // iterate through the task manifests
+ // and all output dirs into the set of dirs to
+ // create.
+ // hopefully there is a lot of overlap, so the
+ // final number of dirs to create is small.
+ for (TaskManifest task: taskManifests) {
+ final List<DirEntry> destDirectories = task.getDestDirectories();
+ Collections.sort(destDirectories, (o1, o2) ->
+ o1.getLevel() - o2.getLevel());
+ for (DirEntry entry: destDirectories) {
+ // add the dest entry
+ final Path path = entry.getDestPath();
+ if (!leaves.containsKey(path)) {
+ leaves.put(path, entry);
+
+ // if it is a file to delete, record this.
+ if (entry.getStatus() == EntryStatus.file) {
+ filesToDelete.add(path);
+ }
+ final Path parent = path.getParent();
+ if (parent != null && leaves.containsKey(parent)) {
+ // there's a parent dir, move it from the leaf list
+ // to parent list
+ parents.put(parent,
+ leaves.remove(parent));
+ }
+ }
+ }
+ }
+
+ // at this point then there is a map of all directories which
+ // are leaf entries and so need to be created if not present,
+ // and the maximum level is known.
+ // we can iterate through all levels deleting any files if there are any.
+
+ // Prepare parent directories.
+ deleteFiles(filesToDelete);
+
+ // Now the real work.
+ final int createCount = leaves.size();
+ LOG.info("Preparing {} directory/directories", createCount);
+ // now probe for and create the leaf dirs, which are those at the
+ // bottom level
+ Duration d = measureDurationOfInvocation(getIOStatistics(), OP_CREATE_DIRECTORIES, () ->
+ TaskPool.foreach(leaves.values())
+ .executeWith(getIOProcessors(createCount))
+ .onFailure(this::reportMkDirFailure)
+ .stopOnFailure()
+ .run(this::createOneDirectory));
+ LOG.info("Time to prepare directories {}", humanTime(d.toMillis()));
+ return createdDirectories;
+ }
+
+ /**
+ * How many failures have been reported.
+ */
+ private final AtomicInteger failureCount = new AtomicInteger();
+
+ /**
+ * report a single directory failure.
+ * @param path path which could not be deleted
+ * @param e exception raised.
+ */
+ private void reportMkDirFailure(DirEntry dirEntry, Exception e) {
+ Path path = dirEntry.getDestPath();
+ final int count = failureCount.incrementAndGet();
+ LOG.warn("{}: mkdir failure #{} Failed to create directory \"{}\": {}",
+ getName(), count, path, e.toString());
+ LOG.debug("{}: Full exception details",
+ getName(), e);
+ }
+
+ /**
+ * Delete all directories where there is a file.
+ * @param filesToDelete set of dirs to where there is a file.
+ * @throws IOException IO problem
+ */
+ private void deleteFiles(final Set<Path> filesToDelete)
+ throws IOException {
+
+ final int size = filesToDelete.size();
+ if (size == 0) {
+ // nothing to delete.
+ return;
+ }
+ LOG.info("{}: Directory entries containing files to delete: {}", getName(), size);
+ Duration d = measureDurationOfInvocation(getIOStatistics(),
+ OP_PREPARE_DIR_ANCESTORS, () ->
+ TaskPool.foreach(filesToDelete)
+ .executeWith(getIOProcessors(size))
+ .stopOnFailure()
+ .run(dir -> {
+ updateAuditContext(OP_PREPARE_DIR_ANCESTORS);
+ deleteDirWithFile(dir);
+ }));
+ LOG.info("Time to delete files {}", humanTime(d.toMillis()));
+ }
+
+ /**
+ * Prepare a parent directory.
+ * @param dir directory to probe
+ * @throws IOException failure in probe other than FNFE
+ */
+ private void deleteDirWithFile(Path dir) throws IOException {
+ // report progress back
+ progress();
+ LOG.info("{}: Deleting file {}", getName(), dir);
+ delete(dir, false, OP_DELETE);
+ // note its final state
+ addToDirectoryMap(dir, DirMapState.fileNowDeleted);
+ }
+
+
+ /**
+ * Create a directory is required, updating the directory map
+ * and, if the operation took place, the list of created dirs.
+ * Reports progress on invocation.
+ * @param dirEntry entry
+ * @throws IOException failure.
+ */
+ private void createOneDirectory(final DirEntry dirEntry) throws IOException {
+ // report progress back
+ progress();
+ final Path dir = dirEntry.getDestPath();
+ updateAuditContext(OP_STAGE_JOB_CREATE_TARGET_DIRS);
+ final DirMapState state = maybeCreateOneDirectory(dirEntry);
+ switch (state) {
+ case dirFoundInStore:
+ addToDirectoryMap(dir, state);
+ break;
+ case dirWasCreated:
+ case dirCreatedOnSecondAttempt:
+ addCreatedDirectory(dir);
+ addToDirectoryMap(dir, state);
+ break;
+ default:
+ break;
+ }
+
+ }
+
+
+ /**
+ * Try to efficiently and robustly create a directory in a method which is
+ * expected to be executed in parallel with operations creating
+ * peer directories.
+ * @param path path to create
+ * @return true if dir created/found
+ * @throws IOException IO Failure.
+ */
+ private DirMapState maybeCreateOneDirectory(DirEntry dirEntry) throws IOException {
+ final EntryStatus status = dirEntry.getStatus();
+ if (status == EntryStatus.dir) {
+ return DirMapState.dirFoundInStore;
+ }
+ // present in case directories are ever created in task commits
+ if (status == EntryStatus.created_dir) {
+ return DirMapState.dirWasCreated;
+ }
+
+ // here the dir doesn't exist because
+ // it was a file and has been deleted, or
+ // checks failed. create it.
+ final Path path = dirEntry.getDestPath();
+
+ LOG.info("Creating directory {}", path);
+
+ try {
+ if (mkdirs(path, false)) {
+ // success -return immediately.
+ return DirMapState.dirWasCreated;
+ }
+ getIOStatistics().incrementCounter(OP_MKDIRS_RETURNED_FALSE);
+
+ LOG.info("{}: mkdirs({}) returned false, attempting to recover",
+ getName(), path);
+ } catch (IOException e) {
+ // can be caused by file existing, etc.
+ LOG.info("{}: mkdir({}) raised exception {}", getName(), path, e.toString());
+ LOG.debug("{}: Mkdir stack", getName(), e);
+ }
+
+ // fallback to checking the FS, in case a different process did it.
+ FileStatus st = getFileStatusOrNull(path);
+ if (st != null) {
+ if (!st.isDirectory()) {
+ // is bad: delete a file
+ LOG.info("{}: Deleting file where a directory should go: {}",
+ getName(), st);
+ delete(path, false, OP_DELETE_FILE_UNDER_DESTINATION);
+ } else {
+ // is good.
+ LOG.warn("{}: Even though mkdirs({}) failed, there is now a directory there",
+ getName(), path);
+ return DirMapState.dirFoundInStore;
+ }
+ } else {
+ // nothing found. This should never happen.
+ LOG.warn("{}: Although mkdirs({}) returned false, there's nothing at that path to prevent it",
+ getName(), path);
+
+ }
+
+ // try to create the directory again
+ // if this fails, and IOE is still raised, that
+ // propagate to the caller.
+ if (!mkdirs(path, false)) {
+
+ // mkdirs failed again
+ getIOStatistics().incrementCounter(OP_MKDIRS_RETURNED_FALSE);
+
+ // require the dir to exist, raising an exception if it does not.
+ directoryMustExist("Creating directory ", path);
+ }
+
+ // we only get here if the second attempt recovered
+ return DirMapState.dirCreatedOnSecondAttempt;
+
+ }
+
+ /**
+ * Add a created dir to the list of created dirs.
+ * @param dir new dir.
+ */
+ private void addCreatedDirectory(final Path dir) {
+ synchronized (createdDirectories) {
+ createdDirectories.add(dir);
+ }
+ }
+
+ /**
+ * Add a dir to the directory map if there is not already an entry there.
+ * @param dir directory.
+ * @param state state of entry
+ */
+ private void addToDirectoryMap(final Path dir,
+ DirMapState state) {
+ if (!dirMap.containsKey(dir)) {
+ dirMap.put(dir, state);
+ }
+ }
+
+
+ /**
+ * Result of the operation.
+ */
+ public static final class Result {
+
+ /** directories created. */
+ private final Set<Path> createdDirectories;
+
+ /**
+ * Map of dirs built up during preparation.
+ */
+ private final Map<Path, DirMapState> dirMap;
+
+ public Result(Set<Path> createdDirectories,
+ Map<Path, DirMapState> dirMap) {
+ this.createdDirectories = requireNonNull(createdDirectories);
+ this.dirMap = requireNonNull(dirMap);
+ }
+
+ public Set<Path> getCreatedDirectories() {
+ return createdDirectories;
+ }
+
+ public Map<Path, DirMapState> getDirMap() {
+ return dirMap;
+ }
+
+ @Override
+ public String toString() {
+ return "Result{" +
+ "directory count=" + createdDirectories.size() +
+ '}';
+ }
+ }
+
+ /**
+ * Enumeration of dir states in the dir map.
+ */
+ public enum DirMapState {
+ dirFoundInStore,
+ dirFoundInMap,
+ dirWasCreated,
+ dirCreatedOnSecondAttempt,
+ fileNowDeleted,
+ ancestorWasDirOrMissing,
+ parentWasNotFile,
+ parentOfCreatedDir
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java
new file mode 100644
index 0000000..98975bf
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/JobOrTaskStage.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.util.functional.FunctionRaisingIOE;
+
+/**
+ * Job Stage is a function raising an IOE. and which also
+ * provides IOStats.
+ * @param <IN> Type of arguments to the stage.
+ * @param <OUT> Type of result.
+ */
+public interface JobOrTaskStage<IN, OUT> extends FunctionRaisingIOE<IN, OUT>,
+ IOStatisticsSource {
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java
new file mode 100644
index 0000000..3720f54
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/LoadManifestsStage.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_MANIFEST_FILE_SIZE;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_LOAD_ALL_MANIFESTS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_LOAD_MANIFESTS;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.maybeAddIOStatistics;
+
+/**
+ * Stage to load all the task manifests in the job attempt directory.
+ * Invoked in Job Commit.
+ * Manifests are loaded in parallel.
+ * The IOStatistics snapshot passed in is built up with the statistics
+ * and the statistics stripped from the manifest if prune == true.
+ * This keeps the memory footprint of each manifest down.
+ */
+public class LoadManifestsStage extends
+ AbstractJobOrTaskStage<
+ Boolean,
+ LoadManifestsStage.Result> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ LoadManifestsStage.class);
+
+ /**
+ * Summary of manifest loading.
+ */
+ private final SummaryInfo summaryInfo = new SummaryInfo();
+
+ /**
+ * Should manifests be pruned of IOStatistics?
+ */
+ private boolean pruneManifests;
+
+ /**
+ * List of loaded manifests.
+ */
+ private final List<TaskManifest> manifests = new ArrayList<>();
+
+ public LoadManifestsStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_LOAD_MANIFESTS, true);
+ }
+
+ /**
+ * Load the manifests.
+ * @param prune should manifests be pruned of IOStatistics?
+ * @return the summary and a list of manifests.
+ * @throws IOException IO failure.
+ */
+ @Override
+ protected LoadManifestsStage.Result executeStage(
+ final Boolean prune) throws IOException {
+
+ final Path manifestDir = getTaskManifestDir();
+ LOG.info("{}: Executing Manifest Job Commit with manifests in {}",
+ getName(),
+ manifestDir);
+ pruneManifests = prune;
+ // build a list of all task manifests successfully committed
+ //
+ msync(manifestDir);
+ final RemoteIterator<FileStatus> manifestFiles = listManifests();
+
+ final List<TaskManifest> manifestList = loadAllManifests(manifestFiles);
+ LOG.info("{}: Summary of {} manifests loaded in {}: {}",
+ getName(),
+ manifestList.size(),
+ manifestDir,
+ summaryInfo);
+
+ // collect any stats
+ maybeAddIOStatistics(getIOStatistics(), manifestFiles);
+ return new LoadManifestsStage.Result(summaryInfo, manifestList);
+ }
+
+ /**
+ * Load all the manifests.
+ * @param manifestFiles list of manifest files.
+ * @return the loaded manifests.
+ * @throws IOException IO Failure.
+ */
+ private List<TaskManifest> loadAllManifests(
+ final RemoteIterator<FileStatus> manifestFiles) throws IOException {
+
+ trackDurationOfInvocation(getIOStatistics(), OP_LOAD_ALL_MANIFESTS, () ->
+ TaskPool.foreach(manifestFiles)
+ .executeWith(getIOProcessors())
+ .stopOnFailure()
+ .run(this::processOneManifest));
+ return manifests;
+ }
+
+ /**
+ * Method invoked to process one manifest.
+ * @param status file to process.
+ * @throws IOException failure to load/parse
+ */
+ private void processOneManifest(FileStatus status)
+ throws IOException {
+ updateAuditContext(OP_LOAD_ALL_MANIFESTS);
+
+ TaskManifest m = fetchTaskManifest(status);
+ progress();
+
+ // update the manifest list in a synchronized block.
+
+ synchronized (manifests) {
+ manifests.add(m);
+ // and the summary info in the same block, to
+ // eliminate the need to acquire a second lock.
+ summaryInfo.add(m);
+ }
+ if (pruneManifests) {
+ m.setIOStatistics(null);
+ m.getExtraData().clear();
+ }
+ }
+
+ /**
+ * Precommit preparation of a single manifest file.
+ * To reduce the memory foot print, the IOStatistics and
+ * extra data of each manifest is cleared.
+ * @param status status of file.
+ * @return number of files.
+ * @throws IOException IO Failure.
+ */
+ private TaskManifest fetchTaskManifest(FileStatus status)
+ throws IOException {
+ if (status.getLen() == 0 || !status.isFile()) {
+ throw new PathIOException(status.getPath().toString(),
+ "Not a valid manifest file; file status = " + status);
+ }
+ // load the manifest, which includes validation.
+ final TaskManifest manifest = loadManifest(status);
+ final String id = manifest.getTaskAttemptID();
+ final int filecount = manifest.getFilesToCommit().size();
+ final long size = manifest.getTotalFileSize();
+ LOG.info("{}: Task Attempt {} file {}: File count: {}; data size={}",
+ getName(), id, status.getPath(), filecount, size);
+ // record file size for tracking of memory consumption.
+ getIOStatistics().addMeanStatisticSample(COMMITTER_TASK_MANIFEST_FILE_SIZE,
+ status.getLen());
+ return manifest;
+ }
+
+ /**
+ * Result of the stage.
+ */
+ public static final class Result {
+ private final SummaryInfo summary;
+
+ private final List<TaskManifest> manifests;
+
+ public Result(SummaryInfo summary,
+ List<TaskManifest> manifests) {
+ this.summary = summary;
+ this.manifests = manifests;
+ }
+
+ public SummaryInfo getSummary() {
+ return summary;
+ }
+
+ public List<TaskManifest> getManifests() {
+ return manifests;
+ }
+
+ }
+
+ /**
+ * Summary information.
+ */
+ public static final class SummaryInfo implements IOStatisticsSource {
+
+ /**
+ * Aggregate IOStatistics.
+ */
+ private IOStatisticsSnapshot iostatistics = snapshotIOStatistics();
+
+ /**
+ * How many manifests were loaded.
+ */
+ private long manifestCount;
+
+ /**
+ * Total number of files to rename.
+ */
+ private long fileCount;
+
+ /**
+ * Total number of directories which may need
+ * to be created.
+ * As there is no dedup, this is likely to be
+ * a (major) overestimate.
+ */
+ private long directoryCount;
+
+ /**
+ * Total amount of data to be committed.
+ */
+ private long totalFileSize;
+
+ /**
+ * Get the IOStatistics.
+ * @return aggregate IOStatistics
+ */
+ @Override
+ public IOStatisticsSnapshot getIOStatistics() {
+ return iostatistics;
+ }
+
+ public long getFileCount() {
+ return fileCount;
+ }
+
+ public long getDirectoryCount() {
+ return directoryCount;
+ }
+
+ public long getTotalFileSize() {
+ return totalFileSize;
+ }
+
+ public long getManifestCount() {
+ return manifestCount;
+ }
+
+ /**
+ * Add all statistics.
+ * @param manifest manifest to add.
+ */
+ public void add(TaskManifest manifest) {
+ manifestCount++;
+ iostatistics.aggregate(manifest.getIOStatistics());
+ fileCount += manifest.getFilesToCommit().size();
+ directoryCount += manifest.getDestDirectories().size();
+ totalFileSize += manifest.getTotalFileSize();
+ }
+
+ /**
+ * To String includes all summary info except statistics.
+ * @return string value
+ */
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "SummaryInfo{");
+ sb.append("manifestCount=").append(manifestCount);
+ sb.append(", fileCount=").append(fileCount);
+ sb.append(", directoryCount=").append(directoryCount);
+ sb.append(", totalFileSize=").append(
+ byteCountToDisplaySize(totalFileSize));
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java
new file mode 100644
index 0000000..e8124c1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/RenameFilesStage.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER_FILE_LIMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_RENAME_FILES;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createManifestOutcome;
+import static org.apache.hadoop.thirdparty.com.google.common.collect.Iterables.concat;
+
+/**
+ * This stage renames all the files.
+ * Input: the manifests and the set of directories created, as returned by
+ * {@link CreateOutputDirectoriesStage}.
+ * If the job is configured to delete target files, if the parent dir
+ * had to be created, the delete() call can be skipped.
+ * It returns a manifest success data file summarizing the
+ * output, but does not add iostatistics to it.
+ */
+public class RenameFilesStage extends
+ AbstractJobOrTaskStage<
+ Pair<List<TaskManifest>, Set<Path>>,
+ ManifestSuccessData> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ RenameFilesStage.class);
+
+ /**
+ * List of all files committed.
+ */
+ private final List<FileEntry> filesCommitted = new ArrayList<>();
+
+ /**
+ * Total file size.
+ */
+ private long totalFileSize = 0;
+
+ private Set<Path> createdDirectories;
+
+ public RenameFilesStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_RENAME_FILES, true);
+ }
+
+ /**
+ * Get the list of files committed.
+ * Access is not synchronized.
+ * @return direct access to the list of files.
+ */
+ public synchronized List<FileEntry> getFilesCommitted() {
+ return filesCommitted;
+ }
+
+ /**
+ * Get the total file size of the committed task.
+ * @return a number greater than or equal to zero.
+ */
+ public synchronized long getTotalFileSize() {
+ return totalFileSize;
+ }
+
+ /**
+ * Rename files in job commit.
+ * @param taskManifests a list of task manifests containing files.
+ * @return the job report.
+ * @throws IOException failure
+ */
+ @Override
+ protected ManifestSuccessData executeStage(
+ Pair<List<TaskManifest>, Set<Path>> args)
+ throws IOException {
+
+ final List<TaskManifest> taskManifests = args.getLeft();
+ createdDirectories = args.getRight();
+
+ final ManifestSuccessData success = createManifestOutcome(getStageConfig(),
+ OP_STAGE_JOB_COMMIT);
+ final int manifestCount = taskManifests.size();
+
+ LOG.info("{}: Executing Manifest Job Commit with {} manifests in {}",
+ getName(), manifestCount, getTaskManifestDir());
+
+ // first step is to aggregate the output of all manifests into a single
+ // list of files to commit.
+ // Which Guava can do in a zero-copy concatenated iterator
+
+ final Iterable<FileEntry> filesToCommit = concat(taskManifests.stream()
+ .map(TaskManifest::getFilesToCommit)
+ .collect(Collectors.toList()));
+
+ TaskPool.foreach(filesToCommit)
+ .executeWith(getIOProcessors())
+ .stopOnFailure()
+ .run(this::commitOneFile);
+
+ // synchronized block to keep spotbugs happy.
+ List<FileEntry> committed = getFilesCommitted();
+ LOG.info("{}: Files committed: {}. Total size {}",
+ getName(), committed.size(), getTotalFileSize());
+
+ // Add a subset of the destination files to the success file;
+ // enough for simple testing
+ success.setFilenamePaths(
+ committed
+ .subList(0, Math.min(committed.size(), SUCCESS_MARKER_FILE_LIMIT))
+ .stream().map(FileEntry::getDestPath)
+ .collect(Collectors.toList()));
+
+ success.setSuccess(true);
+
+ return success;
+ }
+
+ /**
+ * Commit one file by rename, then, if that doesn't fail,
+ * add to the files committed list.
+ * @param entry entry to commit.
+ * @throws IOException faiure.
+ */
+ private void commitOneFile(FileEntry entry) throws IOException {
+ updateAuditContext(OP_STAGE_JOB_RENAME_FILES);
+
+ // report progress back
+ progress();
+
+ // if the dest dir is to be deleted,
+ // look to see if the parent dir was created.
+ // if it was. we know that the file doesn't exist.
+ final boolean deleteDest = getStageConfig().getDeleteTargetPaths()
+ && !createdDirectories.contains(entry.getDestPath().getParent());
+ // do the rename
+ commitFile(entry, deleteDest);
+
+ // update the list and IOStats
+ synchronized (this) {
+ filesCommitted.add(entry);
+ totalFileSize += entry.getSize();
+ }
+
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java
new file mode 100644
index 0000000..eb9c82f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveSuccessFileStage.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.TMP_SUFFIX;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_COMMIT;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_SAVE_SUCCESS;
+
+/**
+ * Save the _SUCCESS file to the destination directory
+ * via a temp file in the job attempt dir.
+ * Returns the path of the file
+ */
+public class SaveSuccessFileStage extends
+ AbstractJobOrTaskStage<ManifestSuccessData, Path> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SaveSuccessFileStage.class);
+
+ public SaveSuccessFileStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_SAVE_SUCCESS, false);
+ }
+
+ /**
+ * Stage name is always job commit.
+ * @param arguments args to the invocation.
+ * @return stage name
+ */
+ @Override
+ protected String getStageName(ManifestSuccessData arguments) {
+ // set it to the job commit stage, always.
+ return OP_STAGE_JOB_COMMIT;
+ }
+
+ /**
+ * Execute.
+ * @param successData success data to save
+ * @return path saved to.
+ * @throws IOException failure
+ */
+ @Override
+ protected Path executeStage(final ManifestSuccessData successData)
+ throws IOException {
+ // Save the marker
+ Path successFile = getStageConfig().getJobSuccessMarkerPath();
+ Path successTempFile = new Path(getJobAttemptDir(), SUCCESS_MARKER + TMP_SUFFIX);
+ LOG.debug("{}: Saving _SUCCESS file to {} via {}", successFile,
+ getName(),
+ successTempFile);
+ save(successData, successTempFile, successFile);
+ return successFile;
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java
new file mode 100644
index 0000000..fdaf018
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SaveTaskManifestStage.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SAVE_MANIFEST;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestPathForTask;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.manifestTempPathForTaskAttempt;
+
+/**
+ * Save a task manifest to the job attempt dir, using the task
+ * ID for the name of the final file.
+ * For atomic writes, the manifest is saved
+ * by writing to a temp file and then renaming it.
+ * Uses both the task ID and task attempt ID to determine the temp filename;
+ * Before the rename of (temp, final-path), any file at the final path
+ * is deleted.
+ * This is so that when this stage is invoked in a task commit, its output
+ * overwrites any of the first commit.
+ * When it succeeds, therefore, unless there is any subsequent commit of
+ * another task, the task manifest at the final path is from this
+ * operation.
+ *
+ * Returns the path where the manifest was saved.
+ */
+public class SaveTaskManifestStage extends
+ AbstractJobOrTaskStage<TaskManifest, Path> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SaveTaskManifestStage.class);
+
+ public SaveTaskManifestStage(final StageConfig stageConfig) {
+ super(true, stageConfig, OP_STAGE_TASK_SAVE_MANIFEST, false);
+ }
+
+ /**
+ * Save the manifest to a temp file and rename to the final
+ * manifest destination.
+ * @param manifest manifest
+ * @return the path to the final entry
+ * @throws IOException IO failure.
+ */
+ @Override
+ protected Path executeStage(final TaskManifest manifest)
+ throws IOException {
+
+ final Path manifestDir = getTaskManifestDir();
+ // final manifest file is by task ID
+ Path manifestFile = manifestPathForTask(manifestDir,
+ getRequiredTaskId());
+ Path manifestTempFile = manifestTempPathForTaskAttempt(manifestDir,
+ getRequiredTaskAttemptId());
+ LOG.info("{}: Saving manifest file to {}", getName(), manifestFile);
+ save(manifest, manifestTempFile, manifestFile);
+ return manifestFile;
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java
new file mode 100644
index 0000000..9b87325
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupJobStage.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_SETUP;
+
+/**
+ * Stage to set up a job by creating the job attempt directory.
+ * The job attempt directory must not exist before the call.
+ */
+public class SetupJobStage extends
+ AbstractJobOrTaskStage<Boolean, Path> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SetupJobStage.class);
+
+ public SetupJobStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_SETUP, false);
+ }
+
+ /**
+ * Execute the job setup stage.
+ * @param deleteMarker: should any success marker be deleted.
+ * @return the job attempted directory.
+ * @throws IOException failure.
+ */
+ @Override
+ protected Path executeStage(final Boolean deleteMarker) throws IOException {
+ final Path path = getJobAttemptDir();
+ LOG.info("{}: Creating Job Attempt directory {}", getName(), path);
+ createNewDirectory("Job setup", path);
+ createNewDirectory("Creating task manifest dir", getTaskManifestDir());
+ // delete any success marker if so instructed.
+ if (deleteMarker) {
+ delete(getStageConfig().getJobSuccessMarkerPath(), false);
+ }
+ return path;
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java
new file mode 100644
index 0000000..e909ee1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/SetupTaskStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SETUP;
+
+/**
+ * Stage to set up task.
+ * This creates the task attempt directory, after verifying
+ * that the job attempt dir exists (i.e. this is invoked
+ * after the job is started and before any cleanup.
+ * Argument passed in is task name:only for logging.
+ */
+public class SetupTaskStage extends
+ AbstractJobOrTaskStage<String, Path> {
+
+ public SetupTaskStage(final StageConfig stageConfig) {
+ super(true, stageConfig, OP_STAGE_TASK_SETUP, false);
+ }
+
+ /**
+ * Set up a task.
+ * @param name task name (for logging)
+ * @return task attempt directory
+ * @throws IOException IO failure.
+ */
+ @Override
+ protected Path executeStage(final String name) throws IOException {
+ return createNewDirectory("Task setup " + name,
+ requireNonNull(getTaskAttemptDir(), "No task attempt directory"));
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java
new file mode 100644
index 0000000..d5bbba5
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageConfig.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+import org.apache.hadoop.util.JsonSerialization;
+import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.SUCCESS_MARKER;
+
+/**
+ * Stage Config.
+ * Everything to configure a stage which is common to all.
+ *
+ * It's isolated from the details of MR datatypes (taskID, taskattempt etc);
+ * at this point it expects parsed values.
+ *
+ * It uses the builder API, but once {@link #build()} is called it goes
+ * read only. This is to ensure that changes cannot
+ * take place when shared across stages.
+ */
+public class StageConfig {
+
+ /**
+ * A flag which freezes the config for
+ * further updates.
+ */
+ private boolean frozen;
+
+ /**
+ * IOStatistics to update.
+ */
+ private IOStatisticsStore iostatistics;
+
+ /**
+ * Job ID; constant over multiple attempts.
+ */
+ private String jobId;
+
+ /**
+ * Where did the job Unique ID come from?
+ */
+ private String jobIdSource = "";
+
+ /**
+ * Number of the job attempt; starts at zero.
+ */
+ private int jobAttemptNumber;
+
+ /**
+ * ID of the task.
+ */
+ private String taskId;
+
+ /**
+ * ID of this specific attempt at a task.
+ */
+ private String taskAttemptId;
+
+ /**
+ * Destination of job.
+ */
+ private Path destinationDir;
+
+ /**
+ * Job attempt dir.
+ */
+ private Path jobAttemptDir;
+
+ /**
+ * temp directory under job dest dir.
+ */
+ private Path outputTempSubDir;
+
+ /**
+ * Task attempt dir.
+ */
+ private Path taskAttemptDir;
+
+ /**
+ * directory where task manifests must go.
+ */
+ private Path taskManifestDir;
+
+ /**
+ * Subdir under the job attempt dir where task
+ * attempts will have subdirectories.
+ */
+ private Path jobAttemptTaskSubDir;
+
+ /**
+ * Callbacks to update store.
+ * This is not made visible to the stages; they must
+ * go through the superclass which
+ * adds statistics and logging.
+ */
+ private ManifestStoreOperations operations;
+
+ /**
+ * Submitter for doing IO against the store other than
+ * manifest processing.
+ */
+ private TaskPool.Submitter ioProcessors;
+
+ /**
+ * Optional progress callback.
+ */
+ private Progressable progressable;
+
+ /**
+ * Callback when a stage is entered.
+ */
+ private StageEventCallbacks enterStageEventHandler;
+
+ /**
+ * Thread local serializer; created on demand
+ * and shareable across a sequence of stages.
+ */
+ private final ThreadLocal<JsonSerialization<TaskManifest>> threadLocalSerializer =
+ ThreadLocal.withInitial(TaskManifest::serializer);
+
+ /**
+ * Delete target paths on commit? Stricter, but
+ * higher IO cost.
+ */
+ private boolean deleteTargetPaths;
+
+ /**
+ * Name for logging.
+ */
+ private String name = "";
+
+ public StageConfig() {
+ }
+
+ /**
+ * Verify that the config is not yet frozen.
+ */
+ private void checkOpen() {
+ Preconditions.checkState(!frozen,
+ "StageConfig is now read-only");
+ }
+
+ /**
+ * The build command makes the config immutable.
+ * Idempotent.
+ * @return the now-frozen config
+ */
+ public StageConfig build() {
+ frozen = true;
+ return this;
+ }
+
+ /**
+ * Set job destination dir.
+ * @param dir new dir
+ * @return this
+ */
+ public StageConfig withDestinationDir(final Path dir) {
+ destinationDir = dir;
+ return this;
+ }
+
+ /**
+ * Set IOStatistics store.
+ * @param store new store
+ * @return this
+ */
+ public StageConfig withIOStatistics(final IOStatisticsStore store) {
+ checkOpen();
+ iostatistics = store;
+ return this;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withIOProcessors(final TaskPool.Submitter value) {
+ checkOpen();
+ ioProcessors = value;
+ return this;
+ }
+
+ /**
+ * Set Job attempt directory.
+ * @param dir new dir
+ * @return this
+ */
+ public StageConfig withJobAttemptDir(final Path dir) {
+ checkOpen();
+ jobAttemptDir = dir;
+ return this;
+ }
+
+ /**
+ * Directory to put task manifests into.
+ * @return a path under the job attempt dir.
+ */
+ public Path getTaskManifestDir() {
+ return taskManifestDir;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return the builder
+ */
+ public StageConfig withTaskManifestDir(Path value) {
+ checkOpen();
+ taskManifestDir = value;
+ return this;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return the builder
+ */
+ public StageConfig withJobAttemptTaskSubDir(Path value) {
+ jobAttemptTaskSubDir = value;
+ return this;
+ }
+
+ /**
+ * Get the path to the subdirectory under $jobID where task
+ * attempts are. List this dir to find all task attempt dirs.
+ * @return a path under the job attempt dir.
+ */
+ public Path getJobAttemptTaskSubDir() {
+ return jobAttemptTaskSubDir;
+ }
+
+ /**
+ * Set the job directories from the attempt directories
+ * information. Does not set task attempt fields.
+ * @param dirs source of directories.
+ * @return this
+ */
+ public StageConfig withJobDirectories(
+ final ManifestCommitterSupport.AttemptDirectories dirs) {
+
+ checkOpen();
+ withJobAttemptDir(dirs.getJobAttemptDir())
+ .withJobAttemptTaskSubDir(dirs.getJobAttemptTaskSubDir())
+ .withDestinationDir(dirs.getOutputPath())
+ .withOutputTempSubDir(dirs.getOutputTempSubDir())
+ .withTaskManifestDir(dirs.getTaskManifestDir());
+
+ return this;
+ }
+
+ /**
+ * Set job ID with no attempt included.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withJobId(final String value) {
+ checkOpen();
+ jobId = value;
+ return this;
+ }
+
+ public Path getOutputTempSubDir() {
+ return outputTempSubDir;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withOutputTempSubDir(final Path value) {
+ checkOpen();
+ outputTempSubDir = value;
+ return this;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withOperations(final ManifestStoreOperations value) {
+ checkOpen();
+ operations = value;
+ return this;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withTaskAttemptId(final String value) {
+ checkOpen();
+ taskAttemptId = value;
+ return this;
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withTaskId(final String value) {
+ checkOpen();
+ taskId = value;
+ return this;
+ }
+
+ /**
+ * Set handler for stage entry events..
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withStageEventCallbacks(StageEventCallbacks value) {
+ checkOpen();
+ enterStageEventHandler = value;
+ return this;
+ }
+
+ /**
+ * Optional progress callback.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withProgressable(final Progressable value) {
+ checkOpen();
+ progressable = value;
+ return this;
+ }
+
+ /**
+ * Set the Task attempt directory.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withTaskAttemptDir(final Path value) {
+ checkOpen();
+ taskAttemptDir = value;
+ return this;
+ }
+
+ /**
+ * Set the job attempt number.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withJobAttemptNumber(final int value) {
+ checkOpen();
+ jobAttemptNumber = value;
+ return this;
+ }
+
+ /**
+ * Set the Job ID source.
+ * @param value new value
+ * @return this
+ */
+ public StageConfig withJobIdSource(final String value) {
+ checkOpen();
+ jobIdSource = value;
+ return this;
+ }
+
+ /**
+ * Set name of task/job.
+ * @param value new value
+ * @return the builder
+ */
+ public StageConfig withName(String value) {
+ name = value;
+ return this;
+ }
+
+ /**
+ * Get name of task/job.
+ * @return name for logging.
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Handler for stage entry events.
+ * @return the handler.
+ */
+ public StageEventCallbacks getEnterStageEventHandler() {
+ return enterStageEventHandler;
+ }
+
+ /**
+ * IOStatistics to update.
+ */
+ public IOStatisticsStore getIOStatistics() {
+ return iostatistics;
+ }
+
+ /**
+ * Job ID.
+ */
+ public String getJobId() {
+ return jobId;
+ }
+
+ /**
+ * ID of the task.
+ */
+ public String getTaskId() {
+ return taskId;
+ }
+
+ /**
+ * ID of this specific attempt at a task.
+ */
+ public String getTaskAttemptId() {
+ return taskAttemptId;
+ }
+
+ /**
+ * Job attempt dir.
+ */
+ public Path getJobAttemptDir() {
+ return jobAttemptDir;
+ }
+
+ /**
+ * Destination of job.
+ */
+ public Path getDestinationDir() {
+ return destinationDir;
+ }
+
+ /**
+ * Get the location of the success marker.
+ * @return a path under the destination directory.
+ */
+ public Path getJobSuccessMarkerPath() {
+ return new Path(destinationDir, SUCCESS_MARKER);
+ }
+
+ /**
+ * Callbacks to update store.
+ * This is not made visible to the stages; they must
+ * go through the wrapper classes in this class, which
+ * add statistics and logging.
+ */
+ public ManifestStoreOperations getOperations() {
+ return operations;
+ }
+
+ /**
+ * Submitter for doing IO against the store other than
+ * manifest processing.
+ */
+ public TaskPool.Submitter getIoProcessors() {
+ return ioProcessors;
+ }
+
+ /**
+ * Get optional progress callback.
+ * @return callback or null
+ */
+ public Progressable getProgressable() {
+ return progressable;
+ }
+
+ /**
+ * Task attempt directory.
+ * @return the task attempt dir.
+ */
+ public Path getTaskAttemptDir() {
+ return taskAttemptDir;
+ }
+
+ /**
+ * Get the job attempt number.
+ * @return the value
+ */
+ public int getJobAttemptNumber() {
+ return jobAttemptNumber;
+ }
+
+ public String getJobIdSource() {
+ return jobIdSource;
+ }
+
+ /**
+ * Get a thread local task manifest serializer.
+ * @return a serializer.
+ */
+ public JsonSerialization<TaskManifest> currentManifestSerializer() {
+ return threadLocalSerializer.get();
+ }
+
+ /**
+ * Set builder value.
+ * @param value new value
+ * @return the builder
+ */
+ public StageConfig withDeleteTargetPaths(boolean value) {
+ checkOpen();
+ deleteTargetPaths = value;
+ return this;
+ }
+
+ public boolean getDeleteTargetPaths() {
+ return deleteTargetPaths;
+ }
+
+ /**
+ * Enter the stage; calls back to
+ * {@link #enterStageEventHandler} if non-null.
+ * @param stage stage entered
+ */
+ public void enterStage(String stage) {
+ if (enterStageEventHandler != null) {
+ enterStageEventHandler.enterStage(stage);
+ }
+ }
+
+ /**
+ * Exit the stage; calls back to
+ * {@link #enterStageEventHandler} if non-null.
+ * @param stage stage entered
+ */
+ public void exitStage(String stage) {
+ if (enterStageEventHandler != null) {
+ enterStageEventHandler.exitStage(stage);
+ }
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java
new file mode 100644
index 0000000..e23f720
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/StageEventCallbacks.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+/**
+ * Callbacks on stage entry/exit.
+ */
+public interface StageEventCallbacks {
+
+ /**
+ * Callback on stage entry.
+ * @param stage new stage
+ */
+ void enterStage(String stage);
+
+ /**
+ * Callback on stage exit.
+ * @param stage stage exited.
+ */
+ void exitStage(String stage);
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java
new file mode 100644
index 0000000..4765bcc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/TaskAttemptScanDirectoryStage.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.LongSummaryStatistics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DirEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.EntryStatus;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.util.DurationInfo;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_DIRECTORY_COUNT_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_DIRECTORY_DEPTH_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_FILE_COUNT_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASK_FILE_SIZE_MEAN;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_TASK_SCAN_DIRECTORY;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createTaskManifest;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.maybeAddIOStatistics;
+
+/**
+ * Stage to scan a directory tree and build a task manifest.
+ * This is executed by the task committer.
+ */
+public final class TaskAttemptScanDirectoryStage
+ extends AbstractJobOrTaskStage<Void, TaskManifest> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TaskAttemptScanDirectoryStage.class);
+
+ public TaskAttemptScanDirectoryStage(
+ final StageConfig stageConfig) {
+ super(true, stageConfig, OP_STAGE_TASK_SCAN_DIRECTORY, false);
+ }
+
+ /**
+ * Build the Manifest.
+ * @return the manifest
+ * @throws IOException failure.
+ */
+ @Override
+ protected TaskManifest executeStage(final Void arguments)
+ throws IOException {
+
+ final Path taskAttemptDir = getRequiredTaskAttemptDir();
+ final TaskManifest manifest = createTaskManifest(getStageConfig());
+
+ LOG.info("{}: scanning directory {}",
+ getName(), taskAttemptDir);
+
+ final int depth = scanDirectoryTree(manifest,
+ taskAttemptDir,
+ getDestinationDir(),
+ 0, true);
+ List<FileEntry> filesToCommit = manifest.getFilesToCommit();
+ LongSummaryStatistics fileSummary = filesToCommit.stream()
+ .mapToLong(FileEntry::getSize)
+ .summaryStatistics();
+ long fileDataSize = fileSummary.getSum();
+ long fileCount = fileSummary.getCount();
+ int dirCount = manifest.getDestDirectories().size();
+ LOG.info("{}: directory {} contained {} file(s); data size {}",
+ getName(),
+ taskAttemptDir,
+ fileCount,
+ fileDataSize);
+ LOG.info("{}: Directory count = {}; maximum depth {}",
+ getName(),
+ dirCount,
+ depth);
+ // add statistics about the task output which, when aggregated, provides
+ // insight into structure of job, task skew, etc.
+ IOStatisticsStore iostats = getIOStatistics();
+ iostats.addSample(COMMITTER_TASK_DIRECTORY_COUNT_MEAN, dirCount);
+ iostats.addSample(COMMITTER_TASK_DIRECTORY_DEPTH_MEAN, depth);
+ iostats.addSample(COMMITTER_TASK_FILE_COUNT_MEAN, fileCount);
+ iostats.addSample(COMMITTER_TASK_FILE_SIZE_MEAN, fileDataSize);
+
+ return manifest;
+ }
+
+ /**
+ * Recursively scan a directory tree.
+ * The manifest will contain all files to rename
+ * (source and dest) and directories to create.
+ * All files are processed before any of the subdirs are.
+ * This helps in statistics gathering.
+ * There's some optimizations which could be done with async
+ * fetching of the iterators of those subdirs, but as this
+ * is generally off-critical path then that "enhancement"
+ * can be postponed until data suggests this needs improvement.
+ * @param manifest manifest to update
+ * @param srcDir dir to scan
+ * @param destDir destination directory
+ * @param depth depth from the task attempt dir.
+ * @param parentDirExists does the parent dir exist?
+ * @return the maximum depth of child directories
+ * @throws IOException IO failure.
+ */
+ private int scanDirectoryTree(
+ TaskManifest manifest,
+ Path srcDir,
+ Path destDir,
+ int depth,
+ boolean parentDirExists) throws IOException {
+
+ // generate some task progress in case directory scanning is very slow.
+ progress();
+
+ int maxDepth = 0;
+ int files = 0;
+ boolean dirExists = parentDirExists;
+ List<FileStatus> subdirs = new ArrayList<>();
+ try (DurationInfo ignored = new DurationInfo(LOG, false,
+ "Task Attempt %s source dir %s, dest dir %s",
+ getTaskAttemptId(), srcDir, destDir)) {
+
+ // list the directory. This may block until the listing is complete,
+ // or, if the FS does incremental or asynchronous fetching,
+ // then the next()/hasNext() call will block for the results
+ // unless turned off, ABFS does to this async
+ final RemoteIterator<FileStatus> listing = listStatusIterator(srcDir);
+
+ // when the FS (especially ABFS) does an asyn fetch of the listing,
+ // we can probe for the status of the destination dir while that
+ // page is being fetched.
+ // probe for and add the dest dir entry for all but
+ // the base dir
+
+ if (depth > 0) {
+ final EntryStatus status;
+ if (parentDirExists) {
+ final FileStatus destDirStatus = getFileStatusOrNull(destDir);
+ status = EntryStatus.toEntryStatus(destDirStatus);
+ dirExists = destDirStatus != null;
+ } else {
+ // if there is no parent dir, then there is no need to look
+ // for this directory -report it as missing automatically.
+ status = EntryStatus.not_found;
+ }
+ manifest.addDirectory(DirEntry.dirEntry(
+ destDir,
+ status,
+ depth));
+ }
+
+ // process the listing; this is where abfs will block
+ // to wait the result of the list call.
+ while (listing.hasNext()) {
+ final FileStatus st = listing.next();
+ if (st.isFile()) {
+ // this is a file, so add to the list of files to commit.
+ files++;
+ final FileEntry entry = fileEntry(st, destDir);
+ manifest.addFileToCommit(entry);
+ LOG.debug("To rename: {}", entry);
+ } else {
+ if (st.isDirectory()) {
+ // will need to scan this directory too.
+ subdirs.add(st);
+ } else {
+ // some other object. ignoring
+ LOG.info("Ignoring FS object {}", st);
+ }
+ }
+ }
+ // add any statistics provided by the listing.
+ maybeAddIOStatistics(getIOStatistics(), listing);
+ }
+
+ // now scan the subdirectories
+ LOG.debug("{}: Number of subdirectories under {} found: {}; file count {}",
+ getName(), srcDir, subdirs.size(), files);
+
+ for (FileStatus st : subdirs) {
+ Path destSubDir = new Path(destDir, st.getPath().getName());
+ final int d = scanDirectoryTree(manifest,
+ st.getPath(),
+ destSubDir,
+ depth + 1,
+ dirExists);
+ maxDepth = Math.max(maxDepth, d);
+ }
+
+ return 1 + maxDepth;
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java
new file mode 100644
index 0000000..8b72112
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/ValidateRenamedFilesStage.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.OutputValidationException;
+import org.apache.hadoop.util.functional.TaskPool;
+
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.OP_STAGE_JOB_VALIDATE_OUTPUT;
+import static org.apache.hadoop.thirdparty.com.google.common.collect.Iterables.concat;
+
+/**
+ * This stage validates all files by scanning the manifests
+ * and verifying every file in every manifest is of the given size.
+ * Returns a list of all files committed.
+ *
+ * Its cost is one getFileStatus() call (parallelized) per file.
+ * Raises a {@link OutputValidationException} on a validation failure.
+ */
+public class ValidateRenamedFilesStage extends
+ AbstractJobOrTaskStage<
+ List<TaskManifest>,
+ List<FileEntry>> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ValidateRenamedFilesStage.class);
+
+ /**
+ * Set this to halt all workers.
+ */
+ private final AtomicBoolean halt = new AtomicBoolean();
+
+ /**
+ * List of all files committed.
+ */
+ private List<FileEntry> filesCommitted = new ArrayList<>();
+
+ public ValidateRenamedFilesStage(final StageConfig stageConfig) {
+ super(false, stageConfig, OP_STAGE_JOB_VALIDATE_OUTPUT, true);
+ }
+
+ /**
+ * Get the list of files committed.
+ * @return a possibly empty list.
+ */
+ private synchronized List<FileEntry> getFilesCommitted() {
+ return filesCommitted;
+ }
+
+ /**
+ * Add a file entry to the list of committed files.
+ * @param entry entry
+ */
+ private synchronized void addFileCommitted(FileEntry entry) {
+ filesCommitted.add(entry);
+ }
+
+ /**
+ * Validate the task manifests.
+ * This is done by listing all the directories
+ * and verifying that every file in the source list
+ * has a file in the destination of the same size.
+ * If two tasks have both written the same file or
+ * a source file was changed after the task was committed,
+ * then a mistmatch will be detected -provided the file
+ * length is now different.
+ * @param taskManifests list of manifests.
+ * @return list of files committed.
+ */
+ @Override
+ protected List<FileEntry> executeStage(
+ final List<TaskManifest> taskManifests)
+ throws IOException {
+
+ // set the list of files to be as big as the number of tasks.
+ // synchronized to stop complaints.
+ synchronized (this) {
+ filesCommitted = new ArrayList<>(taskManifests.size());
+ }
+
+ // validate all the files.
+
+ final Iterable<FileEntry> filesToCommit = concat(taskManifests.stream()
+ .map(TaskManifest::getFilesToCommit)
+ .collect(Collectors.toList()));
+
+ TaskPool.foreach(filesToCommit)
+ .executeWith(getIOProcessors())
+ .stopOnFailure()
+ .run(this::validateOneFile);
+
+ return getFilesCommitted();
+ }
+
+ /**
+ * Validate a file.
+ * @param entry entry to probe for
+ * @throws IOException IO problem.
+ * @throws OutputValidationException if the entry is not valid
+ */
+ private void validateOneFile(FileEntry entry) throws IOException {
+ updateAuditContext(OP_STAGE_JOB_VALIDATE_OUTPUT);
+
+ if (halt.get()) {
+ // told to stop
+ return;
+ }
+ // report progress back
+ progress();
+ // look validate the file.
+ // raising an FNFE if the file isn't there.
+ FileStatus destStatus;
+ final Path sourcePath = entry.getSourcePath();
+ Path destPath = entry.getDestPath();
+ try {
+ destStatus = getFileStatus(destPath);
+
+ // it must be a file
+ if (!destStatus.isFile()) {
+ throw new OutputValidationException(destPath,
+ "Expected a file renamed from " + sourcePath
+ + "; found " + destStatus);
+ }
+ final long sourceSize = entry.getSize();
+ final long destSize = destStatus.getLen();
+
+ // etags, if the source had one.
+ final String sourceEtag = entry.getEtag();
+ if (isNotBlank(sourceEtag)) {
+ final String destEtag = ManifestCommitterSupport.getEtag(destStatus);
+ if (!sourceEtag.equals(destEtag)) {
+ LOG.warn("Etag of dest file {}: {} does not match that of manifest entry {}",
+ destPath, destStatus, entry);
+ throw new OutputValidationException(destPath,
+ String.format("Expected the file"
+ + " renamed from %s"
+ + " with etag %s and length %s"
+ + " but found a file with etag %s and length %d",
+ sourcePath,
+ sourceEtag,
+ sourceSize,
+ destEtag,
+ destSize));
+
+ }
+ }
+ // check the expected length after any etag validation
+ if (destSize != sourceSize) {
+ LOG.warn("Length of dest file {}: {} does not match that of manifest entry {}",
+ destPath, destStatus, entry);
+ throw new OutputValidationException(destPath,
+ String.format("Expected the file"
+ + " renamed from %s"
+ + " with length %d"
+ + " but found a file of length %d",
+ sourcePath,
+ sourceSize,
+ destSize));
+ }
+
+ } catch (FileNotFoundException e) {
+ // file didn't exist
+ throw new OutputValidationException(destPath,
+ "Expected a file, but it was not found", e);
+ }
+ addFileCommitted(entry);
+ }
+
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java
new file mode 100644
index 0000000..8b86046
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/stages/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Intermediate manifest committer.
+ *
+ * The individual and aggregate stages of the task and job commit process.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.mapreduce.lib.output.committer.manifest.stages;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 844c91c..d315a00 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2231,4 +2231,23 @@
</description>
</property>
+<!-- not yet enabled by default.
+
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+ <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+ <description>
+ The default committer factory for ABFS is for the manifest committer with
+ abfs-specific tuning.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.gs</name>
+ <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+ <description>
+ The default committer factory for google cloud storage is for the manifest committer.
+ </description>
+</property>
+-->
</configuration>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md
new file mode 100644
index 0000000..b446be2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md
@@ -0,0 +1,605 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# The Manifest Committer for Azure and Google Cloud Storage
+
+This document how to use the _Manifest Committer_.
+
+The _Manifest_ committer is a committer for work which provides
+performance on ABFS for "real world" queries,
+and performance and correctness on GCS.
+It also works with other filesystems, including HDFS.
+However, the design is optimized for object stores where
+listing operatons are slow and expensive.
+
+The architecture and implementation of the committer is covered in
+[Manifest Committer Architecture](manifest_committer_architecture.html).
+
+
+The protocol and its correctness are covered in
+[Manifest Committer Protocol](manifest_committer_protocol.html).
+
+It was added in March 2022, and should be considered unstable
+in early releases.
+
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+## Problem:
+
+The only committer of work from Spark to Azure ADLS Gen 2 "abfs://" storage
+which is safe to use is the "v1 file committer".
+
+This is "correct" in that if a task attempt fails, its output is guaranteed not
+to be included in the final out. The "v2" commit algorithm cannot meet that
+guarantee, which is why it is no longer the default.
+
+But: it is slow, especially on jobs where deep directory trees of output are used.
+Why is it slow? It's hard to point at a particular cause, primarily because of
+the lack of any instrumentation in the `FileOutputCommitter`.
+Stack traces of running jobs generally show `rename()`, though list operations
+do surface too.
+
+On Google GCS, neither the v1 nor v2 algorithm are _safe_ because the google
+filesystem doesn't have the atomic directory rename which the v1 algorithm
+requires.
+
+A further issue is that both Azure and GCS storage may encounter scale issues
+with deleting directories with many descendants.
+This can trigger timeouts because the FileOutputCommitter assumes that
+cleaning up after the job is a fast call to `delete("_temporary", true)`.
+
+## Solution.
+
+The _Intermediate Manifest_ committer is a new committer for
+work which should deliver performance on ABFS
+for "real world" queries, and performance and correctness on GCS.
+
+This committer uses the extension point which came in for the S3A committers.
+Users can declare a new committer factory for abfs:// and gcs:// URLs.
+A suitably configured spark deployment will pick up the new committer.
+
+Directory performance issues in job cleanup can be addressed by two options
+1. The committer will parallelize deletion of task attempt directories before
+ deleting the `_temporary` directory.
+1. Cleanup can be disabled. .
+
+The committer can be used with any filesystem client which has a "real" file rename()
+operation.
+It has been optimised for remote object stores where listing and file probes
+are expensive -the design is less likely to offer such signifcant speedup
+on HDFS -though the parallel renaming operations will speed up jobs
+there compared to the classic v1 algorithm.
+
+# <a name="how"></a> How it works
+
+The full details are covered in [Manifest Committer Architecture](manifest_committer_architecture.html).
+
+# <a name="use"></a> Using the committer
+
+The hooks put in to support the S3A committers were designed to allow every
+filesystem schema to provide their own committer.
+See [Switching To an S3A Committer](../../hadoop-aws/tools/hadoop-aws/committers.html#Switching_to_an_S3A_Committer)
+
+A factory for the abfs schema would be defined in
+`mapreduce.outputcommitter.factory.scheme.abfs` ; and a similar one for `gcs`.
+
+Some matching spark configuration changes, especially for parquet binding, will be required.
+These can be done in `core-site.xml`, if it is not defined in the `mapred-default.xml` JAR.
+
+
+```xml
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+ <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.gs</name>
+ <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+</property>
+```
+
+## Binding to the manifest committer in Spark.
+
+In Apache Spark, the configuration can be done either with command line options (after the '--conf') or by using the `spark-defaults.conf` file. The following is an example of using `spark-defaults.conf` also including the configuration for Parquet with a subclass of the parquet
+committer which uses the factory mechansim internally.
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.abfs org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.gs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+```
+
+
+### <a name="committerinfo"></a> Using the Cloudstore `committerinfo` command to probe committer bindings.
+
+The hadoop committer settings can be validated in a recent build of [cloudstore](https://github.com/steveloughran/cloudstore)
+and its `committerinfo` command.
+This command instantiates a committer for that path through the same factory mechanism as MR and spark jobs use,
+then prints its `toString` value.
+
+```
+hadoop jar cloudstore-1.0.jar committerinfo abfs://testing@ukwest.dfs.core.windows.net/
+
+2021-09-16 19:42:59,731 [main] INFO commands.CommitterInfo (StoreDurationInfo.java:<init>(53)) - Starting: Create committer
+Committer factory for path abfs://testing@ukwest.dfs.core.windows.net/ is
+ org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory@3315d2d7
+ (classname org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory)
+2021-09-16 19:43:00,897 [main] INFO manifest.ManifestCommitter (ManifestCommitter.java:<init>(144)) - Created ManifestCommitter with
+ JobID job__0000, Task Attempt attempt__0000_r_000000_1 and destination abfs://testing@ukwest.dfs.core.windows.net/
+Created committer of class org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitter:
+ ManifestCommitter{ManifestCommitterConfig{destinationDir=abfs://testing@ukwest.dfs.core.windows.net/,
+ role='task committer',
+ taskAttemptDir=abfs://testing@ukwest.dfs.core.windows.net/_temporary/manifest_job__0000/0/_temporary/attempt__0000_r_000000_1,
+ createJobMarker=true,
+ jobUniqueId='job__0000',
+ jobUniqueIdSource='JobID',
+ jobAttemptNumber=0,
+ jobAttemptId='job__0000_0',
+ taskId='task__0000_r_000000',
+ taskAttemptId='attempt__0000_r_000000_1'},
+ iostatistics=counters=();
+
+gauges=();
+
+minimums=();
+
+maximums=();
+
+means=();
+}
+
+```
+
+
+## Verifying that the committer was used
+
+The new committer will write a JSON summary of the operation, including statistics, in the `_SUCCESS` file.
+
+If this file exists and is zero bytes long: the classic `FileOutputCommitter` was used.
+
+If this file exists and is greater than zero bytes long, either the manifest committer was used,
+or in the case of S3A filesystems, one of the S3A committers. They all use the same JSON format.
+
+# <a name="configuration"></a> Configuration options
+
+Here are the main configuration options of the committer.
+
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.manifest.committer.delete.target.files` | Delete target files? | `false` |
+| `mapreduce.manifest.committer.io.threads` | Thread count for parallel operations | `64` |
+| `mapreduce.manifest.committer.summary.report.directory` | directory to save reports. | `""` |
+| `mapreduce.manifest.committer.cleanup.parallel.delete` | Delete temporary directories in parallel | `true` |
+| `mapreduce.fileoutputcommitter.cleanup.skipped` | Skip cleanup of `_temporary` directory| `false` |
+| `mapreduce.fileoutputcommitter.cleanup-failures.ignored` | Ignore errors during cleanup | `false` |
+| `mapreduce.fileoutputcommitter.marksuccessfuljobs` | Create a `_SUCCESS` marker file on successful completion. (and delete any existing one in job setup) | `true` |
+
+There are some more, as covered in the (Advanced)[#advanced] section.
+
+
+## <a name="scaling"></a> Scaling jobs `mapreduce.manifest.committer.io.threads`
+
+The core reason that this committer is faster than the classic `FileOutputCommitter`
+is that it tries to parallelize as much file IO as it can during job commit, specifically:
+
+* task manifest loading
+* deletion of files where directories will be created
+* directory creation
+* file-by-file renaming
+* deletion of task attempt directories in job cleanup
+
+These operations are all performed in the same thread pool, whose size is set
+in the option `mapreduce.manifest.committer.io.threads`.
+
+Larger values may be used.
+
+XML
+```xml
+<property>
+ <name>mapreduce.manifest.committer.io.threads</name>
+ <value>200</value>
+</property>
+```
+
+spark-defaults.conf
+```
+spark.hadoop.mapreduce.manifest.committer.io.threads 200
+```
+
+A larger value than that of the number of cores allocated to
+the MapReduce AM or Spark Driver does not directly overload
+the CPUs, as the threads are normally waiting for (slow) IO
+against the object store/filesystem to complete.
+
+Caveats
+* In Spark, multiple jobs may be committed in the same process,
+ each of which will create their own thread pool during job
+ commit or cleanup.
+* Azure rate throttling may be triggered if too many IO requests
+ are made against the store. The rate throttling option
+ `mapreduce.manifest.committer.io.rate` can help avoid this.
+
+
+## <a name="deleting"></a> Optional: deleting target files in Job Commit
+
+The classic `FileOutputCommitter` deletes files at the destination paths
+before renaming the job's files into place.
+
+This is optional in the manifest committers, set in the option
+`mapreduce.manifest.committer.delete.target.files` with a default value of `false`.
+
+This increases performance and is safe to use when all files created by a job
+have unique filenames.
+
+Apache Spark does generate unique filenames for ORC and Parquet since
+[SPARK-8406](https://issues.apache.org/jira/browse/SPARK-8406)
+_Adding UUID to output file name to avoid accidental overwriting_
+
+Avoiding checks for/deleting target files saves one delete call per file being committed, so can
+save a significant amount of store IO.
+
+When appending to existing tables, using formats other than ORC and parquet,
+unless confident that unique identifiers
+are added to each filename, enable deletion of the target files.
+
+```
+spark.hadoop.mapreduce.manifest.committer.delete.target.files true
+```
+
+*Note 1:* the committer will skip deletion operations when it
+created the directory into which a file is to be renamed.
+This makes it slightly more efficient, at least if jobs
+appending data are creating and writing into new partitions.
+
+*Note 2:* the committer still requires tasks within a single
+job to create unique files. This is foundational for
+any job to generate correct data.
+
+# <a name="SUCCESS"></a> Job Summaries in `_SUCCESS` files
+
+The original hadoop committer creates a zero byte `_SUCCESS` file in the root of the output directory
+unless disabled.
+
+This committer writes a JSON summary which includes
+* The name of the committer.
+* Diagnostics information.
+* A list of some of the files created (for testing; a full list is excluded as it can get big).
+* IO Statistics.
+
+If, after running a query, this `_SUCCESS` file is zero bytes long,
+*the new committer has not been used*
+
+If it is not empty, then it can be examined.
+
+## <a name="printer"></a> Viewing `_SUCCESS` file files through the `ManifestPrinter` tool.
+
+The summary files are JSON, and can be viewed in any text editor.
+
+For a more succinct summary, including better display of statistics, use the `ManifestPrinter` tool.
+
+```bash
+hadoop org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter <path>
+```
+
+This works for the files saved at the base of an output directory, and
+any reports saved to a report directory.
+
+## <a name="summaries"></a> Collecting Job Summaries `mapreduce.manifest.committer.summary.report.directory`
+
+The committer can be configured to save the `_SUCCESS` summary files to a report directory,
+irrespective of whether the job succeed or failed, by setting a fileystem path in
+the option `mapreduce.manifest.committer.summary.report.directory`.
+
+The path does not have to be on the same
+store/filesystem as the destination of work. For example, a local fileystem could be used.
+
+XML
+
+```xml
+<property>
+ <name>mapreduce.manifest.committer.summary.report.directory</name>
+ <value>file:///tmp/reports</value>
+</property>
+```
+
+spark-defaults.conf
+
+```
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory file:///tmp/reports
+```
+
+This allows for the statistics of jobs to be collected irrespective of their outcome, Whether or not
+saving the `_SUCCESS` marker is enabled, and without problems caused by a chain of queries
+overwriting the markers.
+
+
+# <a name="cleanup"></a> Cleanup
+
+Job cleanup is convoluted as it is designed to address a number of issues which
+may surface in cloud storage.
+
+* Slow performance for deletion of directories.
+* Timeout when deleting very deep and wide directory trees.
+* General resilience to cleanup issues escalating to job failures.
+
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.fileoutputcommitter.cleanup.skipped` | Skip cleanup of `_temporary` directory| `false` |
+| `mapreduce.fileoutputcommitter.cleanup-failures.ignored` | Ignore errors during cleanup | `false` |
+| `mapreduce.manifest.committer.cleanup.parallel.delete` | Delete task attempt directories in parallel | `true` |
+
+The algorithm is:
+
+```
+if `mapreduce.fileoutputcommitter.cleanup.skipped`:
+ return
+if `mapreduce.manifest.committer.cleanup.parallel.delete`:
+ attempt parallel delete of task directories; catch any exception
+if not `mapreduce.fileoutputcommitter.cleanup.skipped`:
+ delete(`_temporary`); catch any exception
+if caught-exception and not `mapreduce.fileoutputcommitter.cleanup-failures.ignored`:
+ throw caught-exception
+```
+
+It's a bit complicated, but the goal is to perform a fast/scalable delete and
+throw a meaningful exception if that didn't work.
+
+When working with ABFS and GCS, these settings should normally be left alone.
+If somehow errors surface during cleanup, enabling the option to
+ignore failures will ensure the job still completes.
+Disabling cleanup even avoids the overhead of cleanup, but
+requires a workflow or manual operation to clean up all
+`_temporary` directories on a regular basis.
+
+
+# <a name="abfs"></a> Working with Azure ADLS Gen2 Storage
+
+To switch to the manifest committer, the factory for committers for destinations with `abfs://` URLs must
+be switched to the manifest committer factory, either for the application or
+the entire cluster.
+
+```xml
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+ <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+```
+
+This allows for ADLS Gen2 -specific performance and consistency logic to be used from within the committer.
+In particular:
+* the [Etag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) header
+can be collected in listings and used in the job commit phase.
+* IO rename operations are rate limited
+* recovery is attempted when throttling triggers rename failures.
+
+*Warning* This committer is not compatible with older Azure storage services
+(WASB or ADLS Gen 1).
+
+The core set of Azure-optimized options becomes
+
+```xml
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.abfs</name>
+ <value>org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory</value>
+</property>
+
+<property>
+ <name>spark.hadoop.fs.azure.io.rate.limit</name>
+ <value>10000</value>
+</property>
+```
+
+And optional settings for debugging/performance analysis
+
+```xml
+<property>
+ <name>mapreduce.manifest.committer.summary.report.directory</name>
+ <value>abfs:// Path within same store/separate store</value>
+ <description>Optional: path to where job summaries are saved</description>
+</property>
+```
+
+## <a name="abfs-options"></a> Full set of ABFS options for spark
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.abfs org.apache.hadoop.fs.azurebfs.commit.AzureManifestCommitterFactory
+spark.hadoop.fs.azure.io.rate.limit 10000
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory (optional: URI of a directory for job summaries)
+```
+
+## Experimental: ABFS Rename Rate Limiting `fs.azure.io.rate.limit`
+
+To avoid triggering store throttling and backoff delays, as well as other
+throttling-related failure conditions file renames during job commit
+are throttled through a "rate limiter" which limits the number of
+rename operations per second a single instance of the ABFS FileSystem client
+may issue.
+
+| Option | Meaning |
+|--------|---------|
+| `fs.azure.io.rate.limit` | Rate limit in operations/second for IO operations. |
+
+Set the option to `0` remove all rate limiting.
+
+The default value of this is set to 10000, which is the default IO capacity for
+an ADLS storage account.
+
+```xml
+<property>
+ <name>fs.azure.io.rate.limit</name>
+ <value>10000</value>
+ <description>maximum number of renames attempted per second</description>
+</property>
+```
+
+This capacity is set at the level of the filesystem client, and so not
+shared across all processes within a single application, let
+alone other applications sharing the same storage account.
+
+It will be shared with all jobs being committed by the same
+Spark driver, as these do share that filesystem connector.
+
+If rate limiting is imposed, the statistic `store_io_rate_limited` will
+report the time to acquire permits for committing files.
+
+If server-side throttling took place, signs of this can be seen in
+* The store service's logs and their throttling status codes (usually 503 or 500).
+* The job statistic `commit_file_rename_recovered`. This statistic indicates that
+ ADLS throttling manifested as failures in renames, failures which were recovered
+ from in the comitter.
+
+If these are seen -or other applications running at the same time experience
+throttling/throttling-triggered problems, consider reducing the value of
+`fs.azure.io.rate.limit`, and/or requesting a higher IO capacity from Microsoft.
+
+*Important* if you do get extra capacity from Microsoft and you want to use
+it to speed up job commits, increase the value of `fs.azure.io.rate.limit`
+either across the cluster, or specifically for those jobs which you wish
+to allocate extra priority to.
+
+This is still a work in progress; it may be expanded to support
+all IO operations performed by a single filesystem instance.
+
+# <a name="gcs"></a> Working with Google Cloud Storage
+
+The manifest committer is compatible with and tested against Google cloud storage through
+the gcs-connector library from google, which provides a Hadoop filesystem client for the
+schema `gs`.
+
+Google cloud storage has the semantics needed for the commit protocol
+to work safely.
+
+The Spark settings to switch to this committer are
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.gs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory (optional: URI of a directory for job summaries)
+```
+
+The store's directory delete operations are `O(files)` so the value
+of `mapreduce.manifest.committer.cleanup.parallel.delete`
+SHOULD be left at the default of `true`.
+
+For mapreduce, declare the binding in `core-site.xml`or `mapred-site.xml`
+```xml
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.gcs</name>
+ <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory</value>
+</property>
+```
+
+
+# <a name="hdfs"></a> Working with HDFS
+
+This committer _does_ work with HDFS, it has just been targeted at object stores with
+reduced performance on some operations, especially listing and renaming,
+and semantics too reduced for the classic `FileOutputCommitter` to rely on
+(specifically GCS).
+
+To use on HDFS, set the `ManifestCommitterFactory` as the committer factory for `hdfs://` URLs.
+
+Because HDFS does fast directory deletion, there is no need to parallelize deletion
+of task attempt directories during cleanup, so set
+`mapreduce.manifest.committer.cleanup.parallel.delete` to `false`
+
+The final spark bindings becomes
+
+```
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.hdfs org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory
+spark.hadoop.mapreduce.manifest.committer.cleanup.parallel.delete false
+spark.sql.parquet.output.committer.class org.apache.spark.internal.io.cloud.BindingParquetOutputCommitter
+spark.sql.sources.commitProtocolClass org.apache.spark.internal.io.cloud.PathOutputCommitProtocol
+
+spark.hadoop.mapreduce.manifest.committer.summary.report.directory (optional: URI of a directory for job summaries)
+```
+
+# <a name="advanced"></a> Advanced Topics
+
+## Advanced Configuration options
+
+There are some advanced options which are intended for development and testing,
+rather than production use.
+
+| Option | Meaning | Default Value |
+|--------|---------|---------------|
+| `mapreduce.manifest.committer.store.operations.classname` | Classname for Manifest Store Operations | `""` |
+| `mapreduce.manifest.committer.validate.output` | Perform output validation? | `false` |
+
+
+## Validating output `mapreduce.manifest.committer.validate.output`
+
+The option `mapreduce.manifest.committer.validate.output` triggers a check of every renamed file to
+verify it has the expected length.
+
+This adds the overhead of a `HEAD` request per file, and so is recommended for testing only.
+
+There is no verification of the actual contents.
+
+## Controlling storage integration `mapreduce.manifest.committer.store.operations.classname`
+
+The manifest committer interacts with filesystems through implementations of the interface
+`ManifestStoreOperations`.
+It is possible to provide custom implementations for store-specific features.
+There is one of these for ABFS; when the abfs-specific committer factory is used this
+is automatically set.
+
+It can be explicitly set.
+```xml
+<property>
+ <name>mapreduce.manifest.committer.store.operations.classname</name>
+ <value>org.apache.hadoop.fs.azurebfs.commit.AbfsManifestStoreOperations</value>
+</property>
+```
+
+The default implementation may also be configured.
+
+```xml
+<property>
+ <name>mapreduce.manifest.committer.store.operations.classname</name>
+ <value>org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem</value>
+</property>
+```
+
+There is no need to alter these values, except when writing new implementations for other stores,
+something which is only needed if the store provides extra integration support for the
+committer.
+
+## <a name="concurrent"></a> Support for concurrent test runs.
+
+It *may* be possible to run multiple jobs targeting the same directory tree.
+
+For this to work, a number of conditions must be met:
+
+* When using spark, unique job IDs must be set. This meangs the Spark distribution
+ MUST contain the patches for
+ [SPARK-33402](https://issues.apache.org/jira/browse/SPARK-33402)
+ and
+ [SPARK-33230](https://issues.apache.org/jira/browse/SPARK-33230).
+* Cleanup of the `_temporary` directory must be disabled by setting
+ `mapreduce.fileoutputcommitter.cleanup.skipped` to `true`.
+* All jobs/tasks must create files with unique filenames.
+* All jobs must create output with the same directory partition structure.
+* Remember to delete the `_temporary` directory later!
+
+This has *NOT BEEN TESTED*
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md
new file mode 100644
index 0000000..d2b4f1e
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer_architecture.md
@@ -0,0 +1,335 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+# Manifest Committer Architecture
+
+This document describes the architecture and other implementation/correctness
+aspects of the [Manifest Committer](manifest_committer.html)
+
+The protocol and its correctness are covered in [Manifest Committer Protocol](manifest_committer_protocol.html).
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+The _Manifest_ committer is a committer for work which provides performance on ABFS for "real world"
+queries, and performance and correctness on GCS.
+
+This committer uses the extension point which came in for the S3A committers.
+Users can declare a new committer factory for `abfs://` and `gcs://` URLs.
+It can be used through Hadoop MapReduce and Apache Spark.
+
+## Background
+
+### Terminology
+
+| Term | Meaning|
+|------|--------|
+| Committer | A class which can be invoked by MR/Spark to perform the task and job commit operations. |
+| Spark Driver | The spark process scheduling the work and choreographing the commit operation.|
+| Job | In MapReduce. the entire application. In spark, this is a single stage in a chain of work |
+| Job Attempt | A single attempt at a job. MR supports multiple Job attempts with recovery on partial job failure. Spark says "start again from scratch" |
+| Task | a subsection of a job, such as processing one file, or one part of a file |
+| Task ID | ID of the task, unique within this job. Usually starts at 0 and is used in filenames (part-0000, part-001, etc.) |
+| Task attempt (TA) | An attempt to perform a task. It may fail, in which case MR/spark will schedule another. |
+| Task Attempt ID | A unique ID for the task attempt. The Task ID + an attempt counter.|
+| Destination directory | The final destination of work.|
+| Job Attempt Directory | A temporary directory used by the job attempt. This is always _underneath_ the destination directory, so as to ensure it is in the same encryption zone as HDFS, storage volume in other filesystems, etc.|
+| Task Attempt directory | (also known as "Task Attempt Working Directory"). Directory exclusive for each task attempt under which files are written |
+| Task Commit | Taking the output of a Task Attempt and making it the final/exclusive result of that "successful" Task.|
+| Job Commit | aggregating all the outputs of all committed tasks and producing the final results of the job. |
+
+
+
+The purpose of a committer is to ensure that the complete output of
+a job ends up in the destination, even in the presence of failures of tasks.
+
+- _Complete:_ the output includes the work of all successful tasks.
+- _Exclusive:_ the output of unsuccessful tasks is not present.
+- _Concurrent:_ When multiple tasks are committed in parallel the output is the same as when
+ the task commits are serialized. This is not a requirement of Job Commit.
+- _Abortable:_ jobs and tasks may be aborted prior to job commit, after which their output is not visible.
+- _Continuity of correctness:_ once a job is committed, the output of any failed,
+ aborted, or unsuccessful task MUST NO appear at some point in the future.
+
+For Hive's classic hierarchical-directory-structured tables, job committing
+requires the output of all committed tasks to be put into the correct location
+in the directory tree.
+
+The committer built into `hadoop-mapreduce-client-core` module is the `FileOutputCommitter`.
+
+
+
+## The Manifest Committer: A high performance committer for Spark on Azure and Google storage.
+
+The Manifest Committer is a higher performance committer for ABFS and GCS storage
+for jobs which create file across deep directory trees through many tasks.
+
+It will also work on `hdfs://` and indeed, `file://` URLs, but
+it is optimized to address listing and renaming performance and throttling
+issues in cloud storage.
+
+It *will not* work correctly with S3, because it relies on an atomic rename-no-overwrite
+operation to commit the manifest file. It will also have the performance
+problems of copying rather than moving all the generated data.
+
+Although it will work with MapReduce
+there is no handling of multiple job attempts with recovery from previous failed
+attempts.
+
+### The Manifest
+
+A Manifest file is designed which contains (along with IOStatistics and some
+other things)
+
+1. A list of destination directories which must be created if they do not exist.
+1. A list of files to rename, recorded as (absolute source, absolute destination,
+ file-size) entries.
+
+### Task Commit
+
+Task attempts are committed by:
+
+1. Recursively listing the task attempt working dir to build
+ 1. A list of directories under which files are renamed.
+ 2. A list of files to rename: source, destination, size and optionally, etag.
+2. Saving this information in a manifest file in the job attempt directory with
+ a filename derived from the Task ID.
+ Note: writing to a temp file and then renaming to the final path will be used
+ to ensure the manifest creation is atomic.
+
+
+No renaming takes place —the files are left in their original location.
+
+The directory treewalk is single-threaded, then it is `O(directories)`,
+with each directory listing using one or more paged LIST calls.
+
+This is simple, and for most tasks, the scan is off the critical path of of the job.
+
+Statistics analysis may justify moving to parallel scans in future.
+
+
+### Job Commit
+
+Job Commit consists of:
+
+1. List all manifest files in the job attempt directory.
+1. Load each manifest file, create directories which do not yet exist, then
+ rename each file in the rename list.
+1. Save a JSON `_SUCCESS` file with the same format as the S3A committer (for
+ testing; use write and rename for atomic save)
+
+The job commit phase supports parallelization for many tasks and many files
+per task, specifically:
+
+1. Manifest tasks are loaded and processed in a pool of "manifest processor"
+ threads.
+2. Directory creation and file rename operations are each processed in a pool of "
+ executor" threads: many renames can execute in parallel as they use minimal
+ network IO.
+3. job cleanup can parallelize deletion of task attempt directories. This
+ is relevant as directory deletion is `O(files)` on Google cloud storage,
+ and also on ABFS when OAuth authentication is used.
+
+
+### Ancestor directory preparation
+
+Optional scan of all ancestors ...if any are files, delete.
+
+
+### Parent directory creation
+
+1. Probe shared directory map for directory existing. If found: operation is
+ complete.
+1. if the map is empty, call `getFileStatus()` on the path. Not found: create
+ directory, add entry and those of all parent paths Found and is directory:
+ add entry and those of all parent paths Found and is file: delete. then
+ create as before.
+
+Efficiently handling concurrent creation of directories (or delete+create) is going to be a
+troublespot; some effort is invested there to build the set of directories to
+create.
+
+### File Rename
+
+Files are renamed in parallel.
+
+A pre-rename check for anything being at that path (and deleting it) will be optional.
+With spark creating new UUIDs for each file, this isn't going to happen, and
+saves HTTP requests.
+
+
+### Validation
+
+Optional scan of all committed files and verify length and, if known,
+etag. For testing and diagnostics.
+
+## Benefits
+
+* Pushes the source tree list operations into the task commit phase, which is
+ generally off the critical path of execution
+* Provides an atomic task commit to GCS, as there is no expectation that
+ directory rename is atomic
+* It is possible to pass IOStatistics from workers in manifest.
+* Allows for some pre-rename operations similar to the S3A "Partitioned Staging
+ committer". This can be configured to delete all existing entries in
+ directories scheduled to be created -or fail if those partitions are
+ non-empty.
+ See [Partitioned Staging Committer](../../hadoop-aws/tools/hadoop-aws/committers.html#The_.E2.80.9CPartitioned.E2.80.9D_Staging_Committer)
+* Allows for an optional preflight validation check (verify no duplicate files created by different tasks)
+* Manifests can be viewed, size of output determined, etc, during
+ development/debugging.
+
+### Disadvantages
+
+* Needs a new manifest file format.
+* May makes task commit more complex.
+
+This solution is necessary for GCS and should be beneficial on ABFS as listing
+overheads are paid for in the task committers.
+
+# Implementation Details
+
+### Constraints
+
+A key goal is to keep the manifest committer isolated and neither
+touch the existing committer code nor other parts of the hadoop codebase.
+
+It must plug directly into MR and Spark without needing any changes
+other than already implemented for the S3A Committers
+
+* Self-contained: MUST NOT require changes to hadoop-common, etc.
+* Isolated: MUST NOT make changes to existing committers
+* Integrated: MUST bind via `PathOutputCommitterFactory`.
+
+As a result of this there's a bit of copy and paste from elsewhere,
+e.g. `org.apache.hadoop.util.functional.TaskPool`
+is based on S3ACommitter's `org.apache.hadoop.fs.s3a.commit.Tasks`.
+
+The` _SUCCESS` file MUST be compatible with the S3A JSON file.
+This is to ensure any existing test suites which validate
+S3A committer output can be retargeted at jobs executed
+by the manifest committer without any changes.
+
+
+#### Progress callbacks in job commit.
+
+When? Proposed: heartbeat until renaming finally finishes.
+
+#### Error handling and aborting in job commit.
+
+We would want to stop the entire job commit. Some atomic boolean "abort job"
+would need to be checked in the processing of each task committer thread's
+iteraton through a directory (or processing of each file?)
+Failures in listing or renaming will need to be escalated to halting the entire
+job commit. This implies that any IOE raised in asynchronous rename operation or
+in a task committer thread must:
+
+1. be caught
+1. be stored in a shared field/variable
+1. trigger the abort
+1. be rethrown at the end of the `commitJob()` call
+
+#### Avoiding deadlocks
+
+If a job commit stage is using a thread pool for per-task operations, e.g. loading
+files, that same thread pool MUST NOT be used for parallel operations within
+the per-task stage.
+
+As every `JobStage` is executed in sequence within task or job commit, it
+is safe to share the same thread pool across stages.
+
+In the current implementation, there is no parallel "per manifest" operation
+in job commit other than for actually loading the files.
+The operations to create directories and to rename files are actually
+performed without performing parallel processing of individual manifests.
+
+Directory Preparation: merge the directory lists of all manifests,
+then queue for creation the (hopefully very much smaller) set of unique
+directories.
+
+Rename: iterate through all manifests and queue their renames into a pool for
+renaming.
+
+#### Thread pool lifetimes
+
+The lifespan of thread pools is constrained to that of the stage configuration,
+which will be limited to within each of the `PathOutputCommitter` methods
+to setup, commit, abort and cleanup.
+
+This avoids the thread pool lifecycle problems of the S3A Committers.
+
+#### Scale issues similar to S3A HADOOP-16570.
+
+This was a failure in terasorting where many tasks each generated many files;
... 7102 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HADOOP-18162. hadoop-common support for MAPREDUCE-7341 Manifest Committer
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e06ed88012e820efd953e0b7c00681cb4ced831f
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Mar 16 15:12:03 2022 +0000
HADOOP-18162. hadoop-common support for MAPREDUCE-7341 Manifest Committer
* New statistic names in StoreStatisticNames
(for joint use with s3a committers)
* Improvements to IOStatistics implementation classes
* RateLimiting wrapper to guava RateLimiter
* S3A committer Tasks moved over as TaskPool and
added support for RemoteIterator
* JsonSerialization.load() to fail fast if source does not exist
+ tests.
This commit is a prerequisite for the main MAPREDUCE-7341 Manifest Committer
patch.
Contributed by Steve Loughran
Change-Id: Ia92e2ab5083ac3d8d3d713a4d9cb3e9e0278f654
---
.../hadoop/fs/statistics/StoreStatisticNames.java | 6 +
.../fs/statistics/impl/IOStatisticsBinding.java | 29 +-
.../fs/statistics/impl/IOStatisticsStore.java | 11 +
.../impl/PairedDurationTrackerFactory.java | 5 +
.../statistics/impl/StatisticDurationTracker.java | 7 +
.../org/apache/hadoop/util/JsonSerialization.java | 5 +-
.../java/org/apache/hadoop/util/RateLimiting.java | 52 ++
.../apache/hadoop/util/RateLimitingFactory.java | 102 ++++
.../functional/CloseableTaskPoolSubmitter.java | 74 +++
.../apache/hadoop/util/functional/TaskPool.java | 613 +++++++++++++++++++++
.../hadoop/util/functional/TestTaskPool.java | 585 ++++++++++++++++++++
11 files changed, 1487 insertions(+), 2 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index 9ec8dcd..166007f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -113,6 +113,9 @@ public final class StoreStatisticNames {
public static final String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
/** {@value}. */
+ public static final String OP_MSYNC = "op_msync";
+
+ /** {@value}. */
public static final String OP_OPEN = "op_open";
/** {@value}. */
@@ -172,6 +175,9 @@ public final class StoreStatisticNames {
public static final String STORE_IO_THROTTLED
= "store_io_throttled";
+ /** Rate limiting was reported {@value}. */
+ public static final String STORE_IO_RATE_LIMITED = "store_io_rate_limited";
+
/** Requests made of a store: {@value}. */
public static final String STORE_IO_REQUEST
= "store_io_request";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
index 1aece01..37ca37a1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.statistics.impl;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.Serializable;
+import java.time.Duration;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.Callable;
@@ -450,12 +451,37 @@ public final class IOStatisticsBinding {
* @param factory factory of duration trackers
* @param statistic statistic key
* @param input input callable.
+ * @throws IOException IO failure.
*/
public static void trackDurationOfInvocation(
DurationTrackerFactory factory,
String statistic,
InvocationRaisingIOE input) throws IOException {
+ measureDurationOfInvocation(factory, statistic, input);
+ }
+
+ /**
+ * Given an IOException raising callable/lambda expression,
+ * execute it and update the relevant statistic,
+ * returning the measured duration.
+ *
+ * {@link #trackDurationOfInvocation(DurationTrackerFactory, String, InvocationRaisingIOE)}
+ * with the duration returned for logging etc.; added as a new
+ * method to avoid linking problems with any code calling the existing
+ * method.
+ *
+ * @param factory factory of duration trackers
+ * @param statistic statistic key
+ * @param input input callable.
+ * @return the duration of the operation, as measured by the duration tracker.
+ * @throws IOException IO failure.
+ */
+ public static Duration measureDurationOfInvocation(
+ DurationTrackerFactory factory,
+ String statistic,
+ InvocationRaisingIOE input) throws IOException {
+
// create the tracker outside try-with-resources so
// that failures can be set in the catcher.
DurationTracker tracker = createTracker(factory, statistic);
@@ -473,6 +499,7 @@ public final class IOStatisticsBinding {
// set the failed flag.
tracker.close();
}
+ return tracker.asDuration();
}
/**
@@ -622,7 +649,7 @@ public final class IOStatisticsBinding {
* @param statistic statistic to track
* @return a duration tracker.
*/
- private static DurationTracker createTracker(
+ public static DurationTracker createTracker(
@Nullable final DurationTrackerFactory factory,
final String statistic) {
return factory != null
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
index 1b4139e..c083ad8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStore.java
@@ -255,4 +255,15 @@ public interface IOStatisticsStore extends IOStatistics,
*/
void addTimedOperation(String prefix, Duration duration);
+ /**
+ * Add a statistics sample as a min, max and mean and count.
+ * @param key key to add.
+ * @param count count.
+ */
+ default void addSample(String key, long count) {
+ incrementCounter(key, count);
+ addMeanStatisticSample(key, count);
+ addMaximumSample(key, count);
+ addMinimumSample(key, count);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
index 33b13f7..9bc0133 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/PairedDurationTrackerFactory.java
@@ -88,6 +88,11 @@ final class PairedDurationTrackerFactory implements DurationTrackerFactory {
public Duration asDuration() {
return firstDuration.asDuration();
}
+
+ @Override
+ public String toString() {
+ return firstDuration.toString();
+ }
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
index ef9e7cb..04d3013 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java
@@ -103,4 +103,11 @@ public class StatisticDurationTracker extends OperationDuration
}
iostats.addTimedOperation(name, asDuration());
}
+
+ @Override
+ public String toString() {
+ return " Duration of " +
+ (failed? (key + StoreStatisticNames.SUFFIX_FAILURES) : key)
+ + ": " + super.toString();
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index a80f958..111ce8f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -173,6 +173,9 @@ public class JsonSerialization<T> {
@SuppressWarnings("unchecked")
public synchronized T load(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
+ if (!jsonFile.exists()) {
+ throw new FileNotFoundException("No such file: " + jsonFile);
+ }
if (!jsonFile.isFile()) {
throw new FileNotFoundException("Not a file: " + jsonFile);
}
@@ -182,7 +185,7 @@ public class JsonSerialization<T> {
try {
return mapper.readValue(jsonFile, classType);
} catch (IOException e) {
- LOG.error("Exception while parsing json file {}", jsonFile, e);
+ LOG.warn("Exception while parsing json file {}", jsonFile, e);
throw e;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java
new file mode 100644
index 0000000..ae119c0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimiting.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Minimal subset of google rate limiter class.
+ * Can be used to throttle use of object stores where excess load
+ * will trigger cluster-wide throttling, backoff etc. and so collapse
+ * performance.
+ * The time waited is returned as a Duration type.
+ * The google rate limiter implements this by allowing a caller to ask for
+ * more capacity than is available. This will be granted
+ * but the subsequent request will be blocked if the bucket of
+ * capacity hasn't let refilled to the point where there is
+ * capacity again.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface RateLimiting {
+
+ /**
+ * Acquire rate limiter capacity.
+ * If there is not enough space, the permits will be acquired,
+ * but the subsequent call will block until the capacity has been
+ * refilled.
+ * @param requestedCapacity capacity to acquire.
+ * @return time spent waiting for output.
+ */
+ Duration acquire(int requestedCapacity);
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java
new file mode 100644
index 0000000..6214154
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RateLimitingFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.time.Duration;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter;
+
+/**
+ * Factory for Rate Limiting.
+ * This should be only place in the code where the guava RateLimiter is imported.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class RateLimitingFactory {
+
+ private static final RateLimiting UNLIMITED = new NoRateLimiting();
+
+ /**
+ * No waiting took place.
+ */
+ private static final Duration INSTANTLY = Duration.ofMillis(0);
+
+ private RateLimitingFactory() {
+ }
+
+ /**
+ * No Rate Limiting.
+ */
+ private static class NoRateLimiting implements RateLimiting {
+
+
+ @Override
+ public Duration acquire(int requestedCapacity) {
+ return INSTANTLY;
+ }
+ }
+
+ /**
+ * Rate limiting restricted to that of a google rate limiter.
+ */
+ private static final class RestrictedRateLimiting implements RateLimiting {
+ private final RateLimiter limiter;
+
+ /**
+ * Constructor.
+ * @param capacityPerSecond capacity in permits/second.
+ */
+ private RestrictedRateLimiting(int capacityPerSecond) {
+ this.limiter = RateLimiter.create(capacityPerSecond);
+ }
+
+ @Override
+ public Duration acquire(int requestedCapacity) {
+ final double delayMillis = limiter.acquire(requestedCapacity);
+ return delayMillis == 0
+ ? INSTANTLY
+ : Duration.ofMillis((long) (delayMillis * 1000));
+ }
+
+ }
+
+ /**
+ * Get the unlimited rate.
+ * @return a rate limiter which always has capacity.
+ */
+ public static RateLimiting unlimitedRate() {
+ return UNLIMITED;
+ }
+
+ /**
+ * Create an instance.
+ * If the rate is 0; return the unlimited rate.
+ * @param capacity capacity in permits/second.
+ * @return limiter restricted to the given capacity.
+ */
+ public static RateLimiting create(int capacity) {
+
+ return capacity == 0
+ ? unlimitedRate()
+ : new RestrictedRateLimiting(capacity);
+ }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java
new file mode 100644
index 0000000..26b687a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util.functional;
+
+import java.io.Closeable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A task submitter which is closeable, and whose close() call
+ * shuts down the pool. This can help manage
+ * thread pool lifecycles.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class CloseableTaskPoolSubmitter implements TaskPool.Submitter,
+ Closeable {
+
+ /** Executors. */
+ private ExecutorService pool;
+
+ /**
+ * Constructor.
+ * @param pool non-null executor.
+ */
+ public CloseableTaskPoolSubmitter(final ExecutorService pool) {
+ this.pool = requireNonNull(pool);
+ }
+
+ /**
+ * Get the pool.
+ * @return the pool.
+ */
+ public ExecutorService getPool() {
+ return pool;
+ }
+
+ /**
+ * Shut down the pool.
+ */
+ @Override
+ public void close() {
+ if (pool != null) {
+ pool.shutdown();
+ pool = null;
+ }
+ }
+
+ @Override
+ public Future<?> submit(final Runnable task) {
+ return pool.submit(task);
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java
new file mode 100644
index 0000000..0abaab2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java
@@ -0,0 +1,613 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util.functional;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.RemoteIterator;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.util.functional.RemoteIterators.remoteIteratorFromIterable;
+
+/**
+ * Utility class for parallel execution, takes closures for the various
+ * actions.
+ * There is no retry logic: it is expected to be handled by the closures.
+ * From {@code org.apache.hadoop.fs.s3a.commit.Tasks} which came from
+ * the Netflix committer patch.
+ * Apache Iceberg has its own version of this, with a common ancestor
+ * at some point in its history.
+ * A key difference with this class is that the iterator is always,
+ * internally, an {@link RemoteIterator}.
+ * This is to allow tasks to be scheduled while incremental operations
+ * such as paged directory listings are still collecting in results.
+ *
+ * While awaiting completion, this thread spins and sleeps a time of
+ * {@link #SLEEP_INTERVAL_AWAITING_COMPLETION}, which, being a
+ * busy-wait, is inefficient.
+ * There's an implicit assumption that remote IO is being performed, and
+ * so this is not impacting throughput/performance.
+ *
+ * History:
+ * This class came with the Netflix contributions to the S3A committers
+ * in HADOOP-13786.
+ * It was moved into hadoop-common for use in the manifest committer and
+ * anywhere else it is needed, and renamed in the process as
+ * "Tasks" has too many meanings in the hadoop source.
+ * The iterator was then changed from a normal java iterable
+ * to a hadoop {@link org.apache.hadoop.fs.RemoteIterator}.
+ * This allows a task pool to be supplied with incremental listings
+ * from object stores, scheduling work as pages of listing
+ * results come in, rather than blocking until the entire
+ * directory/directory tree etc has been enumerated.
+ *
+ * There is a variant of this in Apache Iceberg in
+ * {@code org.apache.iceberg.util.Tasks}
+ * That is not derived from any version in the hadoop codebase, it
+ * just shares a common ancestor somewhere in the Netflix codebase.
+ * It is the more sophisticated version.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class TaskPool {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TaskPool.class);
+
+ /**
+ * Interval in milliseconds to await completion.
+ */
+ private static final int SLEEP_INTERVAL_AWAITING_COMPLETION = 10;
+
+ private TaskPool() {
+ }
+
+ /**
+ * Callback invoked to process an item.
+ * @param <I> item type being processed
+ * @param <E> exception class which may be raised
+ */
+ @FunctionalInterface
+ public interface Task<I, E extends Exception> {
+ void run(I item) throws E;
+ }
+
+ /**
+ * Callback invoked on a failure.
+ * @param <I> item type being processed
+ * @param <E> exception class which may be raised
+ */
+ @FunctionalInterface
+ public interface FailureTask<I, E extends Exception> {
+
+ /**
+ * process a failure.
+ * @param item item the task is processing
+ * @param exception the exception which was raised.
+ * @throws E Exception of type E
+ */
+ void run(I item, Exception exception) throws E;
+ }
+
+ /**
+ * Builder for task execution.
+ * @param <I> item type
+ */
+ public static class Builder<I> {
+ private final RemoteIterator<I> items;
+ private Submitter service = null;
+ private FailureTask<I, ?> onFailure = null;
+ private boolean stopOnFailure = false;
+ private boolean suppressExceptions = false;
+ private Task<I, ?> revertTask = null;
+ private boolean stopRevertsOnFailure = false;
+ private Task<I, ?> abortTask = null;
+ private boolean stopAbortsOnFailure = false;
+ private int sleepInterval = SLEEP_INTERVAL_AWAITING_COMPLETION;
+
+ /**
+ * Create the builder.
+ * @param items items to process
+ */
+ Builder(RemoteIterator<I> items) {
+ this.items = requireNonNull(items, "items");
+ }
+
+ /**
+ * Create the builder.
+ * @param items items to process
+ */
+ Builder(Iterable<I> items) {
+ this(remoteIteratorFromIterable(items));
+ }
+
+ /**
+ * Declare executor service: if null, the tasks are executed in a single
+ * thread.
+ * @param submitter service to schedule tasks with.
+ * @return this builder.
+ */
+ public Builder<I> executeWith(@Nullable Submitter submitter) {
+
+ this.service = submitter;
+ return this;
+ }
+
+ /**
+ * Task to invoke on failure.
+ * @param task task
+ * @return the builder
+ */
+ public Builder<I> onFailure(FailureTask<I, ?> task) {
+ this.onFailure = task;
+ return this;
+ }
+
+ public Builder<I> stopOnFailure() {
+ this.stopOnFailure = true;
+ return this;
+ }
+
+ /**
+ * Suppress exceptions from tasks.
+ * RemoteIterator exceptions are not suppressable.
+ * @return the builder.
+ */
+ public Builder<I> suppressExceptions() {
+ return suppressExceptions(true);
+ }
+
+ /**
+ * Suppress exceptions from tasks.
+ * RemoteIterator exceptions are not suppressable.
+ * @param suppress new value
+ * @return the builder.
+ */
+ public Builder<I> suppressExceptions(boolean suppress) {
+ this.suppressExceptions = suppress;
+ return this;
+ }
+
+ /**
+ * Task to revert with after another task failed.
+ * @param task task to execute
+ * @return the builder
+ */
+ public Builder<I> revertWith(Task<I, ?> task) {
+ this.revertTask = task;
+ return this;
+ }
+
+ /**
+ * Stop trying to revert if one operation fails.
+ * @return the builder
+ */
+ public Builder<I> stopRevertsOnFailure() {
+ this.stopRevertsOnFailure = true;
+ return this;
+ }
+
+ /**
+ * Task to abort with after another task failed.
+ * @param task task to execute
+ * @return the builder
+ */
+ public Builder<I> abortWith(Task<I, ?> task) {
+ this.abortTask = task;
+ return this;
+ }
+
+ /**
+ * Stop trying to abort if one operation fails.
+ * @return the builder
+ */
+ public Builder<I> stopAbortsOnFailure() {
+ this.stopAbortsOnFailure = true;
+ return this;
+ }
+
+ /**
+ * Set the sleep interval.
+ * @param value new value
+ * @return the builder
+ */
+ public Builder sleepInterval(final int value) {
+ sleepInterval = value;
+ return this;
+ }
+
+ /**
+ * Execute the task across the data.
+ * @param task task to execute
+ * @param <E> exception which may be raised in execution.
+ * @return true if the operation executed successfully
+ * @throws E any exception raised.
+ * @throws IOException IOExceptions raised by remote iterator or in execution.
+ */
+ public <E extends Exception> boolean run(Task<I, E> task) throws E, IOException {
+ requireNonNull(items, "items");
+ if (!items.hasNext()) {
+ // if there are no items, return without worrying about
+ // execution pools, errors etc.
+ return true;
+ }
+ if (service != null) {
+ // thread pool, so run in parallel
+ return runParallel(task);
+ } else {
+ // single threaded execution.
+ return runSingleThreaded(task);
+ }
+ }
+
+ /**
+ * Single threaded execution.
+ * @param task task to execute
+ * @param <E> exception which may be raised in execution.
+ * @return true if the operation executed successfully
+ * @throws E any exception raised.
+ * @throws IOException IOExceptions raised by remote iterator or in execution.
+ */
+ private <E extends Exception> boolean runSingleThreaded(Task<I, E> task)
+ throws E, IOException {
+ List<I> succeeded = new ArrayList<>();
+ List<Exception> exceptions = new ArrayList<>();
+
+ RemoteIterator<I> iterator = items;
+ boolean threw = true;
+ try {
+ while (iterator.hasNext()) {
+ I item = iterator.next();
+ try {
+ task.run(item);
+ succeeded.add(item);
+
+ } catch (Exception e) {
+ exceptions.add(e);
+
+ if (onFailure != null) {
+ try {
+ onFailure.run(item, e);
+ } catch (Exception failException) {
+ LOG.error("Failed to clean up on failure", e);
+ // keep going
+ }
+ }
+
+ if (stopOnFailure) {
+ break;
+ }
+ }
+ }
+
+ threw = false;
+ } catch (IOException iteratorIOE) {
+ // an IOE is reaised here during iteration
+ LOG.debug("IOException when iterating through {}", iterator, iteratorIOE);
+ throw iteratorIOE;
+ } finally {
+ // threw handles exceptions that were *not* caught by the catch block,
+ // and exceptions that were caught and possibly handled by onFailure
+ // are kept in exceptions.
+ if (threw || !exceptions.isEmpty()) {
+ if (revertTask != null) {
+ boolean failed = false;
+ for (I item : succeeded) {
+ try {
+ revertTask.run(item);
+ } catch (Exception e) {
+ LOG.error("Failed to revert task", e);
+ failed = true;
+ // keep going
+ }
+ if (stopRevertsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+
+ if (abortTask != null) {
+ boolean failed = false;
+ while (iterator.hasNext()) {
+ try {
+ abortTask.run(iterator.next());
+ } catch (Exception e) {
+ failed = true;
+ LOG.error("Failed to abort task", e);
+ // keep going
+ }
+ if (stopAbortsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (!suppressExceptions && !exceptions.isEmpty()) {
+ TaskPool.<E>throwOne(exceptions);
+ }
+
+ return exceptions.isEmpty();
+ }
+
+ /**
+ * Parallel execution.
+ * @param task task to execute
+ * @param <E> exception which may be raised in execution.
+ * @return true if the operation executed successfully
+ * @throws E any exception raised.
+ * @throws IOException IOExceptions raised by remote iterator or in execution.
+ */
+ private <E extends Exception> boolean runParallel(final Task<I, E> task)
+ throws E, IOException {
+ final Queue<I> succeeded = new ConcurrentLinkedQueue<>();
+ final Queue<Exception> exceptions = new ConcurrentLinkedQueue<>();
+ final AtomicBoolean taskFailed = new AtomicBoolean(false);
+ final AtomicBoolean abortFailed = new AtomicBoolean(false);
+ final AtomicBoolean revertFailed = new AtomicBoolean(false);
+
+ List<Future<?>> futures = new ArrayList<>();
+
+ IOException iteratorIOE = null;
+ final RemoteIterator<I> iterator = this.items;
+ try {
+ while(iterator.hasNext()) {
+ final I item = iterator.next();
+ // submit a task for each item that will either run or abort the task
+ futures.add(service.submit(() -> {
+ if (!(stopOnFailure && taskFailed.get())) {
+ // run the task
+ boolean threw = true;
+ try {
+ LOG.debug("Executing task");
+ task.run(item);
+ succeeded.add(item);
+ LOG.debug("Task succeeded");
+
+ threw = false;
+
+ } catch (Exception e) {
+ taskFailed.set(true);
+ exceptions.add(e);
+ LOG.info("Task failed {}", e.toString());
+ LOG.debug("Task failed", e);
+
+ if (onFailure != null) {
+ try {
+ onFailure.run(item, e);
+ } catch (Exception failException) {
+ LOG.warn("Failed to clean up on failure", e);
+ // swallow the exception
+ }
+ }
+ } finally {
+ if (threw) {
+ taskFailed.set(true);
+ }
+ }
+
+ } else if (abortTask != null) {
+ // abort the task instead of running it
+ if (stopAbortsOnFailure && abortFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ LOG.info("Aborting task");
+ abortTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to abort task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ abortFailed.set(true);
+ }
+ }
+ }
+ }));
+ }
+ } catch (IOException e) {
+ // iterator failure.
+ LOG.debug("IOException when iterating through {}", iterator, e);
+ iteratorIOE = e;
+ // mark as a task failure so all submitted tasks will halt/abort
+ taskFailed.set(true);
+ }
+
+ // let the above tasks complete (or abort)
+ waitFor(futures, sleepInterval);
+ int futureCount = futures.size();
+ futures.clear();
+
+ if (taskFailed.get() && revertTask != null) {
+ // at least one task failed, revert any that succeeded
+ LOG.info("Reverting all {} succeeded tasks from {} futures",
+ succeeded.size(), futureCount);
+ for (final I item : succeeded) {
+ futures.add(service.submit(() -> {
+ if (stopRevertsOnFailure && revertFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ revertTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to revert task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ revertFailed.set(true);
+ }
+ }
+ }));
+ }
+
+ // let the revert tasks complete
+ waitFor(futures, sleepInterval);
+ }
+
+ // give priority to execution exceptions over
+ // iterator exceptions.
+ if (!suppressExceptions && !exceptions.isEmpty()) {
+ // there's an exception list to build up, cast and throw.
+ TaskPool.<E>throwOne(exceptions);
+ }
+
+ // raise any iterator exception.
+ // this can not be suppressed.
+ if (iteratorIOE != null) {
+ throw iteratorIOE;
+ }
+
+ // return true if all tasks succeeded.
+ return !taskFailed.get();
+ }
+ }
+
+ /**
+ * Wait for all the futures to complete; there's a small sleep between
+ * each iteration; enough to yield the CPU.
+ * @param futures futures.
+ * @param sleepInterval Interval in milliseconds to await completion.
+ */
+ private static void waitFor(Collection<Future<?>> futures, int sleepInterval) {
+ int size = futures.size();
+ LOG.debug("Waiting for {} tasks to complete", size);
+ int oldNumFinished = 0;
+ while (true) {
+ int numFinished = (int) futures.stream().filter(Future::isDone).count();
+
+ if (oldNumFinished != numFinished) {
+ LOG.debug("Finished count -> {}/{}", numFinished, size);
+ oldNumFinished = numFinished;
+ }
+
+ if (numFinished == size) {
+ // all of the futures are done, stop looping
+ break;
+ } else {
+ try {
+ Thread.sleep(sleepInterval);
+ } catch (InterruptedException e) {
+ futures.forEach(future -> future.cancel(true));
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Create a task builder for the iterable.
+ * @param items item source.
+ * @param <I> type of result.
+ * @return builder.
+ */
+ public static <I> Builder<I> foreach(Iterable<I> items) {
+ return new Builder<>(requireNonNull(items, "items"));
+ }
+
+ /**
+ * Create a task builder for the remote iterator.
+ * @param items item source.
+ * @param <I> type of result.
+ * @return builder.
+ */
+ public static <I> Builder<I> foreach(RemoteIterator<I> items) {
+ return new Builder<>(items);
+ }
+
+ public static <I> Builder<I> foreach(I[] items) {
+ return new Builder<>(Arrays.asList(requireNonNull(items, "items")));
+ }
+
+ /**
+ * Throw one exception, adding the others as suppressed
+ * exceptions attached to the one thrown.
+ * This method never completes normally.
+ * @param exceptions collection of exceptions
+ * @param <E> class of exceptions
+ * @throws E an extracted exception.
+ */
+ private static <E extends Exception> void throwOne(
+ Collection<Exception> exceptions)
+ throws E {
+ Iterator<Exception> iter = exceptions.iterator();
+ Exception e = iter.next();
+ Class<? extends Exception> exceptionClass = e.getClass();
+
+ while (iter.hasNext()) {
+ Exception other = iter.next();
+ if (!exceptionClass.isInstance(other)) {
+ e.addSuppressed(other);
+ }
+ }
+
+ TaskPool.<E>castAndThrow(e);
+ }
+
+ /**
+ * Raise an exception of the declared type.
+ * This method never completes normally.
+ * @param e exception
+ * @param <E> class of exceptions
+ * @throws E a recast exception.
+ */
+ @SuppressWarnings("unchecked")
+ private static <E extends Exception> void castAndThrow(Exception e) throws E {
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ }
+ throw (E) e;
+ }
+
+ /**
+ * Interface to whatever lets us submit tasks.
+ */
+ public interface Submitter {
+
+ /**
+ * Submit work.
+ * @param task task to execute
+ * @return the future of the submitted task.
+ */
+ Future<?> submit(Runnable task);
+ }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
new file mode 100644
index 0000000..dfee6fc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestTaskPool.java
@@ -0,0 +1,585 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util.functional;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test Task Pool class.
+ * This is pulled straight out of the S3A version.
+ */
+@RunWith(Parameterized.class)
+public class TestTaskPool extends HadoopTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestTaskPool.class);
+
+ public static final int ITEM_COUNT = 16;
+
+ private static final int FAILPOINT = 8;
+
+ private final int numThreads;
+
+ /**
+ * Thread pool for task execution.
+ */
+ private ExecutorService threadPool;
+
+ /**
+ * Task submitter bonded to the thread pool, or
+ * null for the 0-thread case.
+ */
+ private TaskPool.Submitter submitter;
+
+ private final CounterTask failingTask
+ = new CounterTask("failing committer", FAILPOINT, Item::commit);
+
+ private final FailureCounter failures
+ = new FailureCounter("failures", 0, null);
+
+ private final CounterTask reverter
+ = new CounterTask("reverter", 0, Item::revert);
+
+ private final CounterTask aborter
+ = new CounterTask("aborter", 0, Item::abort);
+
+ /**
+ * Test array for parameterized test runs: how many threads and
+ * to use. Threading makes some of the assertions brittle; there are
+ * more checks on single thread than parallel ops.
+ * @return a list of parameter tuples.
+ */
+ @Parameterized.Parameters(name = "threads={0}")
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {0},
+ {1},
+ {3},
+ {8},
+ {16},
+ });
+ }
+
+ private List<Item> items;
+
+ /**
+ * Construct the parameterized test.
+ * @param numThreads number of threads
+ */
+ public TestTaskPool(int numThreads) {
+ this.numThreads = numThreads;
+ }
+
+ /**
+ * In a parallel test run there is more than one thread doing the execution.
+ * @return true if the threadpool size is >1
+ */
+ public boolean isParallel() {
+ return numThreads > 1;
+ }
+
+ @Before
+ public void setup() {
+ items = IntStream.rangeClosed(1, ITEM_COUNT)
+ .mapToObj(i -> new Item(i,
+ String.format("With %d threads", numThreads)))
+ .collect(Collectors.toList());
+
+ if (numThreads > 0) {
+ threadPool = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat(getMethodName() + "-pool-%d")
+ .build());
+ submitter = new PoolSubmitter();
+ } else {
+ submitter = null;
+ }
+
+ }
+
+ @After
+ public void teardown() {
+ if (threadPool != null) {
+ threadPool.shutdown();
+ threadPool = null;
+ }
+ }
+
+ private class PoolSubmitter implements TaskPool.Submitter {
+
+ @Override
+ public Future<?> submit(final Runnable task) {
+ return threadPool.submit(task);
+ }
+
+ }
+
+ /**
+ * create the builder.
+ * @return pre-inited builder
+ */
+ private TaskPool.Builder<Item> builder() {
+ return TaskPool.foreach(items).executeWith(submitter);
+ }
+
+ private void assertRun(TaskPool.Builder<Item> builder,
+ CounterTask task) throws IOException {
+ boolean b = builder.run(task);
+ assertTrue("Run of " + task + " failed", b);
+ }
+
+ private void assertFailed(TaskPool.Builder<Item> builder,
+ CounterTask task) throws IOException {
+ boolean b = builder.run(task);
+ assertFalse("Run of " + task + " unexpectedly succeeded", b);
+ }
+
+ private String itemsToString() {
+ return "[" + items.stream().map(Item::toString)
+ .collect(Collectors.joining("\n")) + "]";
+ }
+
+ @Test
+ public void testSimpleInvocation() throws Throwable {
+ CounterTask t = new CounterTask("simple", 0, Item::commit);
+ assertRun(builder(), t);
+ t.assertInvoked("", ITEM_COUNT);
+ }
+
+ @Test
+ public void testFailNoStoppingSuppressed() throws Throwable {
+ assertFailed(builder().suppressExceptions(), failingTask);
+ failingTask.assertInvoked("Continued through operations", ITEM_COUNT);
+ items.forEach(Item::assertCommittedOrFailed);
+ }
+
+ @Test
+ public void testFailFastSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .stopOnFailure(),
+ failingTask);
+ if (isParallel()) {
+ failingTask.assertInvokedAtLeast("stop fast", FAILPOINT);
+ } else {
+ failingTask.assertInvoked("stop fast", FAILPOINT);
+ }
+ }
+
+ @Test
+ public void testFailedCallAbortSuppressed() throws Throwable {
+ assertFailed(builder()
+ .stopOnFailure()
+ .suppressExceptions()
+ .abortWith(aborter),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ if (!isParallel()) {
+ aborter.assertInvokedAtLeast("abort", 1);
+ // all uncommitted items were aborted
+ items.stream().filter(i -> !i.committed)
+ .map(Item::assertAborted);
+ items.stream().filter(i -> i.committed)
+ .forEach(i -> assertFalse(i.toString(), i.aborted));
+ }
+ }
+
+ @Test
+ public void testFailedCalledWhenNotStoppingSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailFastCallRevertSuppressed() throws Throwable {
+ assertFailed(builder()
+ .stopOnFailure()
+ .revertWith(reverter)
+ .abortWith(aborter)
+ .suppressExceptions()
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ if (!isParallel()) {
+ aborter.assertInvokedAtLeast("abort", 1);
+ // all uncommitted items were aborted
+ items.stream().filter(i -> !i.committed)
+ .filter(i -> !i.failed)
+ .forEach(Item::assertAborted);
+ }
+ // all committed were reverted
+ items.stream().filter(i -> i.committed && !i.failed)
+ .forEach(Item::assertReverted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailSlowCallRevertSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .revertWith(reverter)
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ // all committed were reverted
+ // identify which task failed from the set
+ int failing = failures.getItem().id;
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> i.committed)
+ .forEach(Item::assertReverted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailFastExceptions() throws Throwable {
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .run(failingTask));
+ if (isParallel()) {
+ failingTask.assertInvokedAtLeast("stop fast", FAILPOINT);
+ } else {
+ failingTask.assertInvoked("stop fast", FAILPOINT);
+ }
+ }
+
+ @Test
+ public void testFailSlowExceptions() throws Throwable {
+ intercept(IOException.class,
+ () -> builder()
+ .run(failingTask));
+ failingTask.assertInvoked("continued through operations", ITEM_COUNT);
+ items.forEach(Item::assertCommittedOrFailed);
+ }
+
+ @Test
+ public void testFailFastExceptionsWithAbortFailure() throws Throwable {
+ CounterTask failFirst = new CounterTask("task", 1, Item::commit);
+ CounterTask a = new CounterTask("aborter", 1, Item::abort);
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .abortWith(a)
+ .run(failFirst));
+ if (!isParallel()) {
+ // expect the other tasks to be aborted
+ a.assertInvokedAtLeast("abort", ITEM_COUNT - 1);
+ }
+ }
+
+ @Test
+ public void testFailFastExceptionsWithAbortFailureStopped() throws Throwable {
+ CounterTask failFirst = new CounterTask("task", 1, Item::commit);
+ CounterTask a = new CounterTask("aborter", 1, Item::abort);
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .stopAbortsOnFailure()
+ .abortWith(a)
+ .run(failFirst));
+ if (!isParallel()) {
+ // expect the other tasks to be aborted
+ a.assertInvoked("abort", 1);
+ }
+ }
+
+ /**
+ * Fail the last one committed, all the rest will be reverted.
+ * The actual ID of the last task has to be picke dup from the
+ * failure callback, as in the pool it may be one of any.
+ */
+ @Test
+ public void testRevertAllSuppressed() throws Throwable {
+ CounterTask failLast = new CounterTask("task", ITEM_COUNT, Item::commit);
+
+ assertFailed(builder()
+ .suppressExceptions()
+ .stopOnFailure()
+ .revertWith(reverter)
+ .abortWith(aborter)
+ .onFailure(failures),
+ failLast);
+ failLast.assertInvoked("success", ITEM_COUNT);
+ int abCount = aborter.getCount();
+ int revCount = reverter.getCount();
+ assertEquals(ITEM_COUNT, 1 + abCount + revCount);
+ // identify which task failed from the set
+ int failing = failures.getItem().id;
+ // all committed were reverted
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> i.committed)
+ .forEach(Item::assertReverted);
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> !i.committed)
+ .forEach(Item::assertAborted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+
+ /**
+ * The Item which tasks process.
+ */
+ private final class Item {
+
+ private final int id;
+
+ private final String text;
+
+ private volatile boolean committed, aborted, reverted, failed;
+
+ private Item(int item, String text) {
+ this.id = item;
+ this.text = text;
+ }
+
+ boolean commit() {
+ committed = true;
+ return true;
+ }
+
+ boolean abort() {
+ aborted = true;
+ return true;
+ }
+
+ boolean revert() {
+ reverted = true;
+ return true;
+ }
+
+ boolean fail() {
+ failed = true;
+ return true;
+ }
+
+ public Item assertCommitted() {
+ assertTrue(toString() + " was not committed in\n"
+ + itemsToString(),
+ committed);
+ return this;
+ }
+
+ public Item assertCommittedOrFailed() {
+ assertTrue(toString() + " was not committed nor failed in\n"
+ + itemsToString(),
+ committed || failed);
+ return this;
+ }
+
+ public Item assertAborted() {
+ assertTrue(toString() + " was not aborted in\n"
+ + itemsToString(),
+ aborted);
+ return this;
+ }
+
+ public Item assertReverted() {
+ assertTrue(toString() + " was not reverted in\n"
+ + itemsToString(),
+ reverted);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("Item{");
+ sb.append(String.format("[%02d]", id));
+ sb.append(", committed=").append(committed);
+ sb.append(", aborted=").append(aborted);
+ sb.append(", reverted=").append(reverted);
+ sb.append(", failed=").append(failed);
+ sb.append(", text=").append(text);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Class which can count invocations and, if limit > 0, will raise
+ * an exception on the specific invocation of {@link #note(Object)}
+ * whose count == limit.
+ */
+ private class BaseCounter {
+
+ private final AtomicInteger counter = new AtomicInteger(0);
+
+ private final int limit;
+
+ private final String name;
+
+ private Item item;
+
+ private final Optional<Function<Item, Boolean>> action;
+
+ /**
+ * Base counter, tracks items.
+ * @param name name for string/exception/logs.
+ * @param limit limit at which an exception is raised, 0 == never
+ * @param action optional action to invoke after the increment,
+ * before limit check
+ */
+ BaseCounter(String name,
+ int limit,
+ Function<Item, Boolean> action) {
+ this.name = name;
+ this.limit = limit;
+ this.action = Optional.ofNullable(action);
+ }
+
+ /**
+ * Apply the action to an item; log at info afterwards with both the
+ * before and after string values of the item.
+ * @param i item to process.
+ * @throws IOException failure in the action
+ */
+ void process(Item i) throws IOException {
+ this.item = i;
+ int count = counter.incrementAndGet();
+ if (limit == count) {
+ i.fail();
+ LOG.info("{}: Failed {}", this, i);
+ throw new IOException(String.format("%s: Limit %d reached for %s",
+ this, limit, i));
+ }
+ String before = i.toString();
+ action.map(a -> a.apply(i));
+ LOG.info("{}: {} -> {}", this, before, i);
+ }
+
+ int getCount() {
+ return counter.get();
+ }
+
+ Item getItem() {
+ return item;
+ }
+
+ void assertInvoked(String text, int expected) {
+ assertEquals(toString() + ": " + text, expected, getCount());
+ }
+
+ void assertInvokedAtLeast(String text, int expected) {
+ int actual = getCount();
+ assertTrue(toString() + ": " + text
+ + "-expected " + expected
+ + " invocations, but got " + actual
+ + " in " + itemsToString(),
+ expected <= actual);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "BaseCounter{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", count=").append(counter.get());
+ sb.append(", limit=").append(limit);
+ sb.append(", item=").append(item);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ private final class CounterTask
+ extends BaseCounter implements TaskPool.Task<Item, IOException> {
+
+ private CounterTask(String name, int limit,
+ Function<Item, Boolean> action) {
+ super(name, limit, action);
+ }
+
+ @Override
+ public void run(Item item) throws IOException {
+ process(item);
+ }
+
+ }
+
+ private final class FailureCounter
+ extends BaseCounter
+ implements TaskPool.FailureTask<Item, IOException> {
+
+ private Exception exception;
+
+ private FailureCounter(String name, int limit,
+ Function<Item, Boolean> action) {
+ super(name, limit, action);
+ }
+
+ @Override
+ public void run(Item item, Exception ex) throws IOException {
+ process(item);
+ this.exception = ex;
+ }
+
+ private Exception getException() {
+ return exception;
+ }
+
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: HADOOP-18163. hadoop-azure support for the Manifest Committer of MAPREDUCE-7341
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 3238bdab8984c4ccec31af703b4edec8781f8caf
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Wed Mar 16 15:41:03 2022 +0000
HADOOP-18163. hadoop-azure support for the Manifest Committer of MAPREDUCE-7341
Follow-on patch to MAPREDUCE-7341, adding ABFS support and tests
* resilient rename
* tests for job commit through the manifest committer.
contains
- HADOOP-17976. ABFS etag extraction inconsistent between LIST and HEAD calls
- HADOOP-16204. ABFS tests to include terasort
Contributed by Steve Loughran.
Change-Id: I0a7d4043bdf19bcb00c033fc389730109b93b77f
---
hadoop-project/pom.xml | 7 +
hadoop-tools/hadoop-azure/pom.xml | 81 ++++-
.../src/config/checkstyle-suppressions.xml | 3 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 9 +
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 201 +++++++++---
.../fs/azurebfs/AzureBlobFileSystemStore.java | 34 +-
.../commit/AbfsManifestStoreOperations.java | 130 ++++++++
.../commit/AzureManifestCommitterFactory.java | 58 ++++
.../azurebfs/commit/ResilientCommitByRename.java | 101 ++++++
.../hadoop/fs/azurebfs/commit/package-info.java | 28 ++
.../fs/azurebfs/constants/ConfigurationKeys.java | 3 +
.../constants/FileSystemConfigurations.java | 5 +
.../hadoop/fs/azurebfs/services/AbfsClient.java | 99 +++++-
.../fs/azurebfs/services/SimpleKeyProvider.java | 10 +-
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 11 +-
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 13 +-
.../ITestAzureBlobFileSystemDelegationSAS.java | 5 +-
.../fs/azurebfs/ITestCustomerProvidedKey.java | 3 +-
.../fs/azurebfs/commit/AbfsCommitTestHelper.java | 49 +++
.../azurebfs/commit/AbstractAbfsClusterITest.java | 260 +++++++++++++++
.../fs/azurebfs/commit/ITestAbfsCleanupStage.java | 54 ++++
.../azurebfs/commit/ITestAbfsCommitTaskStage.java | 54 ++++
.../ITestAbfsCreateOutputDirectoriesStage.java | 54 ++++
.../ITestAbfsJobThroughManifestCommitter.java | 101 ++++++
.../commit/ITestAbfsLoadManifestsStage.java | 55 ++++
.../commit/ITestAbfsManifestCommitProtocol.java | 62 ++++
.../commit/ITestAbfsManifestStoreOperations.java | 175 ++++++++++
.../commit/ITestAbfsRenameStageFailure.java | 69 ++++
.../commit/ITestAbfsTaskManifestFileIO.java | 54 ++++
.../fs/azurebfs/commit/ITestAbfsTerasort.java | 353 +++++++++++++++++++++
.../hadoop/fs/azurebfs/commit/package-info.java | 24 ++
.../azurebfs/contract/AbfsFileSystemContract.java | 2 +-
.../hadoop-azure/src/test/resources/core-site.xml | 25 ++
33 files changed, 2108 insertions(+), 84 deletions(-)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index b9e40e6..c8411a5 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -378,6 +378,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>${hadoop.version}</version>
</dependency>
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index c8c5cc3..40a401b 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -231,6 +231,67 @@
<scope>test</scope>
</dependency>
+ <!--
+ the mapreduce-client-core module is compiled against for the
+ manifest committer support. It is not needed on the classpath
+ except when using this committer, so it only tagged as
+ "provided".
+ It is not exported as a transitive dependency of the JAR.
+ Applications which wish to to use the manifest committer
+ will need to explicity add the mapreduce JAR to their classpath.
+ This is already done by MapReduce and Spark.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!--
+ These are only added to the classpath for tests,
+ and are not exported by the test JAR as transitive
+ dependencies.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-hs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-examples</artifactId>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ <!-- artifacts needed to bring up a Mini MR Yarn cluster-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
@@ -331,7 +392,7 @@
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -362,7 +423,7 @@
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -404,7 +465,7 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
@@ -494,7 +555,7 @@
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -538,7 +599,7 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
@@ -556,6 +617,7 @@
<exclude>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</exclude>
<exclude>**/azurebfs/ITestSmallWriteOptimization.java</exclude>
<exclude>**/azurebfs/services/ITestReadBufferManager.java</exclude>
+ <exclude>**/azurebfs/commit/*.java</exclude>
</excludes>
</configuration>
@@ -584,7 +646,7 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
@@ -597,6 +659,7 @@
<include>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</include>
<include>**/azurebfs/ITestSmallWriteOptimization.java</include>
<include>**/azurebfs/services/ITestReadBufferManager.java</include>
+ <include>**/azurebfs/commit/*.java</include>
</includes>
</configuration>
</execution>
@@ -646,7 +709,7 @@
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -676,7 +739,7 @@
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
<fs.azure.scale.test.huge.huge.partitionsize>${fs.azure.scale.test.huge.partitionsize}</fs.azure.scale.test.huge.huge.partitionsize>
@@ -718,7 +781,7 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
<fs.azure.scale.test.huge.filesize>${fs.azure.scale.test.huge.filesize}</fs.azure.scale.test.huge.filesize>
diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
index 070c8c1..fd2a7c2 100644
--- a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
+++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml
@@ -48,4 +48,7 @@
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]utils[\\/]Base64.java"/>
<suppress checks="ParameterNumber|VisibilityModifier"
files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]ITestSmallWriteOptimization.java"/>
+ <!-- allow tests to use _ for ordering. -->
+ <suppress checks="MethodName"
+ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]commit[\\/]ITestAbfsTerasort.java"/>
</suppressions>
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
index 50cc574..6462cc9 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java
@@ -260,6 +260,11 @@ public class AbfsConfiguration{
DefaultValue = DEFAULT_ENABLE_AUTOTHROTTLING)
private boolean enableAutoThrottling;
+ @IntegerConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_ABFS_IO_RATE_LIMIT,
+ MinValue = 0,
+ DefaultValue = RATE_LIMIT_DEFAULT)
+ private int rateLimit;
+
@StringConfigurationValidatorAnnotation(ConfigurationKey = FS_AZURE_USER_AGENT_PREFIX_KEY,
DefaultValue = DEFAULT_FS_AZURE_USER_AGENT_PREFIX)
private String userAgentId;
@@ -726,6 +731,10 @@ public class AbfsConfiguration{
return this.enableAutoThrottling;
}
+ public int getRateLimit() {
+ return rateLimit;
+ }
+
public String getCustomUserAgentPrefix() {
return this.userAgentId;
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 86bd904..82e1072 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -27,6 +27,7 @@ import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.AccessDeniedException;
+import java.time.Duration;
import java.util.Hashtable;
import java.util.List;
import java.util.ArrayList;
@@ -42,13 +43,17 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.azurebfs.commit.ResilientCommitByRename;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsClientThrottlingIntercept;
import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator;
@@ -94,9 +99,12 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.statistics.IOStatistics;
import org.apache.hadoop.fs.statistics.IOStatisticsSource;
import org.apache.hadoop.fs.store.DataBlocks;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.RateLimiting;
+import org.apache.hadoop.util.RateLimitingFactory;
import org.apache.hadoop.util.functional.RemoteIterators;
import org.apache.hadoop.util.DurationInfo;
import org.apache.hadoop.util.LambdaUtils;
@@ -143,6 +151,9 @@ public class AzureBlobFileSystem extends FileSystem
/** Maximum Active blocks per OutputStream. */
private int blockOutputActiveBlocks;
+ /** Rate limiting for operations which use it to throttle their IO. */
+ private RateLimiting rateLimiting;
+
@Override
public void initialize(URI uri, Configuration configuration)
throws IOException {
@@ -215,7 +226,7 @@ public class AzureBlobFileSystem extends FileSystem
}
AbfsClientThrottlingIntercept.initializeSingleton(abfsConfiguration.isAutoThrottlingEnabled());
-
+ rateLimiting = RateLimitingFactory.create(abfsConfiguration.getRateLimit());
LOG.debug("Initializing AzureBlobFileSystem for {} complete", uri);
}
@@ -262,7 +273,7 @@ public class AzureBlobFileSystem extends FileSystem
InputStream inputStream = abfsStore.openFileForRead(qualifiedPath,
options, statistics, tracingContext);
return new FSDataInputStream(inputStream);
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
return null;
}
@@ -282,8 +293,13 @@ public class AzureBlobFileSystem extends FileSystem
}
@Override
- public FSDataOutputStream create(final Path f, final FsPermission permission, final boolean overwrite, final int bufferSize,
- final short replication, final long blockSize, final Progressable progress) throws IOException {
+ public FSDataOutputStream create(final Path f,
+ final FsPermission permission,
+ final boolean overwrite,
+ final int bufferSize,
+ final short replication,
+ final long blockSize,
+ final Progressable progress) throws IOException {
LOG.debug("AzureBlobFileSystem.create path: {} permission: {} overwrite: {} bufferSize: {}",
f,
permission,
@@ -303,7 +319,7 @@ public class AzureBlobFileSystem extends FileSystem
FsPermission.getUMask(getConf()), tracingContext);
statIncrement(FILES_CREATED);
return new FSDataOutputStream(outputStream, statistics);
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
checkException(f, ex);
return null;
}
@@ -332,8 +348,12 @@ public class AzureBlobFileSystem extends FileSystem
@Override
@SuppressWarnings("deprecation")
- public FSDataOutputStream createNonRecursive(final Path f, final FsPermission permission,
- final EnumSet<CreateFlag> flags, final int bufferSize, final short replication, final long blockSize,
+ public FSDataOutputStream createNonRecursive(final Path f,
+ final FsPermission permission,
+ final EnumSet<CreateFlag> flags,
+ final int bufferSize,
+ final short replication,
+ final long blockSize,
final Progressable progress) throws IOException {
// Check if file should be appended or overwritten. Assume that the file
@@ -357,7 +377,8 @@ public class AzureBlobFileSystem extends FileSystem
}
@Override
- public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress) throws IOException {
+ public FSDataOutputStream append(final Path f, final int bufferSize, final Progressable progress)
+ throws IOException {
LOG.debug(
"AzureBlobFileSystem.append path: {} bufferSize: {}",
f.toString(),
@@ -372,7 +393,7 @@ public class AzureBlobFileSystem extends FileSystem
OutputStream outputStream = abfsStore
.openFileForWrite(qualifiedPath, statistics, false, tracingContext);
return new FSDataOutputStream(outputStream, statistics);
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
checkException(f, ex);
return null;
}
@@ -395,7 +416,7 @@ public class AzureBlobFileSystem extends FileSystem
fileSystemId, FSOperationType.RENAME, true, tracingHeaderFormat,
listener);
// rename under same folder;
- if(makeQualified(parentFolder).equals(qualifiedDstPath)) {
+ if (makeQualified(parentFolder).equals(qualifiedDstPath)) {
return tryGetFileStatus(qualifiedSrcPath, tracingContext) != null;
}
@@ -430,24 +451,99 @@ public class AzureBlobFileSystem extends FileSystem
qualifiedDstPath = makeQualified(adjustedDst);
- abfsStore.rename(qualifiedSrcPath, qualifiedDstPath, tracingContext);
+ abfsStore.rename(qualifiedSrcPath, qualifiedDstPath, tracingContext, null);
return true;
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
LOG.debug("Rename operation failed. ", ex);
checkException(
- src,
- ex,
- AzureServiceErrorCode.PATH_ALREADY_EXISTS,
- AzureServiceErrorCode.INVALID_RENAME_SOURCE_PATH,
- AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND,
- AzureServiceErrorCode.INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE,
- AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND,
- AzureServiceErrorCode.INTERNAL_OPERATION_ABORT);
+ src,
+ ex,
+ AzureServiceErrorCode.PATH_ALREADY_EXISTS,
+ AzureServiceErrorCode.INVALID_RENAME_SOURCE_PATH,
+ AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND,
+ AzureServiceErrorCode.INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE,
+ AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND,
+ AzureServiceErrorCode.INTERNAL_OPERATION_ABORT);
return false;
}
}
+ /**
+ * Private method to create resilient commit support.
+ * @return a new instance
+ * @param path destination path
+ * @throws IOException problem probing store capabilities
+ * @throws UnsupportedOperationException if the store lacks this support
+ */
+ @InterfaceAudience.Private
+ public ResilientCommitByRename createResilientCommitSupport(final Path path)
+ throws IOException {
+
+ if (!hasPathCapability(path,
+ CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME)) {
+ throw new UnsupportedOperationException(
+ "Resilient commit support not available for " + path);
+ }
+ return new ResilientCommitByRenameImpl();
+ }
+
+ /**
+ * Resilient commit support.
+ * Provided as a nested class to avoid contaminating the
+ * FS instance with too many private methods which end up
+ * being used widely (as has happened to the S3A FS)
+ */
+ public class ResilientCommitByRenameImpl implements ResilientCommitByRename {
+
+ /**
+ * Perform the rename.
+ * This will be rate limited, as well as able to recover
+ * from rename errors if the etag was passed in.
+ * @param source path to source file
+ * @param dest destination of rename.
+ * @param sourceEtag etag of source file. may be null or empty
+ * @return the outcome of the operation
+ * @throws IOException any rename failure which was not recovered from.
+ */
+ public Pair<Boolean, Duration> commitSingleFileByRename(
+ final Path source,
+ final Path dest,
+ @Nullable final String sourceEtag) throws IOException {
+
+ LOG.debug("renameFileWithEtag source: {} dest: {} etag {}", source, dest, sourceEtag);
+ statIncrement(CALL_RENAME);
+
+ trailingPeriodCheck(dest);
+ Path qualifiedSrcPath = makeQualified(source);
+ Path qualifiedDstPath = makeQualified(dest);
+
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.RENAME, true, tracingHeaderFormat,
+ listener);
+
+ if (qualifiedSrcPath.equals(qualifiedDstPath)) {
+ // rename to itself is forbidden
+ throw new PathIOException(qualifiedSrcPath.toString(), "cannot rename object onto self");
+ }
+
+ // acquire one IO permit
+ final Duration waitTime = rateLimiting.acquire(1);
+
+ try {
+ final boolean recovered = abfsStore.rename(qualifiedSrcPath,
+ qualifiedDstPath, tracingContext, sourceEtag);
+ return Pair.of(recovered, waitTime);
+ } catch (AzureBlobFileSystemException ex) {
+ LOG.debug("Rename operation failed. ", ex);
+ checkException(source, ex);
+ // never reached
+ return null;
+ }
+
+ }
+ }
+
@Override
public boolean delete(final Path f, final boolean recursive) throws IOException {
LOG.debug(
@@ -525,7 +621,7 @@ public class AzureBlobFileSystem extends FileSystem
* @throws IllegalArgumentException if the path has a trailing period (.)
*/
private void trailingPeriodCheck(Path path) throws IllegalArgumentException {
- while (!path.isRoot()){
+ while (!path.isRoot()) {
String pathToString = path.toString();
if (pathToString.length() != 0) {
if (pathToString.charAt(pathToString.length() - 1) == '.') {
@@ -533,8 +629,7 @@ public class AzureBlobFileSystem extends FileSystem
"ABFS does not allow files or directories to end with a dot.");
}
path = path.getParent();
- }
- else {
+ } else {
break;
}
}
@@ -593,10 +688,10 @@ public class AzureBlobFileSystem extends FileSystem
@Override
public FileStatus getFileStatus(final Path f) throws IOException {
- TracingContext tracingContext = new TracingContext(clientCorrelationId,
- fileSystemId, FSOperationType.GET_FILESTATUS, tracingHeaderFormat,
- listener);
- return getFileStatus(f, tracingContext);
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.GET_FILESTATUS, tracingHeaderFormat,
+ listener);
+ return getFileStatus(f, tracingContext);
}
private FileStatus getFileStatus(final Path path,
@@ -607,7 +702,7 @@ public class AzureBlobFileSystem extends FileSystem
try {
return abfsStore.getFileStatus(qualifiedPath, tracingContext);
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
return null;
}
@@ -631,7 +726,7 @@ public class AzureBlobFileSystem extends FileSystem
fileSystemId, FSOperationType.BREAK_LEASE, tracingHeaderFormat,
listener);
abfsStore.breakLease(qualifiedPath, tracingContext);
- } catch(AzureBlobFileSystemException ex) {
+ } catch (AzureBlobFileSystemException ex) {
checkException(f, ex);
}
}
@@ -658,7 +753,6 @@ public class AzureBlobFileSystem extends FileSystem
return super.makeQualified(path);
}
-
@Override
public Path getWorkingDirectory() {
return this.workingDir;
@@ -681,8 +775,8 @@ public class AzureBlobFileSystem extends FileSystem
@Override
public Path getHomeDirectory() {
return makeQualified(new Path(
- FileSystemConfigurations.USER_HOME_DIRECTORY_PREFIX
- + "/" + abfsStore.getUser()));
+ FileSystemConfigurations.USER_HOME_DIRECTORY_PREFIX
+ + "/" + abfsStore.getUser()));
}
/**
@@ -706,8 +800,8 @@ public class AzureBlobFileSystem extends FileSystem
}
final String blobLocationHost = abfsStore.getAbfsConfiguration().getAzureBlockLocationHost();
- final String[] name = { blobLocationHost };
- final String[] host = { blobLocationHost };
+ final String[] name = {blobLocationHost};
+ final String[] host = {blobLocationHost};
long blockSize = file.getBlockSize();
if (blockSize <= 0) {
throw new IllegalArgumentException(
@@ -782,15 +876,14 @@ public class AzureBlobFileSystem extends FileSystem
}
});
}
- }
- finally {
+ } finally {
executorService.shutdownNow();
}
return true;
}
- /**
+ /**
* Set owner of a path (i.e. a file or a directory).
* The parameters owner and group cannot both be null.
*
@@ -820,9 +913,9 @@ public class AzureBlobFileSystem extends FileSystem
try {
abfsStore.setOwner(qualifiedPath,
- owner,
- group,
- tracingContext);
+ owner,
+ group,
+ tracingContext);
} catch (AzureBlobFileSystemException ex) {
checkException(path, ex);
}
@@ -839,7 +932,10 @@ public class AzureBlobFileSystem extends FileSystem
* @throws IllegalArgumentException If name is null or empty or if value is null
*/
@Override
- public void setXAttr(final Path path, final String name, final byte[] value, final EnumSet<XAttrSetFlag> flag)
+ public void setXAttr(final Path path,
+ final String name,
+ final byte[] value,
+ final EnumSet<XAttrSetFlag> flag)
throws IOException {
LOG.debug("AzureBlobFileSystem.setXAttr path: {}", path);
@@ -963,7 +1059,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"modifyAclEntries is only supported by storage accounts with the "
- + "hierarchical namespace enabled.");
+ + "hierarchical namespace enabled.");
}
if (aclSpec == null || aclSpec.isEmpty()) {
@@ -998,7 +1094,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"removeAclEntries is only supported by storage accounts with the "
- + "hierarchical namespace enabled.");
+ + "hierarchical namespace enabled.");
}
if (aclSpec == null || aclSpec.isEmpty()) {
@@ -1030,7 +1126,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"removeDefaultAcl is only supported by storage accounts with the "
- + "hierarchical namespace enabled.");
+ + "hierarchical namespace enabled.");
}
Path qualifiedPath = makeQualified(path);
@@ -1060,7 +1156,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"removeAcl is only supported by storage accounts with the "
- + "hierarchical namespace enabled.");
+ + "hierarchical namespace enabled.");
}
Path qualifiedPath = makeQualified(path);
@@ -1093,7 +1189,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"setAcl is only supported by storage accounts with the hierarchical "
- + "namespace enabled.");
+ + "namespace enabled.");
}
if (aclSpec == null || aclSpec.size() == 0) {
@@ -1125,7 +1221,7 @@ public class AzureBlobFileSystem extends FileSystem
if (!getIsNamespaceEnabled(tracingContext)) {
throw new UnsupportedOperationException(
"getAclStatus is only supported by storage account with the "
- + "hierarchical namespace enabled.");
+ + "hierarchical namespace enabled.");
}
Path qualifiedPath = makeQualified(path);
@@ -1235,7 +1331,7 @@ public class AzureBlobFileSystem extends FileSystem
private boolean fileSystemExists() throws IOException {
LOG.debug(
- "AzureBlobFileSystem.fileSystemExists uri: {}", uri);
+ "AzureBlobFileSystem.fileSystemExists uri: {}", uri);
try {
TracingContext tracingContext = new TracingContext(clientCorrelationId,
fileSystemId, FSOperationType.TEST_OP, tracingHeaderFormat, listener);
@@ -1526,8 +1622,9 @@ public class AzureBlobFileSystem extends FileSystem
case CommonPathCapabilities.FS_PERMISSIONS:
case CommonPathCapabilities.FS_APPEND:
case CommonPathCapabilities.ETAGS_AVAILABLE:
- case CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME:
return true;
+
+ case CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME:
case CommonPathCapabilities.FS_ACLS:
return getIsNamespaceEnabled(
new TracingContext(clientCorrelationId, fileSystemId,
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index d86a3d9..95035a1 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Listenable
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -855,7 +856,22 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
client.breakLease(getRelativePath(path), tracingContext);
}
- public void rename(final Path source, final Path destination, TracingContext tracingContext) throws
+ /**
+ * Rename a file or directory.
+ * If a source etag is passed in, the operation will attempt to recover
+ * from a missing source file by probing the destination for
+ * existence and comparing etags.
+ * @param source path to source file
+ * @param destination destination of rename.
+ * @param tracingContext trace context
+ * @param sourceEtag etag of source file. may be null or empty
+ * @throws AzureBlobFileSystemException failure, excluding any recovery from overload failures.
+ * @return true if recovery was needed and succeeded.
+ */
+ public boolean rename(final Path source,
+ final Path destination,
+ final TracingContext tracingContext,
+ final String sourceEtag) throws
AzureBlobFileSystemException {
final Instant startAggregate = abfsPerfTracker.getLatencyInstant();
long countAggregate = 0;
@@ -875,23 +891,29 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
String sourceRelativePath = getRelativePath(source);
String destinationRelativePath = getRelativePath(destination);
+ // was any operation recovered from?
+ boolean recovered = false;
do {
try (AbfsPerfInfo perfInfo = startTracking("rename", "renamePath")) {
- AbfsRestOperation op = client
- .renamePath(sourceRelativePath, destinationRelativePath,
- continuation, tracingContext);
+ final Pair<AbfsRestOperation, Boolean> pair =
+ client.renamePath(sourceRelativePath, destinationRelativePath,
+ continuation, tracingContext, sourceEtag);
+
+ AbfsRestOperation op = pair.getLeft();
perfInfo.registerResult(op.getResult());
continuation = op.getResult().getResponseHeader(HttpHeaderConfigurations.X_MS_CONTINUATION);
perfInfo.registerSuccess(true);
countAggregate++;
shouldContinue = continuation != null && !continuation.isEmpty();
-
+ // update the recovery flag.
+ recovered |= pair.getRight();
if (!shouldContinue) {
perfInfo.registerAggregates(startAggregate, countAggregate);
}
}
} while (shouldContinue);
+ return recovered;
}
public void delete(final Path path, final boolean recursive,
@@ -1909,7 +1931,7 @@ public class AzureBlobFileSystemStore implements Closeable, ListingSupport {
* @param result response to process.
* @return the quote-unwrapped etag.
*/
- private static String extractEtagHeader(AbfsHttpOperation result) {
+ public static String extractEtagHeader(AbfsHttpOperation result) {
String etag = result.getResponseHeader(HttpHeaderConfigurations.ETAG);
if (etag != null) {
// strip out any wrapper "" quotes which come back, for consistency with
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java
new file mode 100644
index 0000000..efba924
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AbfsManifestStoreOperations.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.time.Duration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem;
+
+/**
+ * Extension of StoreOperationsThroughFileSystem with ABFS awareness.
+ * Purely for use by jobs committing work through the manifest committer.
+ * The {@link AzureManifestCommitterFactory} will configure
+ * this as the binding to the FS.
+ *
+ * ADLS Gen2 stores support etag-recovery on renames, but not WASB
+ * stores.
+ */
+@InterfaceAudience.LimitedPrivate("mapreduce")
+@InterfaceStability.Unstable
+public class AbfsManifestStoreOperations extends
+ ManifestStoreOperationsThroughFileSystem {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AbfsManifestStoreOperations.class);
+
+ /**
+ * Classname, which can be declared in jpb configurations.
+ */
+ public static final String NAME = AbfsManifestStoreOperations.class.getName();
+
+ /**
+ * Resilient rename calls; only available on an ADLS Gen2 store.
+ * Will be null after binding if the FS isn't compatible.
+ */
+ private ResilientCommitByRename resilientCommitByRename;
+
+ @Override
+ public AzureBlobFileSystem getFileSystem() {
+ return (AzureBlobFileSystem) super.getFileSystem();
+ }
+
+ /**
+ * Bind to the store.
+ *
+ * @param filesystem FS.
+ * @param path path to work under
+ * @throws IOException binding problems.
+ */
+ @Override
+ public void bindToFileSystem(FileSystem filesystem, Path path) throws IOException {
+ if (!(filesystem instanceof AzureBlobFileSystem)) {
+ throw new PathIOException(path.toString(),
+ "Not an abfs filesystem: " + filesystem.getClass());
+ }
+ super.bindToFileSystem(filesystem, path);
+ try {
+ resilientCommitByRename = getFileSystem().createResilientCommitSupport(path);
+ LOG.debug("Bonded to filesystem with resilient commits under path {}", path);
+ } catch (UnsupportedOperationException e) {
+ LOG.debug("No resilient commit support under path {}", path);
+ }
+ }
+
+ @Override
+ public boolean storePreservesEtagsThroughRenames(final Path path) {
+ return true;
+ }
+
+ /**
+ * Resilient commits available on hierarchical stores.
+ * @return true if the FS can use etags on renames.
+ */
+ @Override
+ public boolean storeSupportsResilientCommit() {
+ return resilientCommitByRename != null;
+ }
+
+ /**
+ * Commit a file through an internal ABFS operation.
+ * If resilient commit is unavailable, invokes the superclass, which
+ * will raise an UnsupportedOperationException
+ * @param entry entry to commit
+ * @return the outcome
+ * @throws IOException any failure in resilient commit.
+ * @throws UnsupportedOperationException if not available.
+ */
+ @Override
+ public CommitFileResult commitFile(final FileEntry entry) throws IOException {
+
+ if (resilientCommitByRename != null) {
+ final Pair<Boolean, Duration> result =
+ resilientCommitByRename.commitSingleFileByRename(
+ entry.getSourcePath(),
+ entry.getDestPath(),
+ entry.getEtag());
+ return CommitFileResult.fromResilientCommit(result.getLeft(),
+ result.getRight());
+ } else {
+ return super.commitFile(entry);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java
new file mode 100644
index 0000000..b760fa7
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/AzureManifestCommitterFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterFactory;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+
+/**
+ * A Committer for the manifest committer which performs all bindings needed
+ * to work best with abfs.
+ * This includes, at a minimum, switching to the abfs-specific manifest store operations.
+ *
+ * This classname is referenced in configurations, so MUST NOT change.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class AzureManifestCommitterFactory extends ManifestCommitterFactory {
+
+ /**
+ * Classname, which can be declared in job configurations.
+ */
+ public static final String NAME = ManifestCommitterFactory.class.getName();
+
+ @Override
+ public ManifestCommitter createOutputCommitter(final Path outputPath,
+ final TaskAttemptContext context) throws IOException {
+ final Configuration conf = context.getConfiguration();
+ // use ABFS Store operations
+ conf.set(OPT_STORE_OPERATIONS_CLASS,
+ AbfsManifestStoreOperations.NAME);
+ return super.createOutputCommitter(outputPath, context);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java
new file mode 100644
index 0000000..2e91392
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/ResilientCommitByRename.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.Duration;
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+
+/**
+ * API exclusively for committing files.
+ *
+ * This is only for use by (@link {@link AbfsManifestStoreOperations},
+ * and is intended to be implemented by ABFS.
+ * To ensure that there is no need to add mapreduce JARs to the
+ * classpath just to work with ABFS, this interface
+ * MUST NOT refer to anything in the
+ * {@code org.apache.hadoop.mapreduce} package.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface ResilientCommitByRename extends IOStatisticsSource {
+
+ /**
+ * Rename source file to dest path *Exactly*; no subdirectory games here.
+ * if the method does not raise an exception,then
+ * the data at dest is the data which was at source.
+ *
+ * Requirements
+ *
+ * <pre>
+ * exists(FS, source) else raise FileNotFoundException
+ * source != dest else raise PathIOException
+ * not exists(FS, dest)
+ * isDir(FS, dest.getParent)
+ * </pre>
+ * <ol>
+ * <li>source != dest else raise PathIOException</li>
+ * <li>source must exist else raise FileNotFoundException</li>
+ * <li>source must exist and be a file</li>
+ * <li>dest must not exist; </li>
+ * <li>dest.getParent() must be a dir</li>
+ * <li>if sourceEtag is non-empty, it MAY be used to qualify/validate the rename.</li>
+ * </ol>
+ *
+ * The outcome of the operation is undefined if source is not a file, dest exists,
+ * dest.getParent() doesn't exist/is a file.
+ * That is: implementations SHOULD assume that the code calling this method has
+ * set up the destination directory tree and is only invoking this call on a file.
+ * Accordingly: <i>implementations MAY skip validation checks</i>
+ *
+ * Post Conditions on a successful operation:
+ * <pre>
+ * FS' where:
+ * not exists(FS', source)
+ * and exists(FS', dest)
+ * and data(FS', dest) == data (FS, source)
+ * </pre>
+ * This is exactly the same outcome as `FileSystem.rename()` when the same preconditions
+ * are met. This API call simply restricts the operation to file rename with strict
+ * conditions, (no need to be 'clever' about dest path calculation) and the ability
+ * to pass in etags, modtimes and file status values.
+ *
+ * @param source path to source file
+ * @param dest destination of rename.
+ * @param sourceEtag etag of source file. may be null or empty
+ * @return true if recovery was needed.
+ * @throws FileNotFoundException source file not found
+ * @throws PathIOException failure, including source and dest being the same path
+ * @throws IOException any other exception
+ */
+ Pair<Boolean, Duration> commitSingleFileByRename(
+ Path source,
+ Path dest,
+ @Nullable String sourceEtag) throws IOException;
+
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
new file mode 100644
index 0000000..3567377
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Support for manifest committer.
+ * Unless otherwise stated: classes are private.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index 12beb5a..9d3b2d5 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -220,6 +220,9 @@ public final class ConfigurationKeys {
/** Key for enabling the tracking of ABFS API latency and sending the latency numbers to the ABFS API service */
public static final String FS_AZURE_ABFS_LATENCY_TRACK = "fs.azure.abfs.latency.track";
+ /** Key for rate limit capacity, as used by IO operations which try to throttle themselves. */
+ public static final String FS_AZURE_ABFS_IO_RATE_LIMIT = "fs.azure.io.rate.limit";
+
public static String accountProperty(String property, String account) {
return property + "." + account;
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index f58c61e..63d62a3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -133,5 +133,10 @@ public final class FileSystemConfigurations {
public static final String DATA_BLOCKS_BUFFER_DEFAULT =
DATA_BLOCKS_BUFFER_DISK;
+ /**
+ * IO rate limit. Value: {@value}
+ */
+ public static final int RATE_LIMIT_DEFAULT = 10_000;
+
private FileSystemConfigurations() {}
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
index 69ef0d0..dacce9a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFact
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations;
import org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams;
@@ -67,6 +68,8 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader;
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.*;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_DELETE_CONSIDERED_IDEMPOTENT;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.SERVER_SIDE_ENCRYPTION_ALGORITHM;
@@ -478,8 +481,30 @@ public class AbfsClient implements Closeable {
return op;
}
- public AbfsRestOperation renamePath(String source, final String destination,
- final String continuation, TracingContext tracingContext)
+
+ /**
+ * Rename a file or directory.
+ * If a source etag is passed in, the operation will attempt to recover
+ * from a missing source file by probing the destination for
+ * existence and comparing etags.
+ * The second value in the result will be true to indicate that this
+ * took place.
+ * As rename recovery is only attempted if the source etag is non-empty,
+ * in normal rename operations rename recovery will never happen.
+ * @param source path to source file
+ * @param destination destination of rename.
+ * @param continuation continuation.
+ * @param tracingContext trace context
+ * @param sourceEtag etag of source file. may be null or empty
+ * @return pair of (the rename operation, flag indicating recovery took place)
+ * @throws AzureBlobFileSystemException failure, excluding any recovery from overload failures.
+ */
+ public Pair<AbfsRestOperation, Boolean> renamePath(
+ final String source,
+ final String destination,
+ final String continuation,
+ final TracingContext tracingContext,
+ final String sourceEtag)
throws AzureBlobFileSystemException {
final List<AbfsHttpHeader> requestHeaders = createDefaultHeaders();
@@ -505,9 +530,73 @@ public class AbfsClient implements Closeable {
HTTP_METHOD_PUT,
url,
requestHeaders);
- // no attempt at recovery using timestamps as it was not reliable.
- op.execute(tracingContext);
- return op;
+ try {
+ op.execute(tracingContext);
+ return Pair.of(op, false);
+ } catch (AzureBlobFileSystemException e) {
+ // If we have no HTTP response, throw the original exception.
+ if (!op.hasResult()) {
+ throw e;
+ }
+ boolean etagCheckSucceeded = renameIdempotencyCheckOp(
+ source,
+ sourceEtag, op, destination, tracingContext);
+ if (!etagCheckSucceeded) {
+ // idempotency did not return different result
+ // throw back the exception
+ throw e;
+ }
+ return Pair.of(op, true);
+ }
+ }
+
+ /**
+ * Check if the rename request failure is post a retry and if earlier rename
+ * request might have succeeded at back-end.
+ *
+ * If a source etag was passed in, and the error was 404, get the
+ * etag of any file at the destination.
+ * If it matches the source etag, then the rename is considered
+ * a success.
+ * Exceptions raised in the probe of the destination are swallowed,
+ * so that they do not interfere with the original rename failures.
+ * @param source source path
+ * @param op Rename request REST operation response with non-null HTTP response
+ * @param destination rename destination path
+ * @param sourceEtag etag of source file. may be null or empty
+ * @param tracingContext Tracks identifiers for request header
+ * @return true if the file was successfully copied
+ */
+ public boolean renameIdempotencyCheckOp(
+ final String source,
+ final String sourceEtag,
+ final AbfsRestOperation op,
+ final String destination,
+ TracingContext tracingContext) {
+ Preconditions.checkArgument(op.hasResult(), "Operations has null HTTP response");
+
+ if ((op.isARetriedRequest())
+ && (op.getResult().getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND)
+ && isNotEmpty(sourceEtag)) {
+
+ // Server has returned HTTP 404, which means rename source no longer
+ // exists. Check on destination status and if its etag matches
+ // that of the source, consider it to be a success.
+ LOG.debug("rename {} to {} failed, checking etag of destination",
+ source, destination);
+
+ try {
+ final AbfsRestOperation destStatusOp = getPathStatus(destination,
+ false, tracingContext);
+ final AbfsHttpOperation result = destStatusOp.getResult();
+
+ return result.getStatusCode() == HttpURLConnection.HTTP_OK
+ && sourceEtag.equals(extractEtagHeader(result));
+ } catch (AzureBlobFileSystemException ignored) {
+ // GetFileStatus on the destination failed, the rename did not take place
+ }
+ }
+ return false;
}
public AbfsRestOperation append(final String path, final byte[] buffer,
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
index bb1ec9e..e3adc59 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SimpleKeyProvider.java
@@ -48,9 +48,15 @@ public class SimpleKeyProvider implements KeyProvider {
// Validating the key.
validateStorageAccountKey(key);
} catch (IllegalAccessException | InvalidConfigurationValueException e) {
- throw new KeyProviderException("Failure to initialize configuration", e);
+ LOG.debug("Failure to retrieve storage account key for {}", accountName,
+ e);
+ throw new KeyProviderException("Failure to initialize configuration for "
+ + accountName
+ + " key =\"" + key + "\""
+ + ": " + e, e);
} catch(IOException ioe) {
- LOG.warn("Unable to get key from credential providers. {}", ioe);
+ LOG.warn("Unable to get key for {} from credential providers. {}",
+ accountName, ioe, ioe);
}
return key;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index a57e9bc..3bc8338 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -270,11 +270,12 @@ public abstract class AbstractAbfsIntegrationTest extends
// The SAS tests do not have permission to create a filesystem
// so first create temporary instance of the filesystem using SharedKey
// then re-use the filesystem it creates with SAS auth instead of SharedKey.
- AzureBlobFileSystem tempFs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig);
- ContractTestUtils.assertPathExists(tempFs, "This path should exist",
- new Path("/"));
- abfsConfig.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.name());
- usingFilesystemForSASTests = true;
+ try (AzureBlobFileSystem tempFs = (AzureBlobFileSystem) FileSystem.newInstance(rawConfig)){
+ ContractTestUtils.assertPathExists(tempFs, "This path should exist",
+ new Path("/"));
+ abfsConfig.set(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SAS.name());
+ usingFilesystemForSASTests = true;
+ }
}
public AzureBlobFileSystem getFileSystem() throws IOException {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
index b0e8244..5bd6eaf 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java
@@ -32,7 +32,10 @@ import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
+import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
+import org.apache.hadoop.fs.statistics.IOStatisticsSource;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.APPENDBLOB_MAX_WRITE_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE;
@@ -76,13 +79,19 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
new Random().nextBytes(b);
Path testPath = path(TEST_PATH);
- try (FSDataOutputStream stream = fs.create(testPath)) {
+ FSDataOutputStream stream = fs.create(testPath);
+ try {
stream.write(b);
+ } finally{
+ stream.close();
}
+ IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream);
final byte[] readBuffer = new byte[2 * bufferSize];
int result;
+ IOStatisticsSource statisticsSource = null;
try (FSDataInputStream inputStream = fs.open(testPath)) {
+ statisticsSource = inputStream;
((AbfsInputStream) inputStream.getWrappedStream()).registerListener(
new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(),
fs.getFileSystemId(), FSOperationType.READ, true, 0,
@@ -100,6 +109,8 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest {
inputStream.seek(0);
result = inputStream.read(readBuffer, 0, bufferSize);
}
+ IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, statisticsSource);
+
assertNotEquals("data read in final read()", -1, result);
assertArrayEquals(readBuffer, b);
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
index ea9fba6..965e02a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
@@ -401,7 +401,8 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
fs.create(new Path(src)).close();
AbfsRestOperation abfsHttpRestOperation = fs.getAbfsClient()
.renamePath(src, "/testABC" + "/abc.txt", null,
- getTestTracingContext(fs, false));
+ getTestTracingContext(fs, false), null)
+ .getLeft();
AbfsHttpOperation result = abfsHttpRestOperation.getResult();
String url = result.getMaskedUrl();
String encodedUrl = result.getMaskedEncodedUrl();
@@ -418,7 +419,7 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
intercept(IOException.class, "sig=XXXX",
() -> getFileSystem().getAbfsClient()
.renamePath("testABC/test.xt", "testABC/abc.txt", null,
- getTestTracingContext(getFileSystem(), false)));
+ getTestTracingContext(getFileSystem(), false), null));
}
@Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
index 0873b8e..a361581 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
@@ -526,7 +526,8 @@ public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest {
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
.renamePath(testFileName, newName, null,
- getTestTracingContext(fs, false));
+ getTestTracingContext(fs, false), null)
+ .getLeft();
assertCPKHeaders(abfsRestOperation, false);
assertNoCPKResponseHeadersPresent(abfsRestOperation);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java
new file mode 100644
index 0000000..8160cdc
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbfsCommitTestHelper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_STORE_OPERATIONS_CLASS;
+
+/**
+ * Helper methods for committer tests on ABFS.
+ */
+final class AbfsCommitTestHelper {
+ private AbfsCommitTestHelper() {
+ }
+
+ /**
+ * Prepare the test configuration.
+ * @param contractTestBinding test binding
+ * @return an extracted and patched configuration.
+ */
+ static Configuration prepareTestConfiguration(
+ ABFSContractTestBinding contractTestBinding) {
+ final Configuration conf =
+ contractTestBinding.getRawConfiguration();
+
+ // use ABFS Store operations
+ conf.set(OPT_STORE_OPERATIONS_CLASS,
+ AbfsManifestStoreOperations.NAME);
+
+ return conf;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java
new file mode 100644
index 0000000..5575205
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/AbstractAbfsClusterITest.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+import org.junit.AfterClass;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azure.integration.AzureTestConstants;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.AbstractManifestCommitterTest;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.util.DurationInfo;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled;
+import static org.apache.hadoop.io.IOUtils.closeStream;
+
+/**
+ * Tests which create a yarn minicluster.
+ * These are all considered scale tests; the probe for
+ * scale tests being enabled is executed before the cluster
+ * is set up to avoid wasting time on non-scale runs.
+ */
+public abstract class AbstractAbfsClusterITest extends
+ AbstractManifestCommitterTest {
+
+ public static final int NO_OF_NODEMANAGERS = 2;
+
+ private final ABFSContractTestBinding binding;
+
+
+ /**
+ * The static cluster binding with the lifecycle of this test; served
+ * through instance-level methods for sharing across methods in the
+ * suite.
+ */
+ @SuppressWarnings("StaticNonFinalField")
+ private static ClusterBinding clusterBinding;
+
+ protected AbstractAbfsClusterITest() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ protected int getTestTimeoutMillis() {
+ return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ requireScaleTestsEnabled();
+ if (getClusterBinding() == null) {
+ clusterBinding = demandCreateClusterBinding();
+ }
+ assertNotNull("cluster is not bound", getClusterBinding());
+ }
+
+ @AfterClass
+ public static void teardownClusters() throws IOException {
+ terminateCluster(clusterBinding);
+ clusterBinding = null;
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ /**
+ * This is the cluster binding which every subclass must create.
+ */
+ protected static final class ClusterBinding {
+
+ private String clusterName;
+
+ private final MiniMRYarnCluster yarn;
+
+ public ClusterBinding(
+ final String clusterName,
+ final MiniMRYarnCluster yarn) {
+ this.clusterName = clusterName;
+ this.yarn = requireNonNull(yarn);
+ }
+
+
+ /**
+ * Get the cluster FS, which will either be HDFS or the local FS.
+ * @return a filesystem.
+ * @throws IOException failure
+ */
+ public FileSystem getClusterFS() throws IOException {
+ return FileSystem.getLocal(yarn.getConfig());
+ }
+
+ public MiniMRYarnCluster getYarn() {
+ return yarn;
+ }
+
+ public Configuration getConf() {
+ return getYarn().getConfig();
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void terminate() {
+ closeStream(getYarn());
+ }
+ }
+
+ /**
+ * Create the cluster binding.
+ * The configuration will be patched by propagating down options
+ * from the maven build (S3Guard binding etc) and turning off unwanted
+ * YARN features.
+ *
+ * If an HDFS cluster is requested,
+ * the HDFS and YARN clusters will share the same configuration, so
+ * the HDFS cluster binding is implicitly propagated to YARN.
+ * If one is not requested, the local filesystem is used as the cluster FS.
+ * @param conf configuration to start with.
+ * @return the cluster binding.
+ * @throws IOException failure.
+ */
+ protected static ClusterBinding createCluster(
+ final JobConf conf) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG, "Creating YARN MiniCluster")) {
+ conf.setBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, false);
+ // create a unique cluster name based on the current time in millis.
+ String timestamp = LocalDateTime.now().format(
+ DateTimeFormatter.ofPattern("yyyy-MM-dd-HH.mm.ss.SS"));
+ String clusterName = "yarn-" + timestamp;
+ MiniMRYarnCluster yarnCluster =
+ new MiniMRYarnCluster(clusterName, NO_OF_NODEMANAGERS);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+ return new ClusterBinding(clusterName, yarnCluster);
+ }
+ }
+
+ /**
+ * Terminate the cluster if it is not null.
+ * @param cluster the cluster
+ */
+ protected static void terminateCluster(ClusterBinding cluster) {
+ if (cluster != null) {
+ cluster.terminate();
+ }
+ }
+
+ /**
+ * Get the cluster binding for this subclass.
+ * @return the cluster binding
+ */
+ protected ClusterBinding getClusterBinding() {
+ return clusterBinding;
+ }
+
+ protected MiniMRYarnCluster getYarn() {
+ return getClusterBinding().getYarn();
+ }
+
+
+ /**
+ * We stage work into a temporary directory rather than directly under
+ * the user's home directory, as that is often rejected by CI test
+ * runners.
+ */
+ @Rule
+ public final TemporaryFolder stagingFilesDir = new TemporaryFolder();
+
+
+ /**
+ * binding on demand rather than in a BeforeClass static method.
+ * Subclasses can override this to change the binding options.
+ * @return the cluster binding
+ */
+ protected ClusterBinding demandCreateClusterBinding() throws Exception {
+ return createCluster(new JobConf());
+ }
+
+ /**
+ * Create a job configuration.
+ * This creates a new job conf from the yarn
+ * cluster configuration then calls
+ * {@link #applyCustomConfigOptions(JobConf)} to allow it to be customized.
+ * @return the new job configuration.
+ * @throws IOException failure
+ */
+ protected JobConf newJobConf() throws IOException {
+ JobConf jobConf = new JobConf(getYarn().getConfig());
+ jobConf.addResource(getConfiguration());
+ applyCustomConfigOptions(jobConf);
+ return jobConf;
+ }
+
+ /**
+ * Patch the (job) configuration for this committer.
+ * @param jobConf configuration to patch
+ * @return a configuration which will run this configuration.
+ */
+ protected Configuration patchConfigurationForCommitter(
+ final Configuration jobConf) {
+ enableManifestCommitter(jobConf);
+ return jobConf;
+ }
+
+ /**
+ * Override point to let implementations tune the MR Job conf.
+ * @param jobConf configuration
+ */
+ protected void applyCustomConfigOptions(JobConf jobConf) throws IOException {
+
+ }
+
+
+ /**
+ * Assume that scale tests are enabled.
+ */
+ protected void requireScaleTestsEnabled() {
+ assumeScaleTestsEnabled(getConfiguration());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java
new file mode 100644
index 0000000..a597c35
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCleanupStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCleanupStage;
+
+/**
+ * Cleanup logic on ABFS.
+ */
+public class ITestAbfsCleanupStage extends TestCleanupStage {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsCleanupStage() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java
new file mode 100644
index 0000000..a0aaec8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCommitTaskStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCommitTaskStage;
+
+/**
+ * ABFS storage test of task committer.
+ */
+public class ITestAbfsCommitTaskStage extends TestCommitTaskStage {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsCommitTaskStage() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java
new file mode 100644
index 0000000..6621b80
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsCreateOutputDirectoriesStage.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestCreateOutputDirectoriesStage;
+
+/**
+ * ABFS storage test of directory creation.
+ */
+public class ITestAbfsCreateOutputDirectoriesStage extends TestCreateOutputDirectoriesStage {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsCreateOutputDirectoriesStage() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java
new file mode 100644
index 0000000..4e4c4f5
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsJobThroughManifestCommitter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.assertj.core.api.Assertions;
+import org.junit.FixMethodOrder;
+import org.junit.runners.MethodSorters;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestJobThroughManifestCommitter;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.TaskManifest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+
+/**
+ * Test the Manifest committer stages against ABFS.
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class ITestAbfsJobThroughManifestCommitter
+ extends TestJobThroughManifestCommitter {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsJobThroughManifestCommitter() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return enableManifestCommitter(prepareTestConfiguration(binding));
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+ @Override
+ protected boolean shouldDeleteTestRootAtEndOfTestRun() {
+ return true;
+ }
+
+ /**
+ * Add read of manifest and validate of output's etags.
+ * @param attemptId attempt ID
+ * @param files files which were created.
+ * @param manifest manifest
+ * @throws IOException failure
+ */
+ @Override
+ protected void validateTaskAttemptManifest(String attemptId,
+ List<Path> files,
+ TaskManifest manifest) throws IOException {
+ super.validateTaskAttemptManifest(attemptId, files, manifest);
+ final List<FileEntry> commit = manifest.getFilesToCommit();
+ final ManifestStoreOperations operations = getStoreOperations();
+ for (FileEntry entry : commit) {
+ Assertions.assertThat(entry.getEtag())
+ .describedAs("Etag of %s", entry)
+ .isNotEmpty();
+ final FileStatus sourceStatus = operations.getFileStatus(entry.getSourcePath());
+ final String etag = ManifestCommitterSupport.getEtag(sourceStatus);
+ Assertions.assertThat(etag)
+ .describedAs("Etag of %s", sourceStatus)
+ .isEqualTo(entry.getEtag());
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java
new file mode 100644
index 0000000..acd693e3
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsLoadManifestsStage.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestLoadManifestsStage;
+
+/**
+ * ABFS storage test of saving and loading a large number
+ * of manifests.
+ */
+public class ITestAbfsLoadManifestsStage extends TestLoadManifestsStage {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsLoadManifestsStage() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java
new file mode 100644
index 0000000..aac06f9
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestCommitProtocol.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestManifestCommitProtocol;
+
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+
+/**
+ * Test the Manifest protocol against ABFS.
+ */
+public class ITestAbfsManifestCommitProtocol extends
+ TestManifestCommitProtocol {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsManifestCommitProtocol() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return enableManifestCommitter(prepareTestConfiguration(binding));
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+
+ @Override
+ protected String suitename() {
+ return "ITestAbfsManifestCommitProtocol";
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java
new file mode 100644
index 0000000..922782d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsManifestStoreOperations.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.nio.charset.StandardCharsets;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.AbstractManifestCommitterTest;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.FileEntry;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.ETAGS_PRESERVED_IN_RENAME;
+import static org.apache.hadoop.fs.azurebfs.commit.AbfsCommitTestHelper.prepareTestConfiguration;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Test {@link AbfsManifestStoreOperations}.
+ * As this looks at etag handling through FS operations, it's actually testing how etags work
+ * in ABFS (preservation across renames) and in the client (are they consistent
+ * in LIST and HEAD calls).
+ *
+ * Skipped when tested against wasb-compatible stores.
+ */
+public class ITestAbfsManifestStoreOperations extends AbstractManifestCommitterTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestAbfsManifestStoreOperations.class);
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsManifestStoreOperations() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+
+ // skip tests on non-HNS stores
+ assumeTrue("Resilient rename not available",
+ getFileSystem().hasPathCapability(getContract().getTestPath(),
+ ETAGS_PRESERVED_IN_RENAME));
+
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return enableManifestCommitter(prepareTestConfiguration(binding));
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+ /**
+ * basic consistency across operations, as well as being non-empty.
+ */
+ @Test
+ public void testEtagConsistencyAcrossListAndHead() throws Throwable {
+ describe("Etag values must be non-empty and consistent across LIST and HEAD Calls.");
+ final Path path = methodPath();
+ final FileSystem fs = getFileSystem();
+ ContractTestUtils.touch(fs, path);
+ final ManifestStoreOperations operations = createManifestStoreOperations();
+ Assertions.assertThat(operations)
+ .describedAs("Store operations class loaded via Configuration")
+ .isInstanceOf(AbfsManifestStoreOperations.class);
+
+ final FileStatus st = operations.getFileStatus(path);
+ final String etag = operations.getEtag(st);
+ Assertions.assertThat(etag)
+ .describedAs("Etag of %s", st)
+ .isNotBlank();
+ LOG.info("etag of empty file is \"{}\"", etag);
+
+ final FileStatus[] statuses = fs.listStatus(path);
+ Assertions.assertThat(statuses)
+ .describedAs("List(%s)", path)
+ .hasSize(1);
+ final FileStatus lsStatus = statuses[0];
+ Assertions.assertThat(operations.getEtag(lsStatus))
+ .describedAs("etag of list status (%s) compared to HEAD value of %s", lsStatus, st)
+ .isEqualTo(etag);
+ }
+
+ @Test
+ public void testEtagsOfDifferentDataDifferent() throws Throwable {
+ describe("Verify that two different blocks of data written have different tags");
+
+ final Path path = methodPath();
+ final FileSystem fs = getFileSystem();
+ Path src = new Path(path, "src");
+
+ ContractTestUtils.createFile(fs, src, true,
+ "data1234".getBytes(StandardCharsets.UTF_8));
+ final ManifestStoreOperations operations = createManifestStoreOperations();
+ final FileStatus srcStatus = operations.getFileStatus(src);
+ final String srcTag = operations.getEtag(srcStatus);
+ LOG.info("etag of file 1 is \"{}\"", srcTag);
+
+ // now overwrite with data of same length
+ // (ensure that path or length aren't used exclusively as tag)
+ ContractTestUtils.createFile(fs, src, true,
+ "1234data".getBytes(StandardCharsets.UTF_8));
+
+ // validate
+ final String tag2 = operations.getEtag(operations.getFileStatus(src));
+ LOG.info("etag of file 2 is \"{}\"", tag2);
+
+ Assertions.assertThat(tag2)
+ .describedAs("etag of updated file")
+ .isNotEqualTo(srcTag);
+ }
+
+ @Test
+ public void testEtagConsistencyAcrossRename() throws Throwable {
+ describe("Verify that when a file is renamed, the etag remains unchanged");
+ final Path path = methodPath();
+ final FileSystem fs = getFileSystem();
+ Path src = new Path(path, "src");
+ Path dest = new Path(path, "dest");
+
+ ContractTestUtils.createFile(fs, src, true,
+ "sample data".getBytes(StandardCharsets.UTF_8));
+ final ManifestStoreOperations operations = createManifestStoreOperations();
+ final FileStatus srcStatus = operations.getFileStatus(src);
+ final String srcTag = operations.getEtag(srcStatus);
+ LOG.info("etag of short file is \"{}\"", srcTag);
+
+ Assertions.assertThat(srcTag)
+ .describedAs("Etag of %s", srcStatus)
+ .isNotBlank();
+
+ // rename
+ operations.commitFile(new FileEntry(src, dest, 0, srcTag));
+
+ // validate
+ FileStatus destStatus = operations.getFileStatus(dest);
+ final String destTag = operations.getEtag(destStatus);
+ Assertions.assertThat(destTag)
+ .describedAs("etag of list status (%s) compared to HEAD value of %s", destStatus, srcStatus)
+ .isEqualTo(srcTag);
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
new file mode 100644
index 0000000..5547d08
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure;
+
+/**
+ * Rename failure logic on ABFS.
+ * This will go through the resilient rename operation.
+ */
+public class ITestAbfsRenameStageFailure extends TestRenameStageFailure {
+
+ /**
+ * How many files to create.
+ */
+ private static final int FILES_TO_CREATE = 20;
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsRenameStageFailure() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+ @Override
+ protected boolean requireRenameResilience() {
+ return true;
+ }
+
+ @Override
+ protected int filesToCreate() {
+ return FILES_TO_CREATE;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java
new file mode 100644
index 0000000..d2fe9de
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTaskManifestFileIO.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
+import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestTaskManifestFileIO;
+
+/**
+ * Test Reading/writing manifest file through ABFS.
+ */
+public class ITestAbfsTaskManifestFileIO extends TestTaskManifestFileIO {
+
+ private final ABFSContractTestBinding binding;
+
+ public ITestAbfsTaskManifestFileIO() throws Exception {
+ binding = new ABFSContractTestBinding();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ binding.setup();
+ super.setup();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return AbfsCommitTestHelper.prepareTestConfiguration(binding);
+ }
+
+ @Override
+ protected AbstractFSContract createContract(final Configuration conf) {
+ return new AbfsFileSystemContract(conf, binding.isSecureMode());
+ }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java
new file mode 100644
index 0000000..4b21b83
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsTerasort.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.commit;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Consumer;
+
+import org.junit.Assume;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.examples.terasort.TeraGen;
+import org.apache.hadoop.examples.terasort.TeraSort;
+import org.apache.hadoop.examples.terasort.TeraSortConfigKeys;
+import org.apache.hadoop.examples.terasort.TeraValidate;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.statistics.IOStatisticsLogging;
+import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.util.functional.RemoteIterators;
+
+import static java.util.Optional.empty;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.loadSuccessFile;
+import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterTestSupport.validateSuccessFile;
+
+/**
+ * Runs Terasort against ABFS using the manifest committer.
+ * The tests run in sequence, so each operation is isolated.
+ * Scale test only (it is big and slow)
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+@SuppressWarnings({"StaticNonFinalField", "OptionalUsedAsFieldOrParameterType"})
+public class ITestAbfsTerasort extends AbstractAbfsClusterITest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestAbfsTerasort.class);
+
+ public static final int EXPECTED_PARTITION_COUNT = 10;
+
+ public static final int PARTITION_SAMPLE_SIZE = 1000;
+
+ public static final int ROW_COUNT = 1000;
+
+ /**
+ * This has to be common across all test methods.
+ */
+ private static final Path TERASORT_PATH = new Path("/ITestAbfsTerasort");
+
+ /**
+ * Duration tracker created in the first of the test cases and closed
+ * in {@link #test_140_teracomplete()}.
+ */
+ private static Optional<DurationInfo> terasortDuration = empty();
+
+ /**
+ * Tracker of which stages are completed and how long they took.
+ */
+ private static final Map<String, DurationInfo> COMPLETED_STAGES = new HashMap<>();
+
+ /**
+ * FileSystem statistics are collected from the _SUCCESS markers.
+ */
+ protected static final IOStatisticsSnapshot JOB_IOSTATS =
+ snapshotIOStatistics();
+
+ /** Base path for all the terasort input and output paths. */
+ private Path terasortPath;
+
+ /** Input (teragen) path. */
+ private Path sortInput;
+
+ /** Path where sorted data goes. */
+ private Path sortOutput;
+
+ /** Path for validated job's output. */
+ private Path sortValidate;
+
+ public ITestAbfsTerasort() throws Exception {
+ }
+
+
+ @Override
+ public void setup() throws Exception {
+ // superclass calls requireScaleTestsEnabled();
+ super.setup();
+ prepareToTerasort();
+ }
+
+ /**
+ * Set up the job conf with the options for terasort chosen by the scale
+ * options.
+ * @param conf configuration
+ */
+ @Override
+ protected void applyCustomConfigOptions(JobConf conf) {
+ // small sample size for faster runs
+ conf.setInt(TeraSortConfigKeys.SAMPLE_SIZE.key(),
+ getSampleSizeForEachPartition());
+ conf.setInt(TeraSortConfigKeys.NUM_PARTITIONS.key(),
+ getExpectedPartitionCount());
+ conf.setBoolean(
+ TeraSortConfigKeys.USE_SIMPLE_PARTITIONER.key(),
+ false);
+ }
+
+ private int getExpectedPartitionCount() {
+ return EXPECTED_PARTITION_COUNT;
+ }
+
+ private int getSampleSizeForEachPartition() {
+ return PARTITION_SAMPLE_SIZE;
+ }
+
+ protected int getRowCount() {
+ return ROW_COUNT;
+ }
+
+ /**
+ * Set up the terasort by initializing paths variables
+ * The paths used must be unique across parameterized runs but
+ * common across all test cases in a single parameterized run.
+ */
+ private void prepareToTerasort() {
+ terasortPath = getFileSystem().makeQualified(TERASORT_PATH);
+ sortInput = new Path(terasortPath, "sortin");
+ sortOutput = new Path(terasortPath, "sortout");
+ sortValidate = new Path(terasortPath, "validate");
+ }
+
+ /**
+ * Declare that a stage has completed.
+ * @param stage stage name/key in the map
+ * @param d duration.
+ */
+ private static void completedStage(final String stage,
+ final DurationInfo d) {
+ COMPLETED_STAGES.put(stage, d);
+ }
+
+ /**
+ * Declare a stage which is required for this test case.
+ * @param stage stage name
+ */
+ private static void requireStage(final String stage) {
+ Assume.assumeTrue(
+ "Required stage was not completed: " + stage,
+ COMPLETED_STAGES.get(stage) != null);
+ }
+
+ /**
+ * Execute a single stage in the terasort.
+ * Updates the completed stages map with the stage duration -if successful.
+ * @param stage Stage name for the stages map.
+ * @param jobConf job conf
+ * @param dest destination directory -the _SUCCESS file will be expected here.
+ * @param tool tool to run.
+ * @param args args for the tool.
+ * @param minimumFileCount minimum number of files to have been created
+ * @throws Exception any failure
+ */
+ private void executeStage(
+ final String stage,
+ final JobConf jobConf,
+ final Path dest,
+ final Tool tool,
+ final String[] args,
+ final int minimumFileCount) throws Exception {
+ int result;
+
+ // the duration info is created outside a try-with-resources
+ // clause as it is used later.
+ DurationInfo d = new DurationInfo(LOG, stage);
+ try {
+ result = ToolRunner.run(jobConf, tool, args);
+ } finally {
+ d.close();
+ }
+ dumpOutputTree(dest);
+ assertEquals(stage
+ + "(" + StringUtils.join(", ", args) + ")"
+ + " failed", 0, result);
+ final ManifestSuccessData successFile = validateSuccessFile(getFileSystem(), dest,
+ minimumFileCount, "");
+ JOB_IOSTATS.aggregate(successFile.getIOStatistics());
+
+ completedStage(stage, d);
+ }
+
+ /**
+ * Set up terasort by cleaning out the destination, and note the initial
+ * time before any of the jobs are executed.
+ *
+ * This is executed first <i>for each parameterized run</i>.
+ * It is where all variables which need to be reset for each run need
+ * to be reset.
+ */
+ @Test
+ public void test_100_terasort_setup() throws Throwable {
+ describe("Setting up for a terasort");
+
+ getFileSystem().delete(terasortPath, true);
+ terasortDuration = Optional.of(new DurationInfo(LOG, false, "Terasort"));
+ }
+
+ @Test
+ public void test_110_teragen() throws Throwable {
+ describe("Teragen to %s", sortInput);
+ getFileSystem().delete(sortInput, true);
+
+ JobConf jobConf = newJobConf();
+ patchConfigurationForCommitter(jobConf);
+ executeStage("teragen",
+ jobConf,
+ sortInput,
+ new TeraGen(),
+ new String[]{Integer.toString(getRowCount()), sortInput.toString()},
+ 1);
+ }
+
+
+ @Test
+ public void test_120_terasort() throws Throwable {
+ describe("Terasort from %s to %s", sortInput, sortOutput);
+ requireStage("teragen");
+ getFileSystem().delete(sortOutput, true);
+
+ loadSuccessFile(getFileSystem(), sortInput);
+ JobConf jobConf = newJobConf();
+ patchConfigurationForCommitter(jobConf);
+ executeStage("terasort",
+ jobConf,
+ sortOutput,
+ new TeraSort(),
+ new String[]{sortInput.toString(), sortOutput.toString()},
+ 1);
+ }
+
+ @Test
+ public void test_130_teravalidate() throws Throwable {
+ describe("TeraValidate from %s to %s", sortOutput, sortValidate);
+ requireStage("terasort");
+ getFileSystem().delete(sortValidate, true);
+ loadSuccessFile(getFileSystem(), sortOutput);
+ JobConf jobConf = newJobConf();
+ patchConfigurationForCommitter(jobConf);
+ executeStage("teravalidate",
+ jobConf,
+ sortValidate,
+ new TeraValidate(),
+ new String[]{sortOutput.toString(), sortValidate.toString()},
+ 1);
+ }
+
+ /**
+ * Print the results, and save to the base dir as a CSV file.
+ * Why there? Makes it easy to list and compare.
+ */
+ @Test
+ public void test_140_teracomplete() throws Throwable {
+ terasortDuration.ifPresent(d -> {
+ d.close();
+ completedStage("overall", d);
+ });
+
+ // IO Statistics
+ IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, JOB_IOSTATS);
+
+ // and the summary
+ final StringBuilder results = new StringBuilder();
+ results.append("\"Operation\"\t\"Duration\"\n");
+
+ // this is how you dynamically create a function in a method
+ // for use afterwards.
+ // Works because there's no IOEs being raised in this sequence.
+ Consumer<String> stage = (s) -> {
+ DurationInfo duration = COMPLETED_STAGES.get(s);
+ results.append(String.format("\"%s\"\t\"%s\"\n",
+ s,
+ duration == null ? "" : duration));
+ };
+
+ stage.accept("teragen");
+ stage.accept("terasort");
+ stage.accept("teravalidate");
+ stage.accept("overall");
+ String text = results.toString();
+ File resultsFile = File.createTempFile("results", ".csv");
+ FileUtils.write(resultsFile, text, StandardCharsets.UTF_8);
+ LOG.info("Results are in {}\n{}", resultsFile, text);
+ }
+
+ /**
+ * Reset the duration so if two committer tests are run sequentially.
+ * Without this the total execution time is reported as from the start of
+ * the first test suite to the end of the second.
+ */
+ @Test
+ public void test_150_teracleanup() throws Throwable {
+ terasortDuration = Optional.empty();
+ }
+
+ @Test
+ public void test_200_directory_deletion() throws Throwable {
+ getFileSystem().delete(terasortPath, true);
+ }
+
+ /**
+ * Dump the files under a path -but not fail if the path is not present.,
+ * @param path path to dump
+ * @throws Exception any failure.
+ */
+ protected void dumpOutputTree(Path path) throws Exception {
+ LOG.info("Files under output directory {}", path);
+ try {
+ RemoteIterators.foreach(getFileSystem().listFiles(path, true),
+ (status) -> LOG.info("{}", status));
+ } catch (FileNotFoundException e) {
+ LOG.info("Output directory {} not found", path);
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
new file mode 100644
index 0000000..3d49d62
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit and integration tests for the manifest committer.
+ * JSON job reports will be saved to
+ * {@code target/reports}
+ */
+package org.apache.hadoop.fs.azurebfs.commit;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
index 62bcca1..1319ea4 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/AbfsFileSystemContract.java
@@ -34,7 +34,7 @@ public class AbfsFileSystemContract extends AbstractBondedFSContract {
public static final String CONTRACT_XML = "abfs.xml";
private final boolean isSecure;
- protected AbfsFileSystemContract(final Configuration conf, boolean secure) {
+ public AbfsFileSystemContract(final Configuration conf, boolean secure) {
super(conf);
//insert the base features
addConfResource(CONTRACT_XML);
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml b/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml
new file mode 100644
index 0000000..7d2d11c
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/resources/core-site.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <include xmlns="http://www.w3.org/2001/XInclude" href="azure-test.xml">
+ <fallback/>
+ </include>
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org