You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2021/05/25 09:26:19 UTC
[hadoop] branch trunk updated (996d31f -> 832a3c6)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 996d31f HADOOP-17721. Replace Guava Sets usage by Hadoop's own Sets in hadoop-yarn-project (#3033)
add 832a3c6 HADOOP-17511. Add audit/telemetry logging to S3A connector (#2807)
No new revisions were added by this update.
Summary of changes:
.../org/apache/hadoop/fs/audit/AuditConstants.java | 108 ++
.../hadoop/fs/audit/AuditStatisticNames.java | 42 +-
.../apache/hadoop/fs/audit/CommonAuditContext.java | 288 ++++
.../org/apache/hadoop/fs/audit/package-info.java | 29 +-
.../java/org/apache/hadoop/fs/shell/Command.java | 3 +
.../java/org/apache/hadoop/fs/shell/PathData.java | 18 +-
.../hadoop/fs/statistics/StoreStatisticNames.java | 10 +
.../fs/statistics/impl/EmptyIOStatisticsStore.java | 182 +++
.../fs/statistics/impl/IOStatisticsBinding.java | 42 +
.../apache/hadoop/fs/store}/LogExactlyOnce.java | 12 +-
.../fs/store/audit/ActiveThreadSpanSource.java | 34 +-
.../hadoop/fs/store/audit/AuditEntryPoint.java | 55 +
.../apache/hadoop/fs/store/audit/AuditSpan.java | 109 ++
.../hadoop/fs/store/audit/AuditSpanSource.java | 50 +
.../hadoop/fs/store/audit/AuditingFunctions.java | 121 ++
.../fs/store/audit/HttpReferrerAuditHeader.java | 503 +++++++
.../apache/hadoop/fs/store/audit/package-info.java | 29 +-
.../hadoop/service/launcher/ServiceLauncher.java | 2 +
.../java/org/apache/hadoop/util/ToolRunner.java | 5 +
.../hadoop/util/functional/RemoteIterators.java | 34 +-
.../hadoop/fs/audit/TestCommonAuditContext.java | 161 +++
.../hadoop/fs/contract/ContractTestUtils.java | 10 +-
.../fs/s3a/CredentialInitializationException.java | 9 +
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 38 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 131 +-
.../org/apache/hadoop/fs/s3a/MultipartUtils.java | 110 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1382 ++++++++++++--------
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 45 +-
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 72 +-
.../apache/hadoop/fs/s3a/S3ALocatedFileStatus.java | 1 +
.../org/apache/hadoop/fs/s3a/S3AReadOpContext.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 26 +-
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 5 +
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 72 +-
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 290 ++--
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 18 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 296 +++++
.../LogExactlyOnce.java => api/package-info.java} | 32 +-
.../fs/s3a/audit/AWSAuditEventCallbacks.java | 160 +++
.../hadoop/fs/s3a/audit/AWSRequestAnalyzer.java | 296 +++++
.../AuditFailureException.java} | 29 +-
.../hadoop/fs/s3a/audit/AuditIntegration.java | 137 ++
.../hadoop/fs/s3a/audit/AuditManagerS3A.java | 92 ++
.../AuditSpanS3A.java} | 25 +-
.../hadoop/fs/s3a/audit/OperationAuditor.java | 73 ++
.../fs/s3a/audit/OperationAuditorOptions.java | 74 ++
.../hadoop/fs/s3a/audit/S3AAuditConstants.java | 104 ++
.../apache/hadoop/fs/s3a/audit/S3LogParser.java | 309 +++++
.../org/apache/hadoop/fs/s3a/audit/S3LogVerbs.java | 51 +
.../fs/s3a/audit/impl/AbstractAuditSpanImpl.java | 99 ++
.../s3a/audit/impl/AbstractOperationAuditor.java | 121 ++
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 773 +++++++++++
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 435 ++++++
.../fs/s3a/audit/impl/NoopAuditManagerS3A.java | 185 +++
.../hadoop/fs/s3a/audit/impl/NoopAuditor.java | 99 ++
.../apache/hadoop/fs/s3a/audit/impl/NoopSpan.java | 109 ++
.../impl/S3AInternalAuditConstants.java} | 37 +-
.../impl/package-info.java} | 28 +-
.../apache/hadoop/fs/s3a/audit/package-info.java | 41 +
.../hadoop/fs/s3a/commit/AbstractS3ACommitter.java | 66 +-
.../hadoop/fs/s3a/commit/CommitOperations.java | 31 +-
.../apache/hadoop/fs/s3a/commit/CommitUtils.java | 24 +
.../fs/s3a/commit/InternalCommitterConstants.java | 6 +
.../fs/s3a/commit/MagicCommitIntegration.java | 15 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 2 +-
.../fs/s3a/commit/staging/StagingCommitter.java | 2 +
.../hadoop/fs/s3a/impl/AbstractStoreOperation.java | 40 +-
.../hadoop/fs/s3a/impl/CallableSupplier.java | 52 +-
.../hadoop/fs/s3a/impl/ChangeDetectionPolicy.java | 1 +
.../hadoop/fs/s3a/impl/ContextAccessors.java | 21 +-
.../apache/hadoop/fs/s3a/impl/DeleteOperation.java | 17 +-
.../fs/s3a/impl/ExecutingStoreOperation.java | 40 +-
.../fs/s3a/impl/GetContentSummaryOperation.java | 208 +++
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 44 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 10 +
.../fs/s3a/impl/ListingOperationCallbacks.java | 26 +-
.../apache/hadoop/fs/s3a/impl/MkdirOperation.java | 184 +++
.../apache/hadoop/fs/s3a/impl/RenameOperation.java | 22 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 695 ++++++++++
.../apache/hadoop/fs/s3a/impl/StoreContext.java | 60 +-
.../hadoop/fs/s3a/impl/StoreContextBuilder.java | 18 +-
.../fs/s3a/s3guard/DumpS3GuardDynamoTable.java | 32 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 4 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 57 +-
.../apache/hadoop/fs/s3a/select/SelectBinding.java | 29 +-
.../impl/ForwardingIOStatisticsStore.java | 186 +++
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 9 +-
.../src/site/markdown/tools/hadoop-aws/auditing.md | 389 ++++++
.../tools/hadoop-aws/auditing_architecture.md | 323 +++++
.../src/site/markdown/tools/hadoop-aws/index.md | 2 +
.../tools/hadoop-aws/audit-architecture.png | Bin 0 -> 55701 bytes
.../fs/contract/s3a/ITestS3AContractSeek.java | 2 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 46 +
.../hadoop/fs/s3a/ITestS3AEmptyDirectory.java | 7 +-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 13 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 23 +-
.../hadoop/fs/s3a/ITestS3AMultipartUtils.java | 4 +-
.../hadoop/fs/s3a/ITestS3ARemoteFileChanged.java | 2 +-
.../hadoop/fs/s3a/ITestS3GuardEmptyDirs.java | 45 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 44 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 57 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 46 -
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 11 +-
.../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 213 +++
.../hadoop/fs/s3a/audit/AccessCheckingAuditor.java | 53 +
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 123 ++
.../fs/s3a/audit/ITestAuditAccessChecks.java | 201 +++
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 135 ++
.../fs/s3a/audit/SimpleAWSRequestHandler.java | 54 +
.../hadoop/fs/s3a/audit/TestAuditIntegration.java | 216 +++
.../fs/s3a/audit/TestAuditSpanLifecycle.java | 133 ++
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 323 +++++
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 203 +++
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 3 +-
.../hadoop/fs/s3a/commit/AbstractCommitITest.java | 12 +-
.../fs/s3a/commit/AbstractITCommitProtocol.java | 3 +-
.../fs/s3a/commit/ITestCommitOperations.java | 3 +-
.../apache/hadoop/fs/s3a/impl/ITestXAttrCost.java | 3 +-
.../hadoop/fs/s3a/impl/TestHeaderProcessing.java | 27 +-
.../fs/s3a/impl/TestPartialDeleteFailures.java | 16 +-
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 185 +++
.../fs/s3a/performance/AbstractS3ACostTest.java | 33 +-
.../performance/ITestDirectoryMarkerListing.java | 13 +-
.../s3a/performance/ITestS3AMiscOperationCost.java | 143 ++
.../fs/s3a/performance/ITestS3AMkdirCost.java | 185 +++
.../fs/s3a/performance/OperationCostValidator.java | 3 +-
.../hadoop/fs/s3a/s3guard/ITestS3GuardFsck.java | 2 -
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 47 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 3 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 137 +-
.../s3a/test/MinimalListingOperationCallbacks.java | 10 +-
.../hadoop-aws/src/test/resources/core-site.xml | 13 +
.../hadoop-aws/src/test/resources/log4j.properties | 6 +
134 files changed, 11771 insertions(+), 1326 deletions(-)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/LogExactlyOnce.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditStatisticNames.java (54%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/LogExactlyOnce.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/package-info.java (62%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsStore.java
copy {hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store}/LogExactlyOnce.java (81%)
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/LogExactlyOnce.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/ActiveThreadSpanSource.java (57%)
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditEntryPoint.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditSpan.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditSpanSource.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/HttpReferrerAuditHeader.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/LogExactlyOnce.java => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/package-info.java (62%)
create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/audit/TestCommonAuditContext.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/LogExactlyOnce.java => api/package-info.java} (57%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSAuditEventCallbacks.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AWSRequestAnalyzer.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/LogExactlyOnce.java => audit/AuditFailureException.java} (55%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditIntegration.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/AuditManagerS3A.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/LogExactlyOnce.java => audit/AuditSpanS3A.java} (63%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/OperationAuditorOptions.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3AAuditConstants.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3LogParser.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/S3LogVerbs.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/AbstractAuditSpanImpl.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/AbstractOperationAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/ActiveAuditManagerS3A.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditManagerS3A.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/NoopSpan.java
copy hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/AbstractStoreOperation.java => audit/impl/S3AInternalAuditConstants.java} (53%)
rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{impl/LogExactlyOnce.java => audit/impl/package-info.java} (62%)
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/package-info.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/GetContentSummaryOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java
create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/ForwardingIOStatisticsStore.java
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md
create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing_architecture.md
create mode 100644 hadoop-tools/hadoop-aws/src/site/resources/tools/hadoop-aws/audit-architecture.png
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AccessCheckingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AuditTestSupport.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditManager.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/SimpleAWSRequestHandler.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditIntegration.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestAuditSpanLifecycle.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestLoggingAuditor.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMiscOperationCost.java
create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3AMkdirCost.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org