You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2026/06/02 19:45:18 UTC
(hadoop-cloudstore) branch main updated (9ca1985 -> 56cbe4a)
This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-cloudstore.git
discard 9ca1985 HADOOP-19880. Cloudstore: cloud storage tools for Hadoop
new 56cbe4a HADOOP-19880. Cloudstore: cloud storage tools for Hadoop
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9ca1985)
\
N -- N -- N refs/heads/main (56cbe4a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BUILDING.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
(hadoop-cloudstore) 01/01: HADOOP-19880. Cloudstore: cloud storage tools for Hadoop
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-cloudstore.git
commit 56cbe4a2c68f3cbcf2a98bb4f9aa52119912e79b
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Jun 2 20:32:38 2026 +0100
HADOOP-19880. Cloudstore: cloud storage tools for Hadoop
This is the source tree of 1.5-SNAPSHOT of the cloudstore
cloud diagnostics and utility tools library
This commit does not contain secrets
Contributed by Steve Loughran
With some tests and GHA work by Claude Code
---
.asf.yaml | 80 +
.github/dependabot.yml | 65 +
.github/pull_request_template.md | 26 +
.github/workflows/codeql.yml | 79 +
.github/workflows/dependency-review.yml | 59 +
.github/workflows/maven.yml | 95 +
.github/workflows/scorecard.yml | 84 +
.github/workflows/zizmor.yml | 69 +
.gitignore | 55 +
AGENTS.md | 199 +++
BUILDING.md | 232 +++
CLAUDE.md | 19 +
CONTRIBUTING.md | 66 +
GEMINI.md | 19 +
LICENSE | 201 +++
LICENSE-binary | 202 +++
NOTICE | 5 +
NOTICE-binary | 5 +
README.md | 355 ++++
SECURITY.md | 194 ++
dev-support/bump-version.sh | 90 +
dev-support/check-doc-versions.sh | 70 +
dev-support/checkstyle-noframes-sorted.xsl | 194 ++
dev-support/checkstyle-suppressions.xml | 45 +
dev-support/checkstyle.xml | 231 +++
dev-support/eclipse-formatter.xml | 337 ++++
dev-support/license-header.txt | 17 +
dev-support/update-site-docs.sh | 95 +
osv-scanner.toml | 79 +
pom.xml | 890 +++++++++
src/main/avro/AvroDataSchema.avsc | 50 +
.../java/org/apache/hadoop/fs/gs/GsCredDiag.java | 130 ++
.../java/org/apache/hadoop/fs/gs/PemReader.java | 183 ++
.../java/org/apache/hadoop/fs/gs/package-info.java | 19 +
.../apache/hadoop/fs/s3a/sdk/BucketMetadata.java | 76 +
.../hadoop/fs/s3a/sdk/BulkDeleteCommand.java | 152 ++
.../apache/hadoop/fs/s3a/sdk/CsvVersionWriter.java | 92 +
.../org/apache/hadoop/fs/s3a/sdk/DeleteObject.java | 69 +
.../org/apache/hadoop/fs/s3a/sdk/IamPolicy.java | 69 +
.../apache/hadoop/fs/s3a/sdk/InternalAccess.java | 45 +
.../fs/s3a/sdk/ListAndProcessVersionedObjects.java | 208 +++
.../apache/hadoop/fs/s3a/sdk/ListMultiparts.java | 174 ++
.../org/apache/hadoop/fs/s3a/sdk/ListObjects.java | 235 +++
.../org/apache/hadoop/fs/s3a/sdk/ListVersions.java | 160 ++
.../org/apache/hadoop/fs/s3a/sdk/MkBucket.java | 85 +
.../hadoop/fs/s3a/sdk/MultipartProcessor.java | 93 +
.../hadoop/fs/s3a/sdk/ObjectListingIterator.java | 161 ++
.../java/org/apache/hadoop/fs/s3a/sdk/Regions.java | 107 ++
.../apache/hadoop/fs/s3a/sdk/RestoreObject.java | 80 +
.../apache/hadoop/fs/s3a/sdk/S3ListingSupport.java | 137 ++
.../org/apache/hadoop/fs/s3a/sdk/SessionKeys.java | 244 +++
.../apache/hadoop/fs/s3a/sdk/SummaryProcessor.java | 33 +
.../org/apache/hadoop/fs/s3a/sdk/Undelete.java | 201 +++
.../hadoop/fs/s3a/sdk/VersionedFileCopier.java | 96 +
.../org/apache/hadoop/fs/s3a/sdk/package-info.java | 19 +
.../org/apache/hadoop/fs/store/Cloudstore.java | 228 +++
.../apache/hadoop/fs/store/CommonParameters.java | 95 +
.../org/apache/hadoop/fs/store/MinMeanMax.java | 74 +
.../apache/hadoop/fs/store/StoreDiagConstants.java | 27 +
.../apache/hadoop/fs/store/StoreDurationInfo.java | 199 +++
.../apache/hadoop/fs/store/StoreEntryPoint.java | 885 +++++++++
.../org/apache/hadoop/fs/store/StoreUtils.java | 307 ++++
.../hadoop/fs/store/audit/AuditLogProcessor.java | 434 +++++
.../apache/hadoop/fs/store/audit/AuditTool.java | 146 ++
.../apache/hadoop/fs/store/audit/package-info.java | 20 +
.../apache/hadoop/fs/store/commands/Bandwidth.java | 486 +++++
.../hadoop/fs/store/commands/BucketState.java | 93 +
.../apache/hadoop/fs/store/commands/Command.java | 37 +
.../hadoop/fs/store/commands/CommitterInfo.java | 91 +
.../apache/hadoop/fs/store/commands/Constval.java | 124 ++
.../apache/hadoop/fs/store/commands/EnvEntry.java | 77 +
.../hadoop/fs/store/commands/EtagCommand.java | 102 ++
.../hadoop/fs/store/commands/ExtendedDu.java | 297 ++++
.../hadoop/fs/store/commands/FetchTokens.java | 169 ++
.../apache/hadoop/fs/store/commands/ListFiles.java | 133 ++
.../apache/hadoop/fs/store/commands/LocalHost.java | 73 +
.../hadoop/fs/store/commands/LocateFiles.java | 119 ++
.../hadoop/fs/store/commands/PathCapability.java | 81 +
.../hadoop/fs/store/commands/PrintStatus.java | 95 +
.../org/apache/hadoop/fs/store/commands/Put.java | 129 ++
.../apache/hadoop/fs/store/commands/TLSInfo.java | 190 ++
.../hadoop/fs/store/commands/package-info.java | 19 +
.../hadoop/fs/store/diag/ADLDiagnosticsInfo.java | 90 +
.../hadoop/fs/store/diag/AbfsDiagnosticsInfo.java | 835 +++++++++
.../hadoop/fs/store/diag/CapabilityKeys.java | 326 ++++
.../org/apache/hadoop/fs/store/diag/DiagUtils.java | 139 ++
.../fs/store/diag/DiagnosticsEntryPoint.java | 406 +++++
.../apache/hadoop/fs/store/diag/EndpointProbe.java | 122 ++
.../hadoop/fs/store/diag/GCSDiagnosticsInfo.java | 222 +++
.../hadoop/fs/store/diag/HDFSDiagnosticsInfo.java | 182 ++
.../apache/hadoop/fs/store/diag/OptionSets.java | 278 +++
.../org/apache/hadoop/fs/store/diag/Printout.java | 138 ++
.../hadoop/fs/store/diag/S3ADiagnosticsInfo.java | 1331 ++++++++++++++
.../hadoop/fs/store/diag/StdoutPrintout.java | 197 ++
.../org/apache/hadoop/fs/store/diag/StoreDiag.java | 1234 +++++++++++++
.../hadoop/fs/store/diag/StoreDiagException.java | 48 +
.../hadoop/fs/store/diag/StoreDiagnosticsInfo.java | 746 ++++++++
.../fs/store/diag/StringBuilderPrintout.java | 89 +
.../apache/hadoop/fs/store/diag/TeePrintout.java | 138 ++
.../fs/store/diag/TemplateDiagnosticsInfo.java | 79 +
.../hadoop/fs/store/diag/WasbDiagnosticsInfo.java | 92 +
.../apache/hadoop/fs/store/diag/package-info.java | 19 +
.../fs/store/logging/IOStatisticsIntegration.java | 35 +
.../hadoop/fs/store/logging/Log4JController.java | 33 +
.../apache/hadoop/fs/store/logging/LogControl.java | 51 +
.../fs/store/logging/LogControllerFactory.java | 54 +
.../hadoop/fs/store/logging/package-info.java | 19 +
.../org/apache/hadoop/fs/store/package-info.java | 22 +
.../org/apache/hadoop/fs/store/s3a/S3ASupport.java | 155 ++
.../apache/hadoop/fs/store/s3a/package-info.java | 19 +
.../apache/hadoop/fs/tools/cloudup/Cloudup.java | 829 +++++++++
.../apache/hadoop/fs/tools/cloudup/NanoTimer.java | 167 ++
.../hadoop/fs/tools/cloudup/UploadEntry.java | 237 +++
.../hadoop/fs/tools/cloudup/package-info.java | 83 +
.../hadoop/fs/tools/csv/CsvWriterWithCRC.java | 93 +
.../java/org/apache/hadoop/fs/tools/csv/MkCSV.java | 230 +++
.../hadoop/fs/tools/csv/SimpleCsvWriter.java | 244 +++
.../apache/hadoop/fs/tools/csv/package-info.java | 20 +
src/main/resources/LICENSE | 178 ++
src/main/resources/META-INF/MANIFEST.MF | 2 +
src/site/markdown/auditlogs.md | 291 +++
src/site/markdown/bandwidth.md | 378 ++++
src/site/markdown/bucketmetadata.md | 103 ++
src/site/markdown/bulkdelete.md | 195 ++
src/site/markdown/cloudup.md | 435 +++++
src/site/markdown/committerinfo.md | 97 +
src/site/markdown/constval.md | 31 +
src/site/markdown/etag.md | 134 ++
src/site/markdown/fetchdt.md | 88 +
src/site/markdown/index.md | 47 +
src/site/markdown/locatefiles.md | 123 ++
src/site/markdown/mkbucket.md | 31 +
src/site/markdown/mkcsv.md | 102 ++
src/site/markdown/put.md | 38 +
src/site/markdown/sdk.md | 221 +++
src/site/markdown/sessionkeys.md | 120 ++
src/site/markdown/storediag.md | 1881 ++++++++++++++++++++
src/site/markdown/tlsinfo.md | 586 ++++++
src/site/markdown/versioned-objects.md | 278 +++
src/site/site.xml | 74 +
.../org/apache/hadoop/fs/store/TestCloudstore.java | 65 +
.../org/apache/hadoop/fs/store/TestStoreUtils.java | 393 ++++
.../fs/store/audit/TestAuditLogProcessor.java | 300 ++++
.../hadoop/fs/store/audit/TestAuditTool.java | 106 ++
.../hadoop/fs/store/commands/FieldsForTesting.java | 31 +
.../hadoop/fs/store/commands/TestConstval.java | 81 +
.../hadoop/fs/store/commands/TestLocalHost.java | 39 +
.../hadoop/fs/store/commands/TestTLSInfo.java | 101 ++
.../contract/AbstractCloudupContractTest.java | 186 ++
.../store/contract/AbstractEtagContractTest.java | 54 +
.../contract/AbstractListObjectsContractTest.java | 76 +
.../contract/AbstractListVersionsContractTest.java | 148 ++
.../AbstractPathCapabilityContractTest.java | 61 +
.../contract/AbstractStorediagContractTest.java | 102 ++
.../store/contract/TestLocalCloudupContract.java | 30 +
.../contract/TestLocalPathCapabilityContract.java | 43 +
.../store/contract/TestLocalStorediagContract.java | 33 +
.../contract/hdfs/TestHdfsCloudupContract.java | 51 +
.../hdfs/TestHdfsPathCapabilityContract.java | 59 +
.../contract/hdfs/TestHdfsStorediagContract.java | 52 +
.../store/contract/s3a/ITestS3ABucketMetadata.java | 50 +
.../fs/store/contract/s3a/ITestS3ABucketState.java | 49 +
.../contract/s3a/ITestS3ACloudupContract.java | 30 +
.../store/contract/s3a/ITestS3AEtagContract.java | 34 +
.../fs/store/contract/s3a/ITestS3AIamPolicy.java | 62 +
.../contract/s3a/ITestS3AListObjectsContract.java | 33 +
.../contract/s3a/ITestS3AListVersionsContract.java | 34 +
.../s3a/ITestS3APathCapabilityContract.java | 36 +
.../fs/store/contract/s3a/ITestS3ASessionKeys.java | 108 ++
.../contract/s3a/ITestS3AStorediagContract.java | 30 +
.../fs/store/contract/s3a/S3AStoreContract.java | 40 +
.../fs/store/diag/TestStoreDiagInvocations.java | 112 ++
.../fs/store/logging/TestLog4JController.java | 52 +
.../apache/hadoop/tools/store/StoreTestUtils.java | 183 ++
.../apache/hadoop/tools/store/TestDiagUtils.java | 42 +
src/test/resources/TestAuditLogs/sampleLog1 | 36 +
src/test/resources/TestAuditLogs/sampleLog2 | 36 +
src/test/resources/contract/hdfs.xml | 99 ++
src/test/resources/contract/localfs.xml | 91 +
src/test/resources/contract/s3a.xml | 111 ++
src/test/resources/core-site.xml | 33 +
src/test/resources/log4j.properties | 94 +
src/test/resources/logback-test.xml | 36 +
183 files changed, 29682 insertions(+)
diff --git a/.asf.yaml b/.asf.yaml
new file mode 100644
index 0000000..d1d7182
--- /dev/null
+++ b/.asf.yaml
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# The format of this file is documented at
+# https://github.com/apache/infrastructure-asfyaml/blob/main/README.md
+
+github:
+ description: "Apache Hadoop Cloudstore Tool"
+ homepage: https://hadoop.apache.org/
+ enabled_merge_buttons:
+ squash: true
+ merge: false
+ rebase: true
+ # default commit message when merging with a squash commit
+ # can either be: DEFAULT | PR_TITLE | PR_TITLE_AND_COMMIT_DETAILS | PR_TITLE_AND_DESC
+ squash_commit_message: PR_TITLE_AND_COMMIT_DETAILS
+ autolink_jira:
+ - HADOOP
+ # Projects can enable and disable Dependabot alerts and automatic security update pull requests:
+ # this is on experimentally, we don't want to update hadoop because we want
+ # to pin versions down
+ dependabot_alerts: true
+ dependabot_updates: true
+ copilot_code_review:
+ enabled: true
+ review_drafts: false
+ review_on_push: true
+
+ pull_requests:
+ # auto-delete head branches after being merged
+ del_branch_on_merge: true
+ # see https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches
+ protected_branches:
+ # main is locked down
+ main:
+ required_status_checks:
+ # strict means "Require branches to be up to date before merging".
+ strict: false
+ # contexts are the names of checks that must pass.
+ required_pull_request_reviews:
+ dismiss_stale_reviews: true
+ require_last_push_approval: false
+ # commit then review OK for now
+ required_approving_review_count: 0
+
+ # squash or rebase must be allowed in the repo for this setting to be set to true.
+ required_linear_history: true
+
+ # gh merge signs; manual commits need committers to set up git commit signing.
+ required_signatures: true
+
+ # requires all conversations to be resolved before merging is possible
+ required_conversation_resolution: false
+
+notifications:
+ commits: common-commits@hadoop.apache.org
+ issues: common-issues@hadoop.apache.org
+ pullrequests: common-issues@hadoop.apache.org
+ jira_options: comment link label
+
+features:
+ wiki: false
+ issues: false
+ projects: false
+
+
+
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..a16b455
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Dependabot configuration for cloudstore.
+#
+# IMPORTANT: PRs filed by Dependabot are reviewed and merged by hand. Do NOT
+# add an auto-merge workflow. Hadoop/AWS SDK/GCS-connector versions are
+# manually selected to work around known CVEs and downstream compatibility
+# constraints; an automated bump can silently regress those fixes.
+#
+# Dependabot itself never auto-merges; that requires a separate workflow.
+# This file only causes PRs to be opened, which is exactly the desired
+# behaviour - visibility without automatic acceptance.
+
+version: 2
+updates:
+
+ # Pin third-party GitHub Actions by SHA per SECURITY.md.
+ # Dependabot understands the "uses: org/action@<sha> # vX.Y.Z" pattern
+ # and rewrites both the SHA and the trailing version comment together.
+ - package-ecosystem: github-actions
+ directory: /
+ schedule:
+ interval: weekly
+ open-pull-requests-limit: 5
+ labels:
+ - dependencies
+ - github-actions
+ cooldown:
+ default-days: 7
+
+ # Maven dependencies. Most cloudstore deps are `provided` scope - the host
+ # Hadoop install supplies them at runtime - so a bump here only changes
+ # what the test/CI run resolves against, not what ships in the jar.
+ # Still worth surfacing: plugin versions, junit, assertj, avro etc. are
+ # all in scope.
+ - package-ecosystem: maven
+ directory: /
+ schedule:
+ interval: weekly
+ open-pull-requests-limit: 5
+ labels:
+ - dependencies
+ - maven
+ cooldown:
+ default-days: 7
+
+ # Shut up warnings about hadoop and the asf/gcs bundle JARs
+ ignore:
+ - dependency-name: org.apache.hadoop:*
+ - dependency-name: software.amazon.awssdk:*
+ - dependency-name: com.google.cloud.bigdataoss:*
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 0000000..30a8e42
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,26 @@
+<!--
+ Thanks for sending a pull request!
+ 1. If this is your first time, please read our contributor guidelines: https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute
+ 2. Make sure your PR title starts with JIRA issue id, e.g., 'HADOOP-17799. Your PR title ...'.
+-->
+
+### Description of PR
+
+
+### Which issue does this close?
+
+<!-- insert the issue number prefixed by # for a back reference to be created -->
+
+### How was this patch tested?
+
+
+### Which remote stores did you test against?
+
+
+### For code changes:
+
+- [ ] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')?
+- [ ] Object storage: have the integration tests been executed and the endpoint declared according?
+- [ ] If adding new runtime dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?
+
+
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 0000000..4c833d3
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# CodeQL static analysis for Java. Runs on PRs to long-lived branches and
+# weekly on a schedule. Findings appear in the Security tab; the workflow
+# also produces a status check on PRs so reviewers see the badge.
+#
+# Third-party actions are SHA-pinned per SECURITY.md.
+
+name: CodeQL
+
+on:
+ push:
+ branches: [ main, master, trunk ]
+ pull_request:
+ branches: [ main, master, trunk ]
+ schedule:
+ # Mondays 07:17 UTC -- off the hour to dodge the GHA traffic spike.
+ - cron: '17 7 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ # Group PRs by PR number (cancellable on synchronize); pushes/cron by
+ # ref (queued, never canceled).
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ analyze:
+ name: Analyze (java)
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ # Required for the analyze step to upload SARIF to the Security tab.
+ security-events: write
+ # Needed by some private-repo paths; harmless on public repos.
+ actions: read
+ contents: read
+
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Set up JDK
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
+ with:
+ languages: java-kotlin
+ # Default + security-extended queries. Add security-and-quality if
+ # noise stays manageable.
+ queries: security-extended
+
+ - name: Build with Maven
+ run: mvn -B -DskipTests package
+
+ - name: Perform CodeQL analysis
+ uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
+ with:
+ category: "/language:java-kotlin"
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
new file mode 100644
index 0000000..4c3b135
--- /dev/null
+++ b/.github/workflows/dependency-review.yml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dependency Review: fails the PR if it introduces a new vulnerable
+# transitive dependency or a disallowed license. Surfaces a PR comment
+# summarizing new dependencies.
+#
+# Hadoop / AWS SDK / GCS-connector pins are manually chosen to work around
+# known CVEs in the broader Hadoop ecosystem (see dependabot.yml). The
+# action's failure threshold below ('high') matches that pragmatic stance
+# without rubber-stamping new highs that were added in *this* PR.
+#
+# Third-party actions are SHA-pinned per SECURITY.md.
+
+name: Dependency review
+
+on:
+ pull_request:
+ branches: [ main, master, trunk ]
+
+permissions:
+ contents: read
+ # Required so the action can post the summary comment.
+ pull-requests: write
+
+concurrency:
+ # Group PRs by PR number (cancellable on synchronize). Workflow runs on
+ # pull_request only, so the fallback to github.ref is for future-proofing.
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ review:
+ name: Dependency review
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Dependency review
+ uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
+ with:
+ fail-on-severity: high
+ comment-summary-in-pr: on-failure
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
new file mode 100644
index 0000000..5139f80
--- /dev/null
+++ b/.github/workflows/maven.yml
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Java CI with Maven, split so that:
+#
+# * lint -- license headers + Spotless formatting. Cheap, deterministic;
+# MUST be green to merge (configured in branch protection).
+# Runs in upstream on every push and every PR (incl. fork PRs).
+#
+# * build -- compile + unit tests + site. Heavier; tolerated red, but the
+# author is expected to look. Runs in upstream on pushes and on
+# PRs raised from the *same* repository. PRs from forks fall
+# back to the forker's own Actions runner (their fork builds on
+# push to the PR branch), which keeps upstream cycles for the
+# cheap checks and limits exposure to PR code in the base repo.
+#
+# Third-party actions are SHA-pinned per SECURITY.md. When bumping, update
+# both the SHA and the trailing tag comment in lockstep.
+
+name: Java CI with Maven
+
+on:
+ push:
+ pull_request:
+ branches: [ main, master, trunk ]
+
+permissions:
+ contents: read
+
+concurrency:
+ # Group PRs by PR number (cancellable on synchronize); pushes/cron by
+ # ref (queued, never canceled).
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ # Cancel superseded PR runs. Don't cancel pushes to long-lived branches --
+ # every commit to main/master should be verified end-to-end.
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ lint:
+ name: Lint (RAT + Spotless)
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Set up JDK
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ - name: Apache RAT (license headers)
+ run: mvn -B org.apache.rat:apache-rat-plugin:check
+ - name: Spotless (palantir-java-format)
+ run: mvn -B spotless:check
+
+ build:
+ name: Build (compile + tests + site)
+ needs: lint
+ # Skip on PRs raised from forks -- the forker's own Actions runs the
+ # full build on push to their PR branch. Upstream still gets lint above.
+ # zizmor: ignore[template-injection] -- expression context only; both
+ # operands are GitHub-issued repo slugs ([a-z0-9._-]+/[a-z0-9._-]+),
+ # not user-controlled free text, so no injection surface exists.
+ if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ - name: Set up JDK
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+ with:
+ # apache-rat-plugin 0.18 (wired into verify) requires JDK 17.
+ # Bytecode target remains 1.8 via the compiler plugin.
+ java-version: '17'
+ distribution: 'temurin'
+ - name: Verify (compile + unit tests + RAT + Spotless)
+ run: mvn -B verify
+ - name: Build site
+ run: mvn -B site -DskipTests
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
new file mode 100644
index 0000000..dbee428
--- /dev/null
+++ b/.github/workflows/scorecard.yml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# OSSF Scorecard: weekly supply-chain posture audit. Results upload to the
+# Security tab and (when the repo opts in to the OSSF API) feed the public
+# scorecard badge.
+#
+# Third-party actions are SHA-pinned per SECURITY.md.
+
+name: Scorecard supply-chain security
+
+on:
+ branch_protection_rule:
+ schedule:
+ # Mondays 08:23 UTC.
+ - cron: '23 8 * * 1'
+ push:
+ branches: [ main, master, trunk ]
+
+# Top-level: least-privilege. The analysis job widens to the specific
+# scopes it needs (security-events, actions, id-token).
+permissions:
+ contents: read
+
+concurrency:
+ # Push/cron only; cancel-in-progress evaluates to false so consecutive
+ # runs queue. Block is here for uniformity and to guard a future
+ # pull_request trigger.
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ analysis:
+ name: Scorecard analysis
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ permissions:
+ # Upload SARIF to the Security tab.
+ security-events: write
+ # Read action run metadata (e.g. for Token-Permissions check).
+ actions: read
+ contents: read
+ # Required to publish results to the OSSF results API (opt-in below).
+ id-token: write
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+
+ - name: Run analysis
+ uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+ with:
+ results_file: results.sarif
+ results_format: sarif
+ # Set to true to publish results to the OSSF results API and surface
+ # the public badge. Safe to leave on for a public repo.
+ publish_results: true
+
+ - name: Upload artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: SARIF file
+ path: results.sarif
+ retention-days: 5
+
+ - name: Upload to code-scanning dashboard
+ uses: github/codeql-action/upload-sarif@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4
+ with:
+ sarif_file: results.sarif
diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml
new file mode 100644
index 0000000..5973624
--- /dev/null
+++ b/.github/workflows/zizmor.yml
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# zizmor: static analysis for GitHub Actions workflows. Catches things
+# CodeQL/Scorecard/dependency-review don't: template injection in run:
+# steps, over-broad permissions, unpinned actions, pull_request_target
+# misuse, cache-poisoning vectors, etc.
+#
+# Findings upload to the Security tab as SARIF (category 'zizmor').
+# Treated like CodeQL: visible status check, expected-green, but not a
+# hard merge blocker until the tool has been clean for a while.
+#
+# Third-party actions are SHA-pinned per SECURITY.md.
+
+name: Zizmor
+
+on:
+ push:
+ branches: [ main, master, trunk ]
+ pull_request:
+ branches: [ main, master, trunk ]
+ schedule:
+ # Mondays 09:11 UTC -- off the hour, staggered from codeql/scorecard.
+ - cron: '11 9 * * 1'
+
+permissions:
+ contents: read
+
+concurrency:
+ # Group PRs by PR number (cancellable on synchronize); pushes/cron by
+ # ref (queued, never canceled).
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ zizmor:
+ name: Zizmor workflow audit
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ # Upload SARIF to the Security tab.
+ security-events: write
+ contents: read
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+
+ - name: Run zizmor
+ uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
+ with:
+ # Surface everything initially; tighten if the signal/noise is poor.
+ min-severity: low
+ min-confidence: low
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..33cf126
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+target/
+
+#idea
+.idea/
+*.iml
+.DS_Store
+*.ipr
+*.iws
+build.properties
+atlassian-ide-plugin.xml
+auth-keys.xml
+.junit
+
+# secrets
+auth-keys.xml
+
+# maven versions:set leftover
+pom.xml.versionsBackup
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..8a8e1c5
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,199 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+This file governs AI-assisted work on the Apache Hadoop Cloudstore codebase.
+
+
+## Project
+
+Cloudstore is a Java 8 / Maven diagnostics CLI bundled as a single JAR and invoked via `hadoop jar`. It sits on top of the Apache Hadoop FileSystem APIs and adds tools for troubleshooting cloud connectors (S3A, ABFS, ADL, GCS) — classpath/credential/network checks, faster listings, bandwidth probes, AWS V2 SDK probes, etc. It is intentionally not part of the Hadoop release: agile cadence, lighter testing, and the freedom to compile against multiple Hadoop versions are explicit goals.
+
+All source lives under `org.apache.hadoop.fs.*` deliberately, so it can call package-private Hadoop APIs.
+
+### Key Technologies
+
+* **Language:** Java 8
+* **Build:** Apache Maven
+* **Core Dependencies:**
+ * Apache Hadoop (various modules: `hadoop-client`, `hadoop-common`, `hadoop-cloud-storage`)
+ * AWS SDK for Java 2.x
+ * Google Cloud Storage Connector
+
+Hadoop, AWS SDK v2, and the GCS connector are all `provided` scope — the JAR is designed to be dropped onto an existing Hadoop install.
+
+## Build and run
+
+See [BUILDING.md](./BUILDING.md).
+
+Bytecode is pinned to Java 8 (compiler plugin + enforcer rule `[1.8,)`). The build itself runs fine on JDK 11+, and **needs** JDK 11+ if you invoke `spotless:apply` (palantir-java-format requires it).
+
+```bash
+mvn clean install # compile + unit tests + jar (default: hadoop.version=3.4.0)
+mvn clean verify # adds: ITest*, apache-rat:check, spotless:check
+mvn install -Pnext # next hadoop release
+mvn install -Dtrunk # trunk p;roifle
+mvn install -P snapshots-and-staging # adds ASF staging/snapshot repos
+
+mvn spotless:apply # auto-format Java to palantir-java-format
+mvn org.apache.rat:apache-rat-plugin:check # license-header audit only
+mvn site # render src/site → target/site (fluido skin)
+```
+
+The active build wires `apache-rat-plugin`, `spotless-maven-plugin`, and `maven-failsafe-plugin` into the `verify` phase. RAT and Spotless will fail the build if a newly added file is missing the ASF header or hasn't been run through `spotless:apply`.
+
+
+Releasing is manual and tag-driven — see `BUILDING.md`. Bump the version with `mvn versions:set -DnewVersion=…` *and* search-and-replace `cloudstore-<old>.jar` references; the `-SNAPSHOT` suffix is intentionally never used.
+
+## Running Commands
+
+The tools are executed via the `hadoop jar` command. The general syntax is:
+
+```sh
+hadoop jar target/cloudstore-1.4.jar <command> [options] <arguments...>
+```
+
+- `<command>`: The name of the tool to run (e.g., `list`, `dux`, `storediag`).
+- `[options]`: Common options include:
+ - `-D <key=value>`: Define a Hadoop configuration property.
+ - `-xmlfile <file>`: Load a Hadoop configuration XML file.
+ - `-verbose`: Enable verbose output.
+ - `-debug`: Enable low-level debug logging for the JVM and connectors.
+
+**Example: Listing files in an S3 bucket**
+
+```sh
+hadoop jar target/cloudstore-1.4.jar list -limit 10 s3a://my-bucket/path/
+```
+
+
+## Tests
+
+JUnit 4 + AssertJ. Convention drives which runner picks up a class:
+
+- `Test*` — unit tests, run by surefire under `mvn test` / `mvn install`.
+- `ITest*` — integration tests, run by failsafe under `mvn verify` (forked, 600s timeout, `reuseForks=false`).
+
+Single test / single method:
+
+```bash
+mvn test -Dtest=TestConstval
+mvn test -Dtest=TestConstval#methodName
+mvn verify -Dit.test=ITestLocalStorediagContract
+```
+
+Most tests are local; cloud-touching integration tests need credentials supplied via `src/test/resources/auth-keys.xml` (gitignored). Storediag-specific contract tests live under `org.apache.hadoop.fs.store.contract` and extend `AbstractFSContractTestBase`. The S3A contract test (`ITestS3AStorediagContract`) is gated by the `s3a-it` profile because the `hadoop-aws` test-jar is not consistently published to Maven Central.
+
+## Architecture
+
+### Two-layer command dispatch
+
+Each user-facing command has two classes:
+
+1. A short-name shim in the **default package**, e.g. `src/main/java/dux.java`. It is just `public static void main(String[] args) { ExtendedDu.main(args); }` plus a `help()` method called from `help.java`. This is what `hadoop jar … dux` resolves to — `hadoop jar` uses the bare class name as the entry point, and Java's default package makes it short.
+2. The real implementation under `org.apache.hadoop.fs.store.commands.*` (or `…s3a.sdk*`, `…store.abfs`, `…store.audit`, `…tools.cloudup`, `…tools.csv`, `…gs`).
+
+When adding a new command: create the implementation class extending `StoreEntryPoint`, add a default-package shim, and register it in `help.java` (alphabetically, in the right section — note the AWS V2 SDK commands are listed separately because they require S3A's V2 SDK at runtime).
+
+### Entry-point hierarchy
+
+- `org.apache.hadoop.fs.store.StoreEntryPoint` — `Configured implements Tool, Closeable, Printout`. Provides argument parsing for the common options (`-D`, `-xmlfile`, `-tokenfile`, `-verbose`, `-debug`, `-sysprops`, `-logoverrides`), token loading, log-level overrides, and `Printout` helpers used by diagnostics. Standard option keys live in `CommonParameters`.
+- `org.apache.hadoop.fs.store.diag.DiagnosticsEntryPoint extends StoreEntryPoint` — extra helpers shared by diagnostics commands (`storediag`, `constval`, `tlsinfo`, etc.).
+
+Each implementation class follows the same pattern: `createCommandFormat(min, max)` in the constructor, `run(String[] args)` does the work, plus a `static int exec(String...)` that delegates to `ToolRunner.run` and a `static void main(...)` that calls `exec` and exits.
+
+### Per-store diagnostics
+
+`org.apache.hadoop.fs.store.diag.StoreDiagnosticsInfo` is the polymorphism point for `storediag`. Subclasses describe how to inspect a particular connector: `S3ADiagnosticsInfo`, `ABFSDiagnosticsInfo` (note the `Abfs…` filename), `ADLDiagnosticsInfo`, `GCSDiagnosticsInfo`, `HDFSDiagnosticsInfo`, `WasbDiagnosticsInfo`, `HBossConstants`, plus a `TemplateDiagnosticsInfo` to copy when adding a new store. Each contributes the option keys to print, environment variables to surface, classpath p [...]
+
+### Generated sources
+
+Avro schemas under `src/main/avro` are compiled into `target/generated-sources/avro` during `generate-sources`. The output is added to the compile source roots automatically. The `auditlogs` command uses these.
+
+## Development Conventions
+
+- Logging is SLF4J. Many entry points also write to `System.out` directly (with `@SuppressWarnings("UseOfSystemOutOrSystemErr")`) because the output is meant for humans running the tool.
+- Configuration flows through Hadoop's `Configuration`. Do not add a separate config system.
+- Don't introduce dependencies that aren't `provided` or `test` — the JAR has to stay slim because it's deployed onto whatever Hadoop install the user already has.
+
+### Configuration
+
+The project uses the standard Hadoop `Configuration` framework. Configuration can be supplied through:
+1. Default Hadoop configuration files (`core-site.xml`, `hdfs-site.xml`, etc.).
+2. Custom XML files specified with the `-xmlfile` option.
+3. Individual properties set with the `-D` flag.
+
+
+### Testing
+
+- Unit tests are located in `src/test/java`.
+- Tests are run as part of the `mvn clean install` build process.
+- The project uses JUnit for testing.
+- In tests, always use assertJ assertions instead of junit asserts
+
+### Security
+
+See [SECURITY.MD](./SECURITY.md) for details.
+
+- Review all changes to see if they introduce security issues, especially with logging secrets.
+
+
+## ASF Legal Compliance (Third-Party Code)
+
+This is an [Apache Software Foundation (ASF)](https://www.apache.org/) project released under the **Apache License 2.0**.
+The AI **must** actively enforce and monitor ASF licensing policy:
+
+- **Proactively flag conflicts**: Before introducing any dependency, snippet, or code derived from an external source, verify its license is compatible with Apache 2.0.
+ Incompatible licenses include (non-exhaustive): GPL, AGPL, SSPL, BUSL, CC-BY-NC.
+ Compatible examples: MIT, BSD-2/3, Apache 2.0, ISC, MPL 2.0 (with caveats).
+- **Category X / Category A**: Follow the [ASF Third-Party Licensing Policy](https://www.apache.org/legal/resolved.html).
+ Category A licenses may be included; Category X licenses must **never** be introduced.
+- **Update `LICENSE` and `NOTICE`**: When adding third-party code or binaries that require attribution, add the appropriate notices to `LICENSE` and/or `NOTICE` following the [ASF guide on licenses and notices](https://www.apache.org/dev/licensing-howto.html).
+ If in doubt whether an entry is required, **add it and flag it in the PR description** for committer review.
+- **Generative AI output**: The [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) applies. Be aware that AI-generated code may unintentionally reproduce copyrighted material. Flag any non-trivial generated blocks in commit messages or PR descriptions.
+
+## 3. Pull Request Requirements
+
+Follow [`CONTRIBUTING.md`](./CONTRIBUTING.md) in full. Key points:
+
+- One commit per issue (squash before submitting).
+- All significant changes need a JIRA ticket.
+- Provide tests for every submitted change.
+- Verify coding standards: `make style`.
+- Branch name convention: use the JIRA ticket ID, e.g. `THRIFT-9999`.
+- PRs go from your fork branch → `apache:master`.
+
+---
+
+## 4. AI-Generated Contributions
+
+Per [`CONTRIBUTING.md § AI generated content`](CONTRIBUTING.md#ai-generated-content) and the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html):
+
+- **Always** label AI-assisted commits and PRs. Use one or both of:
+ ```
+ Co-Authored-By: <AI tool name and version>
+ Generated-by: <AI tool name and version>
+ ```
+ Example:
+ ```
+ HADOOP-9999: Fix connection timeout
+
+ Co-Authored-By: Claude Sonnet 4.6 <no...@anthropic.com>
+ ```
+- Apply this label even when AI only generated a portion of the change.
+- The human author remains responsible for reviewing, testing, and standing behind all submitted code.
diff --git a/BUILDING.md b/BUILDING.md
new file mode 100644
index 0000000..b6c85f3
--- /dev/null
+++ b/BUILDING.md
@@ -0,0 +1,232 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Building
+
+
+# Hadoop versions
+
+There's a dependency on hadoop 3.4.2 to be compatible with its avro version and to
+have non-reflective access to the bulk delete API.
+You should still code targeting hadoop 3.4.0 as the minimum version *outside these specific
+cli commands*
+
+## Compiling
+
+With maven
+
+To build a production release
+1. Compile on a JDK17 JVM; the output is still generated for java 8 JVMs.
+2. Compile against a shipping hadoop version (see the profiles).
+
+
+```bash
+mvn spotless:apply # auto-format Java sources to palantir-java-format
+mvn clean install # compile + unit tests + jar
+mvn clean verify # adds: ITest*, apache-rat:check, spotless:check
+mvn org.apache.rat:apache-rat-plugin:check # license-header audit only
+mvn site # render src/site → target/site (fluido skin)
+```
+
+The `verify` lifecycle is the one CI runs. To run cloud-backed contract tests
+opt in via profile + credentials in `src/test/resources/auth-keys.xml`:
+
+## Testing
+
+There are unit tests for store operations against the local fs and a mini hdfs cluster.
+
+There are Integration Tests which run with the `mvn verify` command, currently against S3 stores only.
+The tests are minimal, any basic S3 implementation will suffice.
+
+To bind to a store, follow the [testing s3a](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/testing.html#File_auth-keys.xml) docs.
+
+## Updating cloudstore release versions
+
+For a long time the artifact version was fixed at 1.0 so that curl and other
+tools could fetch a stable URL. That was convenient but produced a different
+problem: support tickets kept attaching the same `cloudstore-<old>.jar`
+long after new releases had shipped, with no version change to make the staleness
+visible. Release number increments are now required for anything other than
+a rapid-iteration multiple-releases-in-a-day workflow.
+
+The project follows the conventional Maven SNAPSHOT lifecycle:
+
+| Phase | pom `<version>` | docs (`cloudstore-X.Y.jar`) |
+|---------------------------|--------------------|-----------------------------|
+| Day-to-day development | `X.Y-SNAPSHOT` | last released `X.Y` |
+| Cutting release `X.Y` | `X.Y` | `X.Y` |
+| Immediately after release | `X.(Y+1)-SNAPSHOT` | `X.Y` |
+
+`-SNAPSHOT` appears in the pom only; the published artifact and every
+documented `cloudstore-X.Y.jar` reference is always a bare release form.
+
+Two scripts in `dev-support/` keep these in sync:
+
+
+**`bump-version.sh <new-version>`**
+
+Bumps `pom.xml`'s `<version>` only.
+Accepts either `X.Y-SNAPSHOT` (for development bumps) or `X.Y`
+(when cutting a release).
+Also rewrites `BUILDING.md`'s `set -gx ver <v>` line so the release command block
+below stays in step (the line always reflects the bare release
+version — a trailing `-SNAPSHOT` is stripped before substituting).
+Does *not* touch `README.md`, `AGENTS.md`, or `src/site/markdown/*`.
+
+**`update-site-docs.sh <new-version>`**
+Updates site documentation for releases.
+Rewrites every `cloudstore-<old>.jar` /
+ `cloudstore-<old>-cyclonedx` reference in `README.md`, `AGENTS.md`,
+ `BUILDING.md`, and `src/site/markdown/*.md`, and bumps the
+ `<cloudstore.docs.version>` property in `pom.xml`.
+Rejects attempts to switch to a `-SNAPSHOT`; SNAPSHOT artifacts are not released and so not documented.
+
+The `verify` phase enforces this: `dev-support/check-doc-versions.sh`
+runs as part of `mvn verify` and fails the build if any markdown doc
+references a `cloudstore-X.Y.jar` that disagrees with
+`${cloudstore.docs.version}` (independent of `${project.version}`, so
+SNAPSHOT pom states do not trip the gate).
+
+### Worked example: cutting release 1.4 then returning to dev
+
+```bash
+# 1. Cut the release: pom 1.4-SNAPSHOT -> 1.4, site docs 1.3 -> 1.4.
+dev-support/bump-version.sh 1.4
+dev-support/update-site-docs.sh 1.4
+# (build, tag, publish — see "Releasing" below.)
+
+# 2. Back to development: pom 1.4 -> 1.5-SNAPSHOT. Docs stay at 1.4.
+dev-support/bump-version.sh 1.5-SNAPSHOT
+```
+
+
+## Releasing
+
+To publish the release use the github command line through the `fish` shell, with a final
+git UI interaction.
+
+Release builds activate the `release` profile, which:
+
+a. enforces a clean git tree via `buildnumber-maven-plugin`;
+b. emits a CycloneDX SBOM next to the jar:
+ - `target/cloudstore-<version>-cyclonedx.json`
+ - `target/cloudstore-<version>-cyclonedx.xml`
+c. emits a SHA-256 digest next to the jar and each SBOM file:
+ - `target/cloudstore-<version>.jar.sha256`
+ - `target/cloudstore-<version>-cyclonedx.json.sha256`
+ - `target/cloudstore-<version>-cyclonedx.xml.sha256`
+
+The SBOM is compile-scope only — `provided` deps (Hadoop, AWS SDK v2, GCS
+connector) are not in it because they are not shipped in the jar.
+
+The `.sha256` files are produced by `checksum-maven-plugin` in the `verify`
+phase. They give downloaders a quick integrity check (`shasum -c
+cloudstore-<version>.jar.sha256`) independent of the gpg signature.
+
+It also generates a build version, with the buildnumber plugin.
+On release builds, this will fail the build if there are uncommitted changes.
+
+Commit all changes before starting a release build.
+
+Commands (for fish)
+```bash
+set -gx ver 1.5 # bumped by dev-support/bump-version.sh (always release form, no -SNAPSHOT)
+mvn clean install -Prelease,sign -DskipTests
+set -gx now (date '+%Y-%m-%d-%H.%M'); echo [$now]
+git commit -S --allow-empty -m "release $now"; git push
+# Guard: every artifact path below interpolates $ver. An unset/empty $ver
+# expands to target/cloudstore-.* (no match), so gh silently uploads only the
+# literal LICENSE-binary/NOTICE-binary. Fail loudly before that happens.
+test -n "$ver"; or begin; echo "ERROR: \$ver is unset/empty -- run: set -gx ver <version>"; exit 1; end
+gh release create tag-release-$now -t release-$now --generate-notes -d \
+ target/cloudstore-$ver.jar \
+ target/cloudstore-$ver.jar.asc \
+ target/cloudstore-$ver.jar.sha256 \
+ target/cloudstore-$ver-cyclonedx.json \
+ target/cloudstore-$ver-cyclonedx.json.asc \
+ target/cloudstore-$ver-cyclonedx.json.sha256 \
+ target/cloudstore-$ver-cyclonedx.xml \
+ target/cloudstore-$ver-cyclonedx.xml.asc \
+ target/cloudstore-$ver-cyclonedx.xml.sha256 \
+ LICENSE-binary \
+ NOTICE-binary
+echo "go to the web ui to review and finalize the release"
+```
+
+* If a new release is made the same day, remember to create a new tag.
+* If you have an env var pointing to the cloudstore JAR, update it!
+
+## Signing release artifacts
+
+Activate the `sign` profile alongside `release` (as in the command
+above) to GPG-sign every attached artifact. The plugin produces a
+detached `.asc` next to each of:
+
+- `target/cloudstore-<version>.jar`
+- `target/cloudstore-<version>.pom`
+- `target/cloudstore-<version>-cyclonedx.json`
+- `target/cloudstore-<version>-cyclonedx.xml`
+
+Prerequisites:
+
+1. An OpenPGP key in the Hadoop committer KEYS file.
+2. `gpg-agent` running with the release key unlocked. A quick way to
+ warm the agent before the build is:
+ `echo test | gpg --clearsign -u <keyid> > /dev/null`.
+
+Flags:
+
+- `-Dgpg.keyName=<keyid>` — pick a specific key. If unset, gpg's
+ default secret key is used.
+- `-Dgpg.skip=true` — disable signing even when `-Psign` is active.
+
+`-Prelease` on its own (without `sign`) still produces a valid jar and
+SBOM — useful for local smoke tests on machines without the release
+key.
+
+Verify a downloaded release. The `.sha256` files contain the bare hex
+digest (Apache convention) rather than the `shasum -c` wire format, so
+compare directly:
+
+```bash
+# integrity: hash matches what the release published.
+[ "$(shasum -a 256 cloudstore-$ver.jar | awk '{print $1}')" = \
+ "$(cat cloudstore-$ver.jar.sha256)" ] && echo "sha256 OK"
+# authenticity: the matching signature is from the expected key.
+gpg --verify cloudstore-$ver.jar.asc cloudstore-$ver.jar
+```
+
+The `gh release create` command above attaches the jar, the SBOM
+(JSON + XML), each of their `.asc` signatures, and each of their
+`.sha256` digests in one shot. For an already-published release,
+append the missing files with `gh release upload`:
+
+```bash
+test -n "$ver"; or begin; echo "ERROR: \$ver is unset/empty -- run: set -gx ver <version>"; exit 1; end
+gh release upload tag-release-$now \
+ target/cloudstore-$ver.jar.asc \
+ target/cloudstore-$ver.jar.sha256 \
+ target/cloudstore-$ver-cyclonedx.json.asc \
+ target/cloudstore-$ver-cyclonedx.json.sha256 \
+ target/cloudstore-$ver-cyclonedx.xml.asc \
+ target/cloudstore-$ver-cyclonedx.xml.sha256
+```
+
+## How to bypass buildnumber checks
+
+```bash
+mvn clean install -Prelease -DskipTests -Dbuildnumber.check=false -Dbuildnumber.update=false
+```
+
+
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..fb8722c
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,19 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+@AGENTS.MD
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..81d9884
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,66 @@
+o <!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Contributing
+
+This project is a subsidiary project of [Apache Hadoop](https://hadoop.apache.org/)
+
+All governance is by the Apache Software Foundation, including release process.
+
+To discuss the project, please get on the [common developer mailing list]( https://hadoop.apache.org/mailing_lists.html).
+
+Do read the Hadoop [How to Contribute](https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute) wiki page.
+
+## Issues and Patches
+
+1. Please create issues on the github project, rather than JIRA.
+2. Create an issue before creating a Pull Request, referring to the issue number in the PR.
+
+## Testing
+
+All PRs will need a local test run against an S3 service endpoint of your choice.
+It doesn't have to be AWS S3 standard: diversity is always interesting, what is
+key is that it's there.
+
+You MUST declare what S3 endpoint you tested against, such as "S3 Frankfurt" or "Local Minio docker image".
+
+See [Testing the S3A filesystem client and its features](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/testing.html) for instructions.
+
+Place your secrets (or better, an XInclude reference to them elsewhere in your local filesystem) in the file `src/test/resources/auth-keys.xml`.
+DO NOT place this file under revision control.
+If you do ever commit cloud credentials to any revision control system: rotate the keys.
+That's even if that's your private internal repo: it's all to easy to accidentally make that public.
+Your IT team will be happier with you saying "I just committed my keys, can I replace them even though I haven't pushed those changes anywhere" to having to deal with the consequences of a leak of keys.
+
+Consider also using a restricted role for the credentials you are storing in XML/JCEKs files. They only need R/W access to test buckets and the ability to create session credentials for `ITestS3ASessionKeys`.
+
+If testing against third party stores or when running all tests with temporary credentials, session credentials will not be available. Set `test.fs.s3a.sts.enabled` to false to disable tests in `ITestS3ASessionKeys`.
+
+
+
+## _Roadmap_: Whatever we need to debug and improve client cloud connectivity
+
+
+* Exploration of higher-performance IO.
+* Diagnostics/testing of integration with foundational Hadoop operations.
+* Improving CLI testing with various probes designed to be invoked in a shell
+ and fail with meaningful exit codes. E.g: verifying that a filesystem has a specific (key, val)
+ property, that a specific env var made it through.
+* Something to scan hadoop installations for duplicate artifacts, which knowledge
+ of JARS which main contain the same classes (aws-shaded, aws-s3, etc),
+ and the knowledge of required consistencies (hadoop-*, jackson-*).
+
+Contributions through PRs welcome.
+
+Bug reports: please include environment and ideally patches.
diff --git a/GEMINI.md b/GEMINI.md
new file mode 100644
index 0000000..fb8722c
--- /dev/null
+++ b/GEMINI.md
@@ -0,0 +1,19 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+@AGENTS.MD
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..261eeb9
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/LICENSE-binary b/LICENSE-binary
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE-binary
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..b90833f
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,5 @@
+Apache Cloudstore
+Copyright 2018-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (https://www.apache.org/).
diff --git a/NOTICE-binary b/NOTICE-binary
new file mode 100644
index 0000000..8c6c6b4
--- /dev/null
+++ b/NOTICE-binary
@@ -0,0 +1,5 @@
+Apache Hadoop Cloudstore
+Copyright 2016 and onwards The Apache Software Foundation.
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f9e814b
--- /dev/null
+++ b/README.md
@@ -0,0 +1,355 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Cloudstore
+
+Cloudstore is a diagnostics library for troubleshooting problems
+interacting with cloud storage through Apache Hadoop
+
+*License*: Apache ASF 2.0
+
+All the implementation classes are under the `org.apache.hadoop.fs` package
+tree but it is not part of the normal Apache Hadoop artifacts.
+
+1. Faster release cycle, so the diagnostics can evolve to track features going
+ into Hadoop-trunk.
+2. Ability to compile against older versions.
+
+
+## Features
+
+### Primarily: diagnostics
+
+Why?
+
+1. Sometimes things fail, and the first problem is classpath;
+2. The second, invariably some client-side config.
+3. Then there's networking and permissions...
+4. The Hadoop FS connectors all assume a well configured system, and don't
+do much in terms of meaningful diagnostics.
+5. This is compounded by the fact that we dare not log secret credentials.
+6. And in support calls, it's all to easy to get those secrets, even
+though its a major security breach to get them.
+
+### Secondary: higher performance cloud IO
+
+The main hadoop `hadoop fs` commands are written assuming a filesystem, where:
+
+* Recursive treewalks are the way to traverse the store.
+* The code was written for Hadoop 1.0 and uses the filesystem APIs of that era.
+* The commands are often used in shell scripts and workflows, including parsing
+the output: we do not dare change the behaviour or output for this reason.
+* And the shell removes stack traces on failures, making it of "limited value"
+when things don't work. And object stores are fairly fussy to get working,
+primarily due to authentication, classpath and network settings
+
+## See also
+
+* [Security](./SECURITY.md)
+* [Building](./BUILDING.md)
+
+
+# Commands
+
+## Common arguments
+
+There are a set of arguments common to all commands
+```
+-D <key=value> Define a property
+-sysprops <file> Java system properties to set
+-tokenfile <file> Hadoop token file to load
+-verbose Verbose output
+-debug Extra debug logs (JVM and Log4j overrides)
+-logoverrides <file> A newline separated list of packages and classes for Log4j overrides
+-xmlfile <file> XML config file to load
+```
+
+### -D <key=value> Define a single Hadoop configuration option
+
+Define a single hadoop option.
+For defining multiple options, use `-xmlfile`
+
+### -sysprops <file>: Java system properties to set
+
+This loads a Java properties file containing java system
+properties as key=value pairs. Each of these
+sets the named java system property.
+
+Blank lines and comment lines beginning with `#` are ignored.
+
+### -tokenfile -tokenfile <file> : Load hadoop tokens
+
+The `--tokenfile` option loads tokens saved with `hdfs fetchdt`. It does
+not need Kerberos, though most filesystems expect Kerberos enabled for
+them to pick up tokens (not S3A, potentially other stores).
+
+### -xmlfile <file>: XML configuration file to load
+
+This loads a hadoop configuration file and adds its values to
+the Hadoop configuration used in the command.
+
+
+
+## storediag
+
+Examine store and print diagnostics, including testing read and optionally write
+operations
+
+See [storediag](src/main/site/storediag.md) for details.
+
+## auditlogs
+
+Parse AWS S3 Server logs into avro files, extracting http referrer-encoded
+audit information where found.
+
+See [auditlogs](src/main/site/auditlogs.md) for details.
+
+## bandwidth
+
+Measure upload/download bandwidth, optionally saving data to a CSV file.
+
+See [bandwidth](src/main/site/bandwidth.md) for details.
+
+## bucketmetadata
+
+Retrieves metadata from an S3 Bucket (v2 SDK only) by probing the store.
+
+See [bucketmetadata](src/main/site/bucketmetadata.md) for details.
+
+## bulkdelete
+
+Performs a bulk delete of files from a store, with higher performance
+against some stores (S3).
+Requires hadoop libraries with the 2024 bulk delete API (hadoop 3.4.1+).
+
+See [bulkdelete](src/main/site/bulkdelete.md) for details.
+
+## cloudup -upload and download files; optimised for cloud storage
+
+See [cloudup](src/main/site/cloudup.md)
+
+## committerinfo
+
+Tries to instantiate a committer using the Hadoop 3.1+ committer factory mechanism, printing out
+what committer a specific path will create.
+
+See [committerinfo](src/main/site/committerinfo.md).
+
+## constval
+
+Loads a class, resolves a constant/static final field and prints its value.
+
+See [constval](src/main/site/constval.md)
+
+
+## dux "Du, extended"
+
+A variant on the hadoop `du` command which does a recursive `listFiles()`
+call on every directory immediately under the source path -in separate threads.
+
+For any store which supports higher performance deep tree listing (S3A in particular)
+This can be significantly faster than du's normal treewalk.
+
+Even without that, because lists are done in separate threads, a speedup is
+almost guaranteed.
+
+There is no scheduling of work into separate threads within a directory; those stores
+which do prefetching in separate threads (recent ABFS and S3A builds) do add some
+parallelism here.
+
+
+```
+Usage: dux
+ -D <key=value> Define a property
+ -tokenfile <file> Hadoop token file to load
+ -xmlfile <file> XML config file to load
+ -threads <threads> number of threads
+ -limit <limit> limit of files to list
+ -verbose print verbose output
+ -bfs do a breadth first search of the path
+ <path>
+```
+
+The `-verbose` option prints out more filesystem statistics, and of
+the list iterators (useful if they publish statistics)
+
+`-limit` puts a limit on the total number of files to scan; this
+is useful when doing deep scans of buckets so as to put an upper bound
+on the scan. Note, when used against S3 an ERROR may be printed in the AWS SDK.
+This is harmless; it comes from the SDK thread pool being closed while
+a list page prefetch is in progress.
+
+
+## etag
+
+Prints the etag of an object, when implemented by the filesystem
+and returned by the object store.
+
+See [etag](src/main/site/etag.md)
+
+## fetchdt
+
+This is an extension of `hdfs fetchdt` which collects delegation tokens
+from a list of filesystems, saving them to a file.
+
+See [fetchdt](src/main/site/fetchdt.md)
+
+## filestatus
+
+Calls `getFileStatus` on the listed paths, prints the values. For stores
+which have more detail on the toString value of any subclass of `FileStatus`,
+this can be more meaningful.
+
+Also prints the time to execute each operation (including instantiating the store),
+and with the `-verbose` option, the store statistics.
+
+```
+hadoop jar cloudstore-1.4.jar \
+ filestatus \
+ s3a://guarded-table/example
+
+2019-07-31 21:48:34,963 [main] INFO commands.PrintStatus (DurationInfo.java:<init>(53)) - Starting: get path status
+s3a://guarded-table/example S3AFileStatus{path=s3a://guarded-table/example; isDirectory=false; length=0; replication=1;
+ blocksize=33554432; modification_time=1564602680000;
+ access_time=0; owner=alice; group=alice;
+ permission=rw-rw-rw-; isSymlink=false; hasAcl=false; isEncrypted=true; isErasureCoded=false}
+ isEmptyDirectory=FALSE eTag=d41d8cd98f00b204e9800998ecf8427e versionId=null
+2019-07-31 21:48:37,182 [main] INFO commands.PrintStatus (DurationInfo.java:close(100)) - get path status: duration 0:02:221
+```
+
+## gcscreds
+
+Help debug gcs credential bindings as set in `fs.gs.auth.service.account.private.key`
+
+it does ths with some better diagnostics of parsing problems.
+
+*warning: at -verbose, this prints your private key*
+
+This is by design, as encoding of the private key when stored in hadoop configuration or password JCEKS files may be the cause of authentication issues.
+
+
+
+```
+hadoop jar cloudstore-1.4.jar gcscreds gs://bucket/
+
+key uses \n for separator -gs connector must convert to line endings
+2022-01-19 17:55:51,016 [main] INFO gs.PemReader (PemReader.java:readNextSection(86)) - title match at line 1
+2022-01-19 17:55:51,020 [main] INFO gs.PemReader (PemReader.java:readNextSection(88)) - scanning for end
+Parsed private key -entry length 28 lines
+factory com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory@d706f19
+```
+
+## list
+
+Do a recursive listing of a path. Uses `listFiles(path, recursive)`, so for any object store
+which can do this as a deep paginated scan, is much, much faster.
+
+```
+Usage: list
+ -D <key=value> Define a property
+ -tokenfile <file> Hadoop token file to load
+ -limit <limit> limit of files to list
+ -verbose print verbose output
+ -xmlfile <file> XML config file to load
+```
+
+Example: list some of the AWS public landsat store.
+
+```bash
+> bin/hadoop jar cloudstore-1.4.jar list -limit 10 s3a://landsat-pds/
+
+Listing up to 10 files under s3a://landsat-pds/
+2019-04-05 21:32:14,523 [main] INFO tools.ListFiles (StoreDurationInfo.java:<init>(53)) - Starting: Directory list
+2019-04-05 21:32:14,524 [main] INFO tools.ListFiles (StoreDurationInfo.java:<init>(53)) - Starting: First listing
+2019-04-05 21:32:15,754 [main] INFO tools.ListFiles (DurationInfo.java:close(100)) - First listing: duration 0:01:230
+[1] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B1.TIF 63,786,465 alice alice [encrypted]
+[2] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B1.TIF.ovr 8,475,353 alice alice [encrypted]
+[3] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B10.TIF 35,027,713 alice alice [encrypted]
+[4] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B10.TIF.ovr 6,029,012 alice alice [encrypted]
+[5] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B10_wrk.IMD 10,213 alice alice [encrypted]
+[6] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B11.TIF 34,131,348 alice alice [encrypted]
+[7] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B11.TIF.ovr 5,891,395 alice alice [encrypted]
+[8] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B11_wrk.IMD 10,213 alice alice [encrypted]
+[9] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B1_wrk.IMD 10,213 alice alice [encrypted]
+[10] s3a://landsat-pds/L8/001/002/LC80010022016230LGN00/LC80010022016230LGN00_B2.TIF 64,369,211 alice alice [encrypted]
+2019-04-05 21:32:15,757 [main] INFO tools.ListFiles (DurationInfo.java:close(100)) - Directory list: duration 0:01:235
+
+Found 10 files, 124 milliseconds per file
+Data size 217,741,136 bytes, 21,774,113 bytes per file
+```
+
+## localhost
+
+Print out localhost information from java APIs and then the hadoop network APIs.
+
+## locatefiles
+
+Use the mapreduce `LocatedFileStatusFetcher` to scan for all non-hidden
+files under a path.
+
+
+See [locatefiles](src/main/site/locatefiles.md)
+
+## mkcsv
+
+Creates a large CSV file designed to trigger/validate the ABFS prefetching bug which
+came in HADOOP-17156.
+
+See [mkcsv](src/main/site/mkcsv.md)
+
+## pathcapability
+
+Probes a filesystem for offering a specific named capability on the given path.
+
+```bash
+bin/hadoop jar cloudstore-1.4.jar pathcapability
+Usage: pathcapability [options] <capability> <path>
+ -D <key=value> Define a property
+ -tokenfile <file> Hadoop token file to load
+ -verbose print verbose output
+ -xmlfile <file> XML config file to load
+```
+
+```bash
+hadoop jar cloudstore-1.4.jar pathcapability fs.s3a.capability.select.sql s3a://landsat-pds/
+
+Using filesystem s3a://landsat-pds
+Path s3a://landsat-pds/ has capability fs.s3a.capability.select.sql
+```
+
+The exit code of the command is 0 if the capability is present and -1 if absent.
+
+## put
+
+Uploads/copies a file, with the ability to set `createFile()` parameters.
+
+See [put](src/main/site/put.md)
+
+## tlsinfo
+
+Print out TLS information including X509 certificates.
+
+See [tlsinfo](src/main/site/tlsinfo.md)
+
+## AWS SDK commands
+
+See [S3 operations through the AWS V2 SDK](src/main/site/sdk.md).
+
+## Development and Future Work
+
+
+
+## Building
+
+See [BUILDING](BUILDING.md)
\ No newline at end of file
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..d71d328
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,194 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Security
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+"OPTIONAL" in this document are to be interpreted as described in
+RFC 2119.
+
+## Before filing a report (including AI-assisted reports)
+
+Cloudstore is a diagnostics CLI invoked manually by a cluster operator via
+`hadoop jar`.
+It is not a service, accepts no untrusted network input other than filenames, and the
+operator running it is trusted. Many findings that look like vulnerabilities
+in a server context are not vulnerabilities here.
+
+You *MUST NOT* file a report for:
+
+- Issues that require the operator to pass malicious arguments to their own
+ invocation, edit their own Hadoop site configuration, or place malicious
+ files on their own classpath.
+- Issues that require the attacker to have access to your hard disk or already
+ possess credentials to access the cluster/remote storage.
+- Output of `sessionkeys` or `gcscreds` containing credentials — these
+ commands exist to print credentials.
+- Output of any command run with `-debug` or `-verbose` containing secrets —
+ documented behaviour; file a regular bug, not a security report.
+- `storediag` revealing bucket names, endpoints, regions, or non-secret
+ configuration — that is the command's purpose.
+- Transitive CVEs in Hadoop, the AWS SDK, the GCS connector, or other
+ `provided`-scope dependencies — cloudstore does not ship them; they come
+ from the host installation
+- CVEs in remote stores interacted with.
+- Scanner output (Snyk, Dependabot, Trivy, Zizmor, etc.) without a working
+- reproducer against the current `main` branch.
+- Relaxed or introspective TLS behaviour in `tlsinfo` or `storediag` — these
+ commands exist to diagnose TLS configuration.
+- Theoretical findings ("an attacker who could X might Y") without a
+ reproduction.
+
+AI-assisted reports are accepted only if the submitter has verified the
+finding by hand against current source and includes a runnable reproducer
+running as a non-root user.
+
+*Unverified LLM-generated reports waste maintainer time and will be closed
+without further response.*
+
+A valid report includes:
+- the cloudstore version (git SHA).
+- the exact command run.
+- endpoint information.
+- the observed credential leak or other in-scope failure, and what was expected instead.
+
+## Reporting a bug in cloudstore
+
+Report security bugs in cloudstore to security@hadoop.apache.org
+
+## Reporting a bug in a third-party module
+
+Security bugs in third-party modules should be reported to their respective
+maintainers.
+
+## The Cloudstore threat model
+
+In the Cloudstore threat model, there are trusted elements such as the
+underlying operating system. Vulnerabilities that require the compromise
+of these trusted elements are outside the scope of the cloudstore threat
+model.
+
+* The cloudstore application may be deployed in production environments to
+ execute operations including `storediag`, `bandwidth`, `cloudup` and more.
+* It is executed by hand at a console on a host system connected to the cluster, or on a standalone system
+ with credentials to access the target store.
+* The output of these commands may be collected into a text file which is then passed
+ to internal or third-party support teams assisting in troubleshooting cloud connectivity and performance issues.
+ These are individuals who may be employed by separate companies, and who are not to be
+ given access to the stores. That is: they MUST NOT see credentials.
+* Storediag output is not expected to be attached to public issue reports, as they leak information about
+ cluster configuration, which would assist in malicious cluster reconnaissance.
+* Some commands `sessionkeys`, `gcscreds` do log secrets.
+
+### Deployment Threat Model
+
+Storediag is designed to be run by users without elevated privileges on a single host,
+either standalone or within/adjacent to a hadoop cluster.
+
+- The user is a normal unprivileged account; they have read/write access to part of the filesystem,
+including where temporary files are created by stores, and any paths in the local fs passed as parameters.
+- Input files shall be read as that user, relying on OS protection.
+- Paths provided for output shall be written by that user, relying on OS protection.
+- There may or may not be checks on overwriting output files; report a normal bug if this an issue.
+
+The user issues commands to examine the local fs, local hadoop configuration and perform operations against them, such as uploading files to remote stores, debugging connectivity issues or measuring bandwidth.
+
+For remote access the user will require access to the remote store
+- If it is an HDFS cluster where security is disabled, the user has full read/write access to the store,
+ this is inherent in security being disabled. Note: this is the default deployment standalone and
+ often the configuration used in transient cloud clusters where the entire DFS is for the single user,
+ short-lived, and protected from the rest of the network through firewall mechanisms.
+- For cloud storage, authentication may come in a number of ways
+ - host environment. For example AWS IAM. Here the cloud credentials are available to all.
+ - hadoop configuration files, cluster wide or local
+ - JCEKS files, local or in a filesystem whose schema does not match that of the store for which authentication is required. These SHOULD have passwords, but MAY NOT.
+ - authentication mechanisms picked by any underlying SDK. For example the AWS SDK examines environment
+ variables and configuration files under ` ~/.aws`.
+It is not a security issue if storediag can access cloud storage with credentials provided to it.
+It is a security issue in the storediag code if the credentials are logged other than in conditions previously listed (`-debug` mode and specific commands).
+If credential logging is performed within the storediag code, it is something to report, as it is the classic [CWE-532](https://cwe.mitre.org/data/definitions/532.html) : Insertion of Sensitive Information into Log File.
+If it is observed within the hadoop codebase, it is potentially more significant, so do report it to security@hadoop.apache.org, *provided the issue can be replicated with a build of the latest commit of hadoop trunk.
+If it is observed from within third party logs, file through their security reporting mechanisms, after verifying that
+ - the issue is in scope of their threat model and can be replicated with their latest release.
+
+### Development Environment Threat Model
+
+The project is built on developer systems, and in CI systems.
+
+* All external PRs SHALL be considered untrusted; their inputs MUST NOT be fed directly or indirectly to shell commands without sanitization.
+* Upstream dependencies from non ASF-projects MAY be subverted by malicious attacks; a cooldown period of at least 72 hours SHALL be kept before manual or automated dependency update.
+* ASF projects SHALL be considered trusted as their manual upvote release process includes an implicit buffer and defense against package
+ ecosystem worms.
+* Components such as maven plugins execute code on developer systems. Their security MUST be evaluated before adoption.
+* Third-party libraries which production code compiles against may be executed during testing. Their security MUST be evaluated.
+* Some developers may use VS.Code. This IDE has a notion of [a trusted workspace](https://code.visualstudio.com/docs/editing/workspaces/workspace-trust), which allows for files in the directory tree to declare executables, files such as `.env' and `tasks.json`. These files must be considered sensitive.
+*
+The CI build output is publicly visible, so the threat model includes unobfuscated logging of any cloud credentials provided by CI runs, or leakage of other secrets.
+
+The threat model includes the risk of subverted github actions and build tooling.
+* All inputs from pull requests, including titles, comments, authors and code SHALL be considered untrusted.
+* Any PR which adds VS.code specific mechanisms to execute code SHALL be rejected.
+* PRs which modify the maven pom.xml file or tests under `src/test/java` SHALL be audited for security risks.
+* Git checksum references MUST be made to GitHub actions, rather than tags; include the version as a comment so dependabot will track and maintain them.
+* [Zizmor](https://zizmor.sh/) SHALL be used to audit GHAs.
+* Github Action triggers on PRs MUST NOT be triggers which provide unrestricted github tokens to the actions.
+ For example, there MUST NOT be `pull_request_target`, `workflow_run`, or `issue_comment` triggers.
+* Github Actions SHALL follow GitHub's [secure use](https://docs.github.com/en/actions/reference/security/secure-use) guidelines, and in particular use [Intermediate Environment Variables](https://docs.github.com/en/actions/reference/security/secure-use#use-an-intermediate-environment-variable) to safely process untrusted inputs.
+
+
+
+
+## Not in the Threat Model
+
+* Any attack which requires the user to manually enter credentials or configuration options.
+* Vulnerabilities in dependencies.
+* Any attack which requires the local operating system to be configured in an unusual configuration/misconfigured as a step in the process.
+* Any attack which requires a hadoop site reconfiguration.
+* Any attack which requires the download and execution of external binaries.
+* Any attack contains an assumption about artifacts on the classpath which is not true in production systems (example, spring jars).
+* Any attack which grants more privileges accessing a remote cloud store/hdfs than the client has through the account credentials is has and/or the kerberos principal.
+* Any attack which accesses arbitrary data against an HDFS cluster for which kerberos is not enabled
+* Any attack against cloud storage where the credentials are supplied by the VM/container within which cloudstore is being executed.
+* Any report of credential printing in `sessionkeys`: it generates then prints session keys of limited duration.
+* Any report of credential printing in `gcscreds`: it prints GCS credentials as part of a diagnostics process.
+* Detailed printing of http communications between client and store when enabled.
+* Any vulnerability which cannot be reproduced in the latest version of the `main` branch.
+* Any vulnerability when running on a version of hadoop older than the latest release and which cannot be reproduced on a build of hadoop trunk branch.
+* Bugs in the test runs which do not leak secrets. File as normal bugs.
+
+## Do report
+
+- Any logging of secrets other than of * and length in storediag operations when `-reveal` is not passed in.
+- Any incompletely unobfuscated logging of secrets in storediag operations when `-reveal` is passed in.
+- Any unobfuscated logging of credentials in test runs.
+
+Note that some low level `-debug` options may print secrets; issues reported here should be considered bugs rather than security issues.
+
+## Scanner Calibration Rules
+
+A scanner should treat a finding as higher-confidence only if
+it plausibly shows one of the following:
+
+- Exposure of a secret or delegated credential to a new audience
+- CREATION of a new unauthorized capability in an component owned by this project.
+- The existence of this on HEAD of the branch `main`
+
+A finding should be downgraded or rejected by default if it instead depends
+primarily on:
+
+- Malformed-input robustness or denial-of-service behavior
+- A malicious catalog, metastore, or external service.
+- A principal that already has equivalent power through legitimate write or maintenance capabilities.
+- A vulnerability that only exists in previous releases.
diff --git a/dev-support/bump-version.sh b/dev-support/bump-version.sh
new file mode 100755
index 0000000..3be82fd
--- /dev/null
+++ b/dev-support/bump-version.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Bump the cloudstore project (pom) version. Accepts a SNAPSHOT or release
+# form, e.g. "1.4-SNAPSHOT" or "1.4". Does NOT rewrite site documentation —
+# release-time doc updates are handled by dev-support/update-site-docs.sh.
+#
+# Also updates BUILDING.md's fish `set -gx ver <v>` line so the release
+# command block always points at the bare release artifact name (a trailing
+# -SNAPSHOT is stripped before substituting).
+#
+# Usage: dev-support/bump-version.sh <new-version>
+# Example: dev-support/bump-version.sh 1.4-SNAPSHOT
+# Example: dev-support/bump-version.sh 1.4
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+ echo "usage: $0 <new-version>" >&2
+ echo " e.g. 1.4-SNAPSHOT (development)" >&2
+ echo " or 1.4 (release)" >&2
+ exit 2
+fi
+
+NEW="$1"
+
+# Accept X.Y or X.Y-SNAPSHOT only.
+if [[ ! "$NEW" =~ ^[0-9]+\.[0-9]+(-SNAPSHOT)?$ ]]; then
+ echo "error: version must be X.Y or X.Y-SNAPSHOT (got '$NEW')" >&2
+ exit 2
+fi
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+OLD="$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' \
+ --non-recursive exec:exec 2>/dev/null)"
+
+if [[ -z "$OLD" ]]; then
+ echo "could not read current version" >&2
+ exit 1
+fi
+
+if [[ "$OLD" == "$NEW" ]]; then
+ echo "current version already $NEW; nothing to do"
+ exit 0
+fi
+
+echo "bumping $OLD -> $NEW"
+
+mvn -q versions:set -DnewVersion="$NEW" -DgenerateBackupPoms=false
+
+# BUILDING.md's release block declares `set -gx ver <v>` in fish (`version`
+# is a reserved variable name in fish). That line is consumed by the
+# release command sequence, which names artifacts via `cloudstore-$ver.jar`,
+# so it must always reflect the bare release version — strip any trailing
+# -SNAPSHOT.
+RELEASE_VER="${NEW%-SNAPSHOT}"
+if [[ -f BUILDING.md ]] && grep -qE "^set -gx ver [0-9]+\.[0-9]+( |$| )" BUILDING.md; then
+ # Note: no \b word-boundary here — BSD/macOS sed does not support it and
+ # would silently match nothing. Anchor on the literal prefix instead.
+ sed -i.bak -E "s|^(set -gx ver )[0-9]+\.[0-9]+|\1${RELEASE_VER}|" BUILDING.md
+ rm -f BUILDING.md.bak
+ echo " rewrote BUILDING.md (set -gx ver ${RELEASE_VER})"
+fi
+
+echo "done. Review with: git diff"
+
+# Remind: if this was a release bump, the site docs still reference the
+# previous release until update-site-docs.sh is run.
+if [[ "$NEW" != *-SNAPSHOT ]]; then
+ echo
+ echo "Release version detected. To roll site docs forward, run:"
+ echo " dev-support/update-site-docs.sh ${NEW}"
+fi
diff --git a/dev-support/check-doc-versions.sh b/dev-support/check-doc-versions.sh
new file mode 100755
index 0000000..d059ae7
--- /dev/null
+++ b/dev-support/check-doc-versions.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fail if any committed markdown file references a `cloudstore-<X.Y>.jar`
+# whose version disagrees with the most recently released cloudstore
+# version (passed in $1; the pom plugin wires this to
+# ${cloudstore.docs.version}). Used as a verify-phase guard; pair with
+# dev-support/update-site-docs.sh which moves both the property and the
+# markdown references together at release time.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+ echo "usage: $0 <expected-version>" >&2
+ exit 2
+fi
+
+EXPECTED="$1"
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+# Search README, AGENTS, BUILDING, and src/site/markdown for stale references.
+# Matches any artifact filename: cloudstore-<v>.jar, cloudstore-<v>-cyclonedx.json,
+# .xml, and their .asc siblings. Anything whose <v> != $EXPECTED is drift.
+BAD=()
+while IFS= read -r -d '' f; do
+ while IFS= read -r match; do
+ # Strip leading "cloudstore-" then take the X.Y prefix.
+ rest="${match#cloudstore-}"
+ found="${rest%%[!0-9.]*}"
+ found="${found%.}" # drop a trailing '.' if the next char was '.jar' etc.
+ if [[ "$found" != "$EXPECTED" ]]; then
+ BAD+=("$f: $match (expected cloudstore-${EXPECTED}-)")
+ fi
+ done < <(grep -oE "cloudstore-[0-9]+\.[0-9]+(-cyclonedx\.(json|xml)|\.jar)(\.asc)?" "$f" || true)
+done < <(find . \( -path ./target -o -path ./releases -o -path './.git' \) -prune -o \
+ \( -name '*.md' -print0 \) 2>/dev/null)
+
+# BUILDING.md declares `set -gx ver <v>` in fish for the release block
+# (fish reserves `version`, so we use `ver`). That version must also match.
+if [[ -f BUILDING.md ]]; then
+ while IFS= read -r found; do
+ if [[ "$found" != "$EXPECTED" ]]; then
+ BAD+=("BUILDING.md: set -gx ver $found, expected $EXPECTED")
+ fi
+ done < <(grep -oE "^set -gx ver [0-9]+\.[0-9]+" BUILDING.md | awk '{print $4}')
+fi
+
+if [[ ${#BAD[@]} -gt 0 ]]; then
+ echo "Stale artifact version references in docs (expected ${EXPECTED}):" >&2
+ printf ' %s\n' "${BAD[@]}" >&2
+ echo >&2
+ echo "Run: dev-support/update-site-docs.sh ${EXPECTED}" >&2
+ exit 1
+fi
diff --git a/dev-support/checkstyle-noframes-sorted.xsl b/dev-support/checkstyle-noframes-sorted.xsl
new file mode 100644
index 0000000..b7826e3
--- /dev/null
+++ b/dev-support/checkstyle-noframes-sorted.xsl
@@ -0,0 +1,194 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html" indent="yes"/>
+<xsl:decimal-format decimal-separator="." grouping-separator="," />
+
+<xsl:key name="files" match="file" use="@name" />
+
+<!-- Checkstyle XML Style Sheet by Stephane Bailliez <sb...@apache.org> -->
+<!-- Part of the Checkstyle distribution found at http://checkstyle.sourceforge.net -->
+<!-- Usage (generates checkstyle_report.html): -->
+<!-- <checkstyle failonviolation="false" config="${check.config}"> -->
+<!-- <fileset dir="${src.dir}" includes="**/*.java"/> -->
+<!-- <formatter type="xml" toFile="${doc.dir}/checkstyle_report.xml"/> -->
+<!-- </checkstyle> -->
+<!-- <style basedir="${doc.dir}" destdir="${doc.dir}" -->
+<!-- includes="checkstyle_report.xml" -->
+<!-- style="${doc.dir}/checkstyle-noframes-sorted.xsl"/> -->
+
+<xsl:template match="checkstyle">
+ <html>
+ <head>
+ <style type="text/css">
+ .bannercell {
+ border: 0px;
+ padding: 0px;
+ }
+ body {
+ margin-left: 10;
+ margin-right: 10;
+ font:normal 80% arial,helvetica,sanserif;
+ background-color:#FFFFFF;
+ color:#000000;
+ }
+ .a td {
+ background: #efefef;
+ }
+ .b td {
+ background: #fff;
+ }
+ th, td {
+ text-align: left;
+ vertical-align: top;
+ }
+ th {
+ font-weight:bold;
+ background: #ccc;
+ color: black;
+ }
+ table, th, td {
+ font-size:100%;
+ border: none
+ }
+ table.log tr td, tr th {
+
+ }
+ h2 {
+ font-weight:bold;
+ font-size:140%;
+ margin-bottom: 5;
+ }
+ h3 {
+ font-size:100%;
+ font-weight:bold;
+ background: #525D76;
+ color: white;
+ text-decoration: none;
+ padding: 5px;
+ margin-right: 2px;
+ margin-left: 2px;
+ margin-bottom: 0;
+ }
+ </style>
+ </head>
+ <body>
+ <a name="top"></a>
+ <!-- jakarta logo -->
+ <table border="0" cellpadding="0" cellspacing="0" width="100%">
+ <tr>
+ <td class="bannercell" rowspan="2">
+ <!--a href="http://jakarta.apache.org/">
+ <img src="http://jakarta.apache.org/images/jakarta-logo.gif" alt="http://jakarta.apache.org" align="left" border="0"/>
+ </a-->
+ </td>
+ <td class="text-align:right"><h2>CheckStyle Audit</h2></td>
+ </tr>
+ <tr>
+ <td class="text-align:right">Designed for use with <a href='http://checkstyle.sourceforge.net/'>CheckStyle</a> and <a href='http://jakarta.apache.org'>Ant</a>.</td>
+ </tr>
+ </table>
+ <hr size="1"/>
+
+ <!-- Summary part -->
+ <xsl:apply-templates select="." mode="summary"/>
+ <hr size="1" width="100%" align="left"/>
+
+ <!-- Package List part -->
+ <xsl:apply-templates select="." mode="filelist"/>
+ <hr size="1" width="100%" align="left"/>
+
+ <!-- For each package create its part -->
+ <xsl:apply-templates select="file[@name and generate-id(.) = generate-id(key('files', @name))]" />
+
+ <hr size="1" width="100%" align="left"/>
+
+
+ </body>
+ </html>
+</xsl:template>
+
+
+
+ <xsl:template match="checkstyle" mode="filelist">
+ <h3>Files</h3>
+ <table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
+ <tr>
+ <th>Name</th>
+ <th>Errors</th>
+ </tr>
+ <xsl:for-each select="file[@name and generate-id(.) = generate-id(key('files', @name))]">
+ <xsl:sort data-type="number" order="descending" select="count(key('files', @name)/error)"/>
+ <xsl:variable name="errorCount" select="count(error)"/>
+ <tr>
+ <xsl:call-template name="alternated-row"/>
+ <td><a href="#f-{@name}"><xsl:value-of select="@name"/></a></td>
+ <td><xsl:value-of select="$errorCount"/></td>
+ </tr>
+ </xsl:for-each>
+ </table>
+ </xsl:template>
+
+
+ <xsl:template match="file">
+ <a name="f-{@name}"></a>
+ <h3>File <xsl:value-of select="@name"/></h3>
+
+ <table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
+ <tr>
+ <th>Error Description</th>
+ <th>Line</th>
+ </tr>
+ <xsl:for-each select="key('files', @name)/error">
+ <xsl:sort data-type="number" order="ascending" select="@line"/>
+ <tr>
+ <xsl:call-template name="alternated-row"/>
+ <td><xsl:value-of select="@message"/></td>
+ <td><xsl:value-of select="@line"/></td>
+ </tr>
+ </xsl:for-each>
+ </table>
+ <a href="#top">Back to top</a>
+ </xsl:template>
+
+
+ <xsl:template match="checkstyle" mode="summary">
+ <h3>Summary</h3>
+ <xsl:variable name="fileCount" select="count(file[@name and generate-id(.) = generate-id(key('files', @name))])"/>
+ <xsl:variable name="errorCount" select="count(file/error)"/>
+ <table class="log" border="0" cellpadding="5" cellspacing="2" width="100%">
+ <tr>
+ <th>Files</th>
+ <th>Errors</th>
+ </tr>
+ <tr>
+ <xsl:call-template name="alternated-row"/>
+ <td><xsl:value-of select="$fileCount"/></td>
+ <td><xsl:value-of select="$errorCount"/></td>
+ </tr>
+ </table>
+ </xsl:template>
+
+ <xsl:template name="alternated-row">
+ <xsl:attribute name="class">
+ <xsl:if test="position() mod 2 = 1">a</xsl:if>
+ <xsl:if test="position() mod 2 = 0">b</xsl:if>
+ </xsl:attribute>
+ </xsl:template>
+</xsl:stylesheet>
+
+
diff --git a/dev-support/checkstyle-suppressions.xml b/dev-support/checkstyle-suppressions.xml
new file mode 100644
index 0000000..ea80c9b
--- /dev/null
+++ b/dev-support/checkstyle-suppressions.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<!DOCTYPE suppressions PUBLIC
+ "-//Checkstyle//DTD SuppressionFilter Configuration 1.1//EN"
+ "https://checkstyle.org/dtds/suppressions_1_1.dtd">
+
+<suppressions>
+ <!-- Generated package-info.java is unlikely to need a Javadoc line. -->
+ <suppress checks="JavadocPackage" files="[\\/]src[\\/]test[\\/].*"/>
+ <suppress checks="JavadocPackage" files="[\\/]src[\\/][\w\-]*generated[\\/].*"/>
+
+ <!-- Test code: relax structural rules. Tests use lowercased static fields,
+ public fixture fields, and short test-helper utility classes. -->
+ <suppress checks="VisibilityModifier" files="[\\/]src[\\/]test[\\/]"/>
+ <suppress checks="ConstantName" files="[\\/]src[\\/]test[\\/]"/>
+ <suppress checks="HideUtilityClassConstructor" files="[\\/]src[\\/]test[\\/]"/>
+ <suppress checks="JavadocStyle" files="[\\/]src[\\/]test[\\/]"/>
+
+ <!-- Per-FS DiagnosticsInfo subclasses follow a project convention: each
+ declares `classnames`, `optionalClassnames`, `options`, `optionalCapabilities`
+ arrays in the same shape across stores so they read like a table. The
+ convention predates Hadoop's Sun-checks-derived ConstantName rule. -->
+ <suppress checks="ConstantName"
+ files="[\\/]src[\\/]main[\\/]java[\\/]org[\\/]apache[\\/]hadoop[\\/]fs[\\/]store[\\/]diag[\\/].*DiagnosticsInfo\.java"/>
+
+ <!-- Default-package CLI shims (e.g. dux.java, storediag.java). Each is a
+ lowercase-named one-liner that delegates to its target Tool class so
+ `hadoop jar cloudstore.jar dux ...` resolves at the bare class name.
+ The lowercase identifier is the public CLI surface and must stay
+ byte-stable. The default package also prevents a package-info.java. -->
+ <suppress checks="TypeName" files="[\\/]src[\\/]main[\\/]java[\\/][a-z][^\\/]*\.java"/>
+ <suppress checks="JavadocPackage" files="[\\/]src[\\/]main[\\/]java[\\/][a-z][^\\/]*\.java"/>
+</suppressions>
diff --git a/dev-support/checkstyle.xml b/dev-support/checkstyle.xml
new file mode 100644
index 0000000..ca4a0f1
--- /dev/null
+++ b/dev-support/checkstyle.xml
@@ -0,0 +1,231 @@
+<?xml version="1.0"?>
+<!DOCTYPE module PUBLIC
+ "-//Checkstyle//DTD Checkstyle Configuration 1.2//EN"
+ "https://checkstyle.org/dtds/configuration_1_2.dtd">
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+
+ Checkstyle configuration for Hadoop that is based on the sun_checks.xml file
+ that is bundled with Checkstyle and includes checks for:
+
+ - the Java Language Specification at
+ http://java.sun.com/docs/books/jls/second_edition/html/index.html
+
+ - the Sun Code Conventions at http://java.sun.com/docs/codeconv/
+
+ - the Javadoc guidelines at
+ http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
+
+ - the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
+
+ - some best practices
+
+ Checkstyle is very configurable. Be sure to read the documentation at
+ http://checkstyle.sf.net (or in your downloaded distribution).
+
+ Most Checks are configurable, be sure to consult the documentation.
+
+ To completely disable a check, just comment it out or delete it from the file.
+
+ Finally, it is worth reading the documentation.
+
+-->
+
+<module name="Checker">
+
+ <module name="SuppressWarningsFilter"/>
+
+ <!-- Checks that a package.html file exists for each package. -->
+ <!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml -->
+ <module name="JavadocPackage"/>
+
+ <!-- Checks whether files end with a new line. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
+ <!-- module name="NewlineAtEndOfFile"/-->
+
+ <!-- Checks that property files contain the same keys. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
+ <module name="Translation"/>
+
+ <!-- We have many existing long files, this check ends up being spurious -->
+ <!--<module name="FileLength">-->
+ <module name="FileTabCharacter"/>
+
+ <!-- Checks for line length violations. -->
+ <!-- See https://checkstyle.sourceforge.io/config_sizes.html#LineLength -->
+ <!-- Raised from Hadoop's 100 to 120: the Spotless / Eclipse Google formatter
+ occasionally produces 100–120 char lines on chained method calls
+ and string concatenations. -->
+ <module name="LineLength">
+ <property name="max" value="120"/>
+ <!-- Lines that are mostly a single URL are exempt: javadoc {@link http://...}
+ and `<a href="...">` references can't always be wrapped without
+ breaking the link. -->
+ <property name="ignorePattern" value="^\s*(\*|//).*https?://"/>
+ </module>
+
+ <module name="TreeWalker">
+
+ <module name="SuppressWarningsHolder"/>
+ <module name="SuppressionCommentFilter"/>
+ <module name="SuppressWithNearbyCommentFilter"/>
+
+
+ <!-- Checks for Javadoc comments. -->
+ <!-- See http://checkstyle.sf.net/config_javadoc.html -->
+ <module name="JavadocType">
+ <property name="scope" value="public"/>
+ <property name="allowMissingParamTags" value="true"/>
+ </module>
+ <module name="JavadocStyle"/>
+
+ <!-- Checks for Naming Conventions. -->
+ <!-- See http://checkstyle.sf.net/config_naming.html -->
+ <module name="ConstantName"/>
+ <module name="LocalFinalVariableName"/>
+ <module name="LocalVariableName"/>
+ <module name="MemberName"/>
+ <module name="MethodName"/>
+ <module name="PackageName"/>
+ <module name="ParameterName"/>
+ <module name="StaticVariableName"/>
+ <module name="TypeName"/>
+
+
+ <!-- Checks for Headers -->
+ <!-- See http://checkstyle.sf.net/config_header.html -->
+ <!-- <module name="Header"> -->
+ <!-- The follow property value demonstrates the ability -->
+ <!-- to have access to ANT properties. In this case it uses -->
+ <!-- the ${basedir} property to allow Checkstyle to be run -->
+ <!-- from any directory within a project. See property -->
+ <!-- expansion, -->
+ <!-- http://checkstyle.sf.net/config.html#properties -->
+ <!-- <property -->
+ <!-- name="headerFile" -->
+ <!-- value="${basedir}/java.header"/> -->
+ <!-- </module> -->
+
+ <!-- Following interprets the header file as regular expressions. -->
+ <!-- <module name="RegexpHeader"/> -->
+
+
+ <!-- Checks for imports -->
+ <!-- See http://checkstyle.sf.net/config_import.html -->
+ <module name="IllegalImport">
+ <property name="regexp" value="true"/>
+ <property name="illegalPkgs" value="sun"/>
+ </module>
+ <module name="RedundantImport"/>
+ <module name="UnusedImports"/>
+
+
+ <!-- Checks for Size Violations. -->
+ <!-- See http://checkstyle.sf.net/config_sizes.html -->
+ <!-- MethodLength disabled: accepting existing long methods (Cloudup.run,
+ StoreDiag.run). Refactoring is out of scope for the build-tooling
+ import. -->
+ <!--<module name="MethodLength"/>-->
+ <module name="ParameterNumber">
+ <property name="ignoreOverriddenMethods" value="true"/>
+ </module>
+
+
+ <!-- Checks for whitespace -->
+ <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+ <module name="EmptyForIteratorPad"/>
+ <module name="MethodParamPad"/>
+ <module name="NoWhitespaceAfter"/>
+ <module name="NoWhitespaceBefore"/>
+ <module name="ParenPad"/>
+ <module name="TypecastParenPad"/>
+ <!-- WhitespaceAfter disabled: handled by Spotless / Eclipse formatter,
+ which produces output Checkstyle's rule disagrees with on edge
+ cases (e.g. semicolons in for-loop headers). -->
+ <!--<module name="WhitespaceAfter">
+ <property name="tokens" value="COMMA, SEMI"/>
+ </module>-->
+
+
+ <!-- Modifier Checks -->
+ <!-- See http://checkstyle.sf.net/config_modifiers.html -->
+ <!-- This one is nitty, disable -->
+ <!-- <module name="ModifierOrder"/> -->
+ <module name="RedundantModifier"/>
+
+
+ <!-- Checks for blocks. You know, those {}'s -->
+ <!-- See http://checkstyle.sf.net/config_blocks.html -->
+ <module name="AvoidNestedBlocks">
+ <property name="allowInSwitchCase" value="true"/>
+ </module>
+ <module name="EmptyBlock"/>
+ <module name="LeftCurly"/>
+ <module name="NeedBraces"/>
+ <module name="RightCurly"/>
+
+
+ <!-- Checks for common coding problems -->
+ <!-- See http://checkstyle.sf.net/config_coding.html -->
+ <!-- module name="AvoidInlineConditionals"/-->
+ <module name="EmptyStatement"/>
+ <module name="EqualsHashCode"/>
+ <module name="HiddenField">
+ <property name="ignoreConstructorParameter" value="true"/>
+ <property name="ignoreSetter" value="true"/>
+ </module>
+ <module name="IllegalInstantiation"/>
+ <module name="InnerAssignment"/>
+ <module name="MissingSwitchDefault"/>
+ <module name="SimplifyBooleanExpression"/>
+ <module name="SimplifyBooleanReturn"/>
+
+ <!-- Checks for class design -->
+ <!-- See http://checkstyle.sf.net/config_design.html -->
+ <module name="FinalClass"/>
+ <module name="HideUtilityClassConstructor"/>
+ <module name="InterfaceIsType"/>
+ <!-- Relaxed from defaults: project routinely uses package-private and
+ protected fields on small data/value classes (EnvEntry, MinMeanMax,
+ EndpointProbe) and on the StoreEntryPoint base where subclasses
+ read the inherited state. public final fields are also accepted as
+ effectively-immutable since the project predates Java records. -->
+ <module name="VisibilityModifier">
+ <property name="packageAllowed" value="true"/>
+ <property name="protectedAllowed" value="true"/>
+ <property name="allowPublicFinalFields" value="true"/>
+ </module>
+
+
+ <!-- Miscellaneous other checks. -->
+ <!-- See http://checkstyle.sf.net/config_misc.html -->
+ <module name="ArrayTypeStyle"/>
+ <!-- Indentation disabled: Spotless (Eclipse formatter, Google profile)
+ owns indentation. Continuation indents in particular don't match
+ the values this rule expects. -->
+ <!--<module name="Indentation">
+ <property name="basicOffset" value="2" />
+ <property name="caseIndent" value="0" />
+ </module>-->
+ <!--<module name="TodoComment"/>-->
+ <module name="UpperEll"/>
+
+ </module>
+
+</module>
diff --git a/dev-support/eclipse-formatter.xml b/dev-support/eclipse-formatter.xml
new file mode 100644
index 0000000..7bb6804
--- /dev/null
+++ b/dev-support/eclipse-formatter.xml
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<profiles version="13">
+<profile kind="CodeFormatterProfile" name="GoogleStyle" version="13">
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_cascading_method_invocation_with_arguments.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_prefer_two_fragments" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_comment_inline_tags" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_local_variable_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter" value="1040"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type.count_dependent" value="1585|-1|1585"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression.count_dependent" value="16|4|80"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration.count_dependent" value="16|4|48"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration.count_dependent" value="16|4|49"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_cascading_method_invocation_with_arguments" value="16"/>
+<setting id="org.eclipse.jdt.core.compiler.source" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration.count_dependent" value="16|4|48"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_local_variable_annotation" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants.count_dependent" value="16|5|48"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="100"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation.count_dependent" value="16|4|48"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package" value="1585"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="16"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.assertIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_type_annotation" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_field_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.problem.enumIdentifier" value="error"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_generic_type_arguments" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.comment_new_line_at_start_of_html_paragraph" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comment_prefix" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_parameter_annotation" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method" value="1585"/>
+<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_parameter.count_dependent" value="1040|-1|1040"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_package.count_dependent" value="1585|-1|1585"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.force_if_else_statement_brace" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="3"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_package_annotation" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_type" value="1585"/>
+<setting id="org.eclipse.jdt.core.compiler.compliance" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_new_anonymous_class" value="20"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable.count_dependent" value="1585|-1|1585"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field.count_dependent" value="1585|-1|1585"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode" value="enabled"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="100"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="2"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_field" value="1585"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.compiler.codegen.targetPlatform" value="1.7"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
+<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration.count_dependent" value="16|4|48"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_method.count_dependent" value="1585|-1|1585"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.wrap_non_simple_member_annotation" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_annotations_on_local_variable" value="1585"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_generic_type_arguments.count_dependent" value="16|-1|16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration.count_dependent" value="16|5|80"/>
+<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="1"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.alignment_for_for_statement" value="16"/>
+<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
+<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
+<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
+</profile>
+</profiles>
diff --git a/dev-support/license-header.txt b/dev-support/license-header.txt
new file mode 100644
index 0000000..2379dda
--- /dev/null
+++ b/dev-support/license-header.txt
@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
diff --git a/dev-support/update-site-docs.sh b/dev-support/update-site-docs.sh
new file mode 100755
index 0000000..acd6466
--- /dev/null
+++ b/dev-support/update-site-docs.sh
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Roll the site documentation forward to a newly-released cloudstore version.
+# Rewrites every `cloudstore-<old>.jar` and `cloudstore-<old>-cyclonedx`
+# reference in README.md, AGENTS.md, BUILDING.md, and src/site/markdown/*.md,
+# and bumps the <cloudstore.docs.version> property in pom.xml so the verify
+# gate (dev-support/check-doc-versions.sh) tracks the documented release.
+#
+# Release versions only — SNAPSHOT artifacts are not documented downloads.
+#
+# Usage: dev-support/update-site-docs.sh <new-version>
+# Example: dev-support/update-site-docs.sh 1.4
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+ echo "usage: $0 <new-version> (release form only, e.g. 1.4)" >&2
+ exit 2
+fi
+
+NEW="$1"
+
+if [[ ! "$NEW" =~ ^[0-9]+\.[0-9]+$ ]]; then
+ echo "error: site docs track release versions only; got '$NEW'" >&2
+ echo " expected X.Y (no -SNAPSHOT, no qualifiers)" >&2
+ exit 2
+fi
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+OLD="$(mvn -q help:evaluate -Dexpression=cloudstore.docs.version \
+ -DforceStdout --non-recursive 2>/dev/null \
+ | tail -n1 | tr -d '[:space:]')"
+
+if [[ -z "$OLD" ]]; then
+ echo "could not read cloudstore.docs.version from pom.xml" >&2
+ exit 1
+fi
+
+if [[ "$OLD" == "$NEW" ]]; then
+ echo "site docs already at $NEW; nothing to do"
+ exit 0
+fi
+
+echo "rolling site docs $OLD -> $NEW"
+
+mvn -q versions:set-property -Dproperty=cloudstore.docs.version \
+ -DnewVersion="$NEW" -DgenerateBackupPoms=false
+
+# Files to rewrite. README + AGENTS + BUILDING at top level, plus all site
+# markdown. Add new doc files here as they appear.
+FILES=(
+ README.md
+ AGENTS.md
+ BUILDING.md
+)
+while IFS= read -r f; do FILES+=("$f"); done < <(find src/site/markdown -name '*.md')
+
+for f in "${FILES[@]}"; do
+ [[ -f "$f" ]] || continue
+ changed=0
+ # cloudstore-<v>.jar (covers cloudstore-<v>.jar.asc as a substring).
+ if grep -q "cloudstore-${OLD}\.jar" "$f"; then
+ sed -i.bak "s|cloudstore-${OLD}\.jar|cloudstore-${NEW}.jar|g" "$f"
+ changed=1
+ fi
+ # cloudstore-<v>-cyclonedx.* (covers .json/.xml + their .asc siblings).
+ if grep -q "cloudstore-${OLD}-cyclonedx" "$f"; then
+ sed -i.bak "s|cloudstore-${OLD}-cyclonedx|cloudstore-${NEW}-cyclonedx|g" "$f"
+ changed=1
+ fi
+ rm -f "${f}.bak"
+ if [[ $changed -eq 1 ]]; then
+ echo " rewrote $f"
+ fi
+done
+
+echo "done. Review with: git diff"
diff --git a/osv-scanner.toml b/osv-scanner.toml
new file mode 100644
index 0000000..9498b55
--- /dev/null
+++ b/osv-scanner.toml
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# osv-scanner
+# run with osv-scanner scan source -r .
+# combined with pom.xml exclusions, this reduces the number of scan alerts to 0.
+# see https://google.github.io/osv-scanner/configuration/
+
+[[PackageOverrides]]
+name = "^com\\.fasterxml\\.jackson\\.core:.*$"
+nameIsRegex = true
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+[[PackageOverrides]]
+name = "^com\\.google\\.guava:.*$"
+nameIsRegex = true
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+
+[[PackageOverrides]]
+name = "com.nimbusds:nimbus-jose-jwt"
+nameIsRegex = false
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+
+[[PackageOverrides]]
+name = "^io\\.netty:.*$"
+nameIsRegex = true
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+[[PackageOverrides]]
+name = "^org\\.bouncycastle:.*$"
+nameIsRegex = true
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+[[PackageOverrides]]
+name = "org.apache.commons:commons-configuration2"
+nameIsRegex = false
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+[[PackageOverrides]]
+name = "org.apache.commons:commons-lang3"
+nameIsRegex = false
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
+
+[[PackageOverrides]]
+name = "^org\\.eclipse\\.jetty:.*$"
+nameIsRegex = true
+ecosystem = "Maven"
+ignore = true
+reason = "transitive test-only dependency; CVEs do not affect the shipped JAR"
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..2e4e09a
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,890 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>cloudstore</artifactId>
+ <version>1.5-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>hadoop cloudstore</name>
+ <url>https://org.apache.hadoop/cloudstore</url>
+
+
+ <description>
+ Cloud Store operations, targeting Hadoop 3.4.0+.
+
+ </description>
+
+ <scm>
+ <connection>scm:git:https://github.com/apache/hadoop-cloudstore.git</connection>
+ <developerConnection>scm:git:https://github.com/apache/hadoop-cloudstore.git</developerConnection>
+ <url>https://github.com/apache/hadoop-cloudstore</url>
+ </scm>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <file.encoding>UTF-8</file.encoding>
+ <javac.version>1.8</javac.version>
+ <enforced.java.version>${javac.version}</enforced.java.version>
+
+ <test.build.dir>${project.build.directory}/test-dir</test.build.dir>
+ <test.build.data>${test.build.dir}</test.build.data>
+
+ <buildnumber.check>false</buildnumber.check>
+ <buildnumber.update>false</buildnumber.update>
+
+ <!-- Most recently released cloudstore version: the version users
+ download. Independent of ${project.version}, which moves through
+ X.Y-SNAPSHOT cycles during development. Bumped by
+ dev-support/update-site-docs.sh in lockstep with the markdown
+ references to cloudstore-X.Y.jar; the verify-phase
+ check-doc-versions execution below uses it to detect drift. -->
+ <cloudstore.docs.version>1.4</cloudstore.docs.version>
+
+ <!-- GPG signing key id; override on the command line with
+ -Dgpg.keyName=<keyid>. Empty default lets gpg pick the default
+ secret key when -Psign is active. -->
+ <gpg.keyName></gpg.keyName>
+
+ <!-- keep sorted -->
+ <apache-rat-plugin.version>0.18</apache-rat-plugin.version>
+ <assertj.version>3.27.7</assertj.version>
+ <avro.version>1.11.5</avro.version>
+ <awsv2sdk.version>2.29.52</awsv2sdk.version>
+ <!-- last version of checkstyle to run on Java 17-->
+ <checkstyle.version>12.3.1</checkstyle.version>
+ <checksum-maven-plugin.version>1.11</checksum-maven-plugin.version>
+ <doxia-module-markdown.version>1.12.0</doxia-module-markdown.version>
+ <gcs-connector.version>hadoop3-2.2.4</gcs-connector.version>
+ <hadoop.version>3.4.2</hadoop.version>
+ <junit.version>4.13.2</junit.version>
+
+ <!-- Add maven build version dependencies here, in order, prefixed with maven -->
+ <maven-buildnumber-plugin.version>3.3.0</maven-buildnumber-plugin.version>
+ <maven-checkstyle-plugin.version>3.6.0</maven-checkstyle-plugin.version>
+ <maven-compiler-plugin.version>3.15.0</maven-compiler-plugin.version>
+ <maven-cyclonedx-plugin.version>2.9.1</maven-cyclonedx-plugin.version>
+ <maven-enforcer-plugin.version>3.6.3</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>3.6.3</maven-exec-plugin.version>
+ <maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
+ <maven-fluido-skin.version>1.11.1</maven-fluido-skin.version>
+ <maven-gpg-plugin.version>3.2.8</maven-gpg-plugin.version>
+ <maven-jar-plugin.version>3.5.0</maven-jar-plugin.version>
+ <maven-javadoc-plugin.version>3.12.0</maven-javadoc-plugin.version>
+ <maven-plugin-build-helper.version>3.6.1</maven-plugin-build-helper.version>
+ <!-- 3.5.0 is the last Doxia-1.x-compatible release -->
+ <maven-project-info-reports-plugin.version>3.9.0</maven-project-info-reports-plugin.version>
+ <!-- pinned: 4.x (M-releases only) requires Doxia 2. -->
+ <maven-site-plugin.version>3.12.1</maven-site-plugin.version>
+ <maven-source-plugin.version>3.4.0</maven-source-plugin.version>
+ <maven-spotless-plugin.version>3.5.1</maven-spotless-plugin.version>
+ <maven-surefire-plugin.version>3.5.6</maven-surefire-plugin.version>
+ <maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion><groupId>com.github.pjfanning</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>com.google.code.gson</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>com.google.re2j</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>com.jcraft</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>org.apache.curator</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>org.apache.zookeeper</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>com.github.luben</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>org.apache.avro</groupId><artifactId>*</artifactId></exclusion>
+ <exclusion><groupId>org.apache.assertj</groupId><artifactId>*</artifactId></exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion><groupId>*</groupId><artifactId>*</artifactId></exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- HDFS for the MiniDFSCluster-backed contract tests. Both the regular
+ jar (DistributedFileSystem) and the test-jar (MiniDFSCluster +
+ HDFSContract) are needed. Test scope only — production code never
+ pulls in HDFS server bits. -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion><groupId>org.fusesource.leveldbjni</groupId><artifactId>*</artifactId></exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion><groupId>*</groupId><artifactId>*</artifactId></exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cloud-storage</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure-datalake</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion><groupId>org.apache.hadoop</groupId><artifactId>hadoop-aliyun</artifactId></exclusion>
+ <exclusion><groupId>org.apache.hadoop</groupId><artifactId>hadoop-cos</artifactId></exclusion>
+ <exclusion><groupId>org.apache.hadoop</groupId><artifactId>hadoop-huaweicloud</artifactId></exclusion>
+ <exclusion><groupId>org.apache.hadoop</groupId><artifactId>hadoop-tos</artifactId></exclusion>
+ <exclusion><groupId>com.microsoft</groupId><artifactId>*</artifactId></exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <version>${hadoop.version}</version>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>provided</scope>
+ </dependency>
+<!--
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+-->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>${assertj.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.google.cloud.bigdataoss/gcs-connector -->
+ <dependency>
+ <groupId>com.google.cloud.bigdataoss</groupId>
+ <artifactId>gcs-connector</artifactId>
+ <version>${gcs-connector.version}</version>
+ <scope>provided</scope>
+ <classifier>shaded</classifier>
+ <!-- who doesn't get that shaded means no dependencies?-->
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
+ <version>${awsv2sdk.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+
+
+ <!-- latest hadoop release -->
+ <profile>
+ <id>latest</id>
+ <properties>
+ <hadoop.version>3.5.0</hadoop.version>
+ </properties>
+ </profile>
+
+ <!-- next hadoop release -->
+ <profile>
+ <id>next</id>
+ <properties>
+ <hadoop.version>3.6.0-SNAPSHOT</hadoop.version>
+ </properties>
+ </profile>
+
+ <!-- cloudera 7.3.2 -->
+ <profile>
+ <id>7.3.2</id>
+ <properties>
+ <hadoop.version>3.4.1.7.3.2.0</hadoop.version>
+ </properties>
+ </profile>
+
+
+ <profile>
+ <id>trunk</id>
+ <activation>
+ <property>
+ <name>trunk</name>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>3.6.0-SNAPSHOT</hadoop.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>client</id>
+ <dependencies>
+
+ <!-- this is here to force a dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>release</id>
+ <properties>
+ <buildnumber.check>true</buildnumber.check>
+ <!-- update=false: do NOT `git pull` during the release build. doUpdate
+ runs `git pull` to refresh the working copy, which fails on setups
+ where the local branch has no matching upstream ref
+ ("fatal: couldn't find remote ref HEAD") and aborts the whole
+ build. The release only needs the clean-tree check (doCheck) and
+ the local revision; pulling is the releaser's job, done
+ deliberately before the build. -->
+ <buildnumber.update>false</buildnumber.update>
+ </properties>
+ <build>
+ <plugins>
+ <!-- Emit a CycloneDX SBOM alongside the release jar.
+ compile scope only: provided/test deps (Hadoop, AWS SDK,
+ GCS, junit, hdfs) are intentionally excluded because they
+ are not shipped inside the jar - they come from the host
+ Hadoop install or the test classpath. -->
+ <plugin>
+ <groupId>org.cyclonedx</groupId>
+ <artifactId>cyclonedx-maven-plugin</artifactId>
+ <version>${maven-cyclonedx-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>sbom-release</id>
+ <phase>package</phase>
+ <goals>
+ <goal>makeBom</goal>
+ </goals>
+ <configuration>
+ <includeCompileScope>true</includeCompileScope>
+ <includeProvidedScope>false</includeProvidedScope>
+ <includeRuntimeScope>false</includeRuntimeScope>
+ <includeSystemScope>false</includeSystemScope>
+ <includeTestScope>false</includeTestScope>
+ <outputFormat>all</outputFormat>
+ <outputName>${project.artifactId}-${project.version}-cyclonedx</outputName>
+ <schemaVersion>1.5</schemaVersion>
+ <projectType>library</projectType>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Emit SHA-256 checksums alongside the release jar and SBOM.
+ Downloaders verify integrity with `shasum -c <file>.sha256`
+ before trusting the artifact; the gpg signature on the
+ original file remains the authenticity primitive.
+ Runs in the `verify` phase, after the cyclonedx-maven-plugin
+ has produced the SBOM files in `package`, so all three
+ inputs exist on disk. -->
+ <plugin>
+ <groupId>net.nicoulaj.maven.plugins</groupId>
+ <artifactId>checksum-maven-plugin</artifactId>
+ <version>${checksum-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>release-artifact-checksums</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>files</goal>
+ </goals>
+ <configuration>
+ <algorithms>
+ <algorithm>SHA-256</algorithm>
+ </algorithms>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ <include>${project.artifactId}-${project.version}-cyclonedx.json</include>
+ <include>${project.artifactId}-${project.version}-cyclonedx.xml</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <csvSummary>false</csvSummary>
+ <xmlSummary>false</xmlSummary>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!-- Opt-in GPG signing of release artifacts. Activate alongside the
+ release profile: `mvn -Prelease,sign clean install -DskipTests`.
+ Signs the jar, pom, and every attached artifact (incl. the
+ CycloneDX SBOM); each gets a matching `.asc` in target/.
+ Requires gpg-agent running with the release key unlocked. -->
+ <profile>
+ <id>sign</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <version>${maven-gpg-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>sign-release-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ <configuration>
+ <gpgArguments>
+ <arg>--pinentry-mode=loopback</arg>
+ </gpgArguments>
+ <keyname>${gpg.keyName}</keyname>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!--
+ This is a profile to enable the use of the ASF snapshot and staging repositories
+ during a build. It is useful when testing against nightly or RC releases of dependencies.
+ -->
+ <profile>
+ <id>snapshots-and-staging</id>
+ <properties>
+ <!-- override point for ASF staging/snapshot repos -->
+ <asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
+ <asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
+ </properties>
+
+ <pluginRepositories>
+ <pluginRepository>
+ <id>ASF Staging</id>
+ <url>${asf.staging}</url>
+ </pluginRepository>
+ <pluginRepository>
+ <id>ASF Snapshots</id>
+ <url>${asf.snapshots}</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </pluginRepository>
+
+ </pluginRepositories>
+ <repositories>
+ <repository>
+ <id>ASF Staging</id>
+ <url>${asf.staging}</url>
+ </repository>
+ <repository>
+ <id>ASF Snapshots</id>
+ <url>${asf.snapshots}</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </repository>
+ </repositories>
+ </profile>
+ </profiles>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <configuration>
+ <source>${javac.version}</source>
+ <target>${javac.version}</target>
+ <useIncrementalCompilation>true</useIncrementalCompilation>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>${maven-enforcer-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <version>${maven-spotless-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <dependencies>
+ <!-- Override the bundled Checkstyle to a version that parses
+ modern Java syntax. Bumping in lockstep with the upstream
+ hadoop-build-tools release that produced the rules. -->
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <configLocation>${project.basedir}/dev-support/checkstyle.xml</configLocation>
+ <suppressionsLocation>${project.basedir}/dev-support/checkstyle-suppressions.xml</suppressionsLocation>
+ <suppressionsFileExpression>checkstyle.suppressions.file</suppressionsFileExpression>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <consoleOutput>true</consoleOutput>
+ <failsOnError>false</failsOnError>
+ <failOnViolation>false</failOnViolation>
+ <linkXRef>false</linkXRef>
+ <inputEncoding>UTF-8</inputEncoding>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>${maven-site-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>${doxia-module-markdown.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>${maven-project-info-reports-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>${maven-source-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${maven-javadoc-plugin.version}</version>
+ </plugin>
+
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
+ <configuration>
+ <!-- <archive>
+ <manifestFile>src/main/resources/META-INF/MANIFEST.MF</manifestFile>
+ </archive>-->
+ </configuration>
+ </plugin>
+
+ <!-- Surefire: capture every test's stdout/stderr into a per-test
+ *-output.txt next to the XML reports so CI artifacts include
+ the console log for failure triage. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkCount>1</forkCount>
+ <reuseForks>false</reuseForks>
+ <!-- <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>-->
+ <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+ <!--
+ <systemPropertyVariables>
+ </systemPropertyVariables>
+ -->
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+
+
+ </pluginManagement>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-build-environment</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireMavenVersion>
+ <version>[3.6.0,)</version>
+ </requireMavenVersion>
+ <!-- build java version, not the version for the artifact -->
+ <requireJavaVersion>
+ <version>[1.17,)</version>
+ </requireJavaVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.diffplug.spotless</groupId>
+ <artifactId>spotless-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>spotless-check</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <java>
+ <includes>
+ <include>src/**/*.java</include>
+ </includes>
+ <excludes>
+ <!-- Lowercase default-package CLI shims (e.g. dux.java, storediag.java).
+ Their class identifiers are the public CLI surface. -->
+ <exclude>src/main/java/*.java</exclude>
+ <!-- Generated avro -->
+ <exclude>target/generated-sources/**/*.java</exclude>
+ </excludes>
+ <eclipse>
+ <file>${project.basedir}/dev-support/eclipse-formatter.xml</file>
+ </eclipse>
+ <removeUnusedImports/>
+ <importOrder/>
+ <licenseHeader>
+ <file>${project.basedir}/dev-support/license-header.txt</file>
+ </licenseHeader>
+ </java>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>rat-check</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <consoleOutput>true</consoleOutput>
+ <inputExcludes>
+ <!-- VCS / IDE / OS scratch -->
+ <inputExclude>.git/**</inputExclude>
+ <inputExclude>.idea/**</inputExclude>
+ <inputExclude>**/.DS_Store</inputExclude>
+ <inputExclude>*.iml</inputExclude>
+ <inputExclude>**/*.iml</inputExclude>
+ <!-- Build artifacts -->
+ <inputExclude>target/**</inputExclude>
+ <inputExclude>**/target/**</inputExclude>
+ <inputExclude>pom.xml.versionsBackup</inputExclude>
+ <!-- Lists/manifests with no place for a header -->
+ <inputExclude>.gitignore</inputExclude>
+ <inputExclude>src/main/resources/cloudstore/required.txt</inputExclude>
+ <inputExclude>src/main/resources/META-INF/MANIFEST.MF</inputExclude>
+ <!-- Test fixture data -->
+ <inputExclude>src/test/resources/auth-keys.xml</inputExclude>
+ <inputExclude>src/test/resources/TestAuditLogs/**</inputExclude>
+ <!-- Third-party formatter profile from google/styleguide (Apache 2.0
+ licensed but not ASF-headered). Used by Spotless. -->
+ <inputExclude>dev-support/eclipse-formatter.xml</inputExclude>
+ <inputExclude>NOTICE*</inputExclude>
+ </inputExcludes>
+ </configuration>
+ </plugin>
+
+ <!-- Verify-phase guard: fail if any markdown doc references a
+ cloudstore-<X.Y>.jar that disagrees with
+ ${cloudstore.docs.version}, i.e. the most recently released
+ version. dev-support/bump-version.sh moves ${project.version}
+ through X.Y-SNAPSHOT cycles without touching docs;
+ dev-support/update-site-docs.sh bumps both
+ ${cloudstore.docs.version} and the markdown references
+ together at release time. -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${maven-exec-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-doc-versions</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${project.basedir}/dev-support/check-doc-versions.sh</executable>
+ <arguments>
+ <argument>${cloudstore.docs.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>${avro.version}</version>
+ <executions>
+ <execution>
+ <id>generate-avro-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <sourceDirectory>src/main/avro</sourceDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/avro</outputDirectory>
+ </configuration>
+ </plugin>
+
+ <!-- Build helper: add generated directory to compile source roots -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>${maven-plugin-build-helper.version}</version>
+ <executions>
+ <execution>
+ <id>add-generated-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/avro</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- kept failing -->
+ <!-- Execution default of goal org.codehaus.mojo:buildnumber-maven-plugin:3.3.0:create failed: begin 4, end 0, length 0 -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <version>${maven-buildnumber-plugin.version}</version>
+ <configuration>
+ <shortRevisionLength>7</shortRevisionLength>
+ <doCheck>${buildnumber.check}</doCheck>
+ <doUpdate>${buildnumber.update}</doUpdate>
+ <revisionOnScmFailure>UNKNOWN</revisionOnScmFailure>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.hadoop.fs.store.Cloudstore</mainClass>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ </manifest>
+ <manifestEntries>
+ <git-SHA-1>${buildNumber}</git-SHA-1>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+ <!-- Pinned to a Doxia-1.x-compatible release; see comment on
+ ${maven-project-info-reports-plugin.version}.
+ Dependency report omitted on purpose: it leaks the full provided
+ Hadoop tree (~1000 transitive entries) into the site without
+ adding value for a single-module diagnostics tool. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>${maven-project-info-reports-plugin.version}</version>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>index</report>
+ <report>summary</report>
+ <report>licenses</report>
+<!-- <report>team</report>
+ <report>scm</report>
+ <report>issue-management</report>-->
+ <report>ci-management</report>
+ <report>plugins</report>
+ <report>plugin-management</report>
+<!-- <report>distribution-management</report>-->
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <configuration>
+ <configLocation>${project.basedir}/dev-support/checkstyle.xml</configLocation>
+ <suppressionsLocation>${project.basedir}/dev-support/checkstyle-suppressions.xml</suppressionsLocation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <linkXRef>false</linkXRef>
+ </configuration>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>checkstyle</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ </plugin>
+ </plugins>
+
+ </reporting>
+</project>
diff --git a/src/main/avro/AvroDataSchema.avsc b/src/main/avro/AvroDataSchema.avsc
new file mode 100644
index 0000000..fab87b0
--- /dev/null
+++ b/src/main/avro/AvroDataSchema.avsc
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{
+ "type" : "record", "name" : "AvroS3LogEntryRecord",
+ "namespace" : "org.apache.hadoop.fs.store.audit",
+ "fields" : [
+ { "name" : "event", "type" :
+ { "type" : "long", "logicalType": "timestamp-micros" }},
+ { "name" : "owner", "type" : "string" },
+ { "name" : "bucket", "type" : "string" },
+ { "name" : "tstamp", "type" : "string" },
+ { "name" : "remoteip", "type" : "string" },
+ { "name" : "requester", "type" : "string" },
+ { "name" : "requestid", "type" : "string" },
+ { "name" : "verb", "type" : "string" },
+ { "name" : "key", "type" : "string" },
+ { "name" : "requesturi", "type" : "string" },
+ { "name" : "http", "type" : "string" },
+ { "name" : "awserrorcode", "type" : "string" },
+ { "name" : "bytessent", "type" : ["long", "null"] },
+ { "name" : "objectsize", "type" : ["long", "null"] },
+ { "name" : "totaltime", "type" : ["long", "null"] },
+ { "name" : "turnaroundtime" , "type" : ["long", "null"] },
+ { "name" : "referrer", "type" : "string" },
+ { "name" : "useragent", "type" : "string" },
+ { "name" : "version", "type" : "string" },
+ { "name" : "hostid", "type" : "string" },
+ { "name" : "sigv", "type" : "string" },
+ { "name" : "cypher", "type" : "string" },
+ { "name" : "auth", "type" : "string" },
+ { "name" : "endpoint", "type" : "string" },
+ { "name" : "tls", "type" : "string" },
+ { "name" : "tail", "type" : "string" },
+ { "name" : "audit", "type" : {"type": "map", "values": "string"} }
+ ]
+}
diff --git a/src/main/java/org/apache/hadoop/fs/gs/GsCredDiag.java b/src/main/java/org/apache/hadoop/fs/gs/GsCredDiag.java
new file mode 100644
index 0000000..a82a18b
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/gs/GsCredDiag.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.gs;
+
+import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CONFIG_PREFIX;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.fs.store.diag.GCSDiagnosticsInfo.SERVICE_ACCOUNT_PRIVATE_KEY;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.auth.oauth2.Credential;
+import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.CredentialFactory;
+import com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.HadoopCredentialConfiguration;
+import com.google.cloud.hadoop.repackaged.gcs.com.google.common.collect.ImmutableList;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.service.launcher.LauncherExitCodes;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Debug GS credentials. Getting past the "java.io.IOException: Invalid PKCS8 data" error message
+ * with gs auth is a nightmare. This class tries to debug it.
+ * <p>
+ * Needs the shaded gs jar on the classpath as we call its internals.
+ * <p>
+ * * code includes methods copied from com.google.cloud.hadoop.util.CredentialFactory
+ * <a href="https://cloud.google.com/docs/authentication/production">...</a>
+ * <a href="https://developers.google.com/accounts/docs/application-default-credentials">...</a>
+ * <p>
+ * Security note, -verbose will print the credentials. Sometimes this is necessary to work out what
+ * is going wrong. Do not file security reports about this behavior.
+ */
+public class GsCredDiag extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GsCredDiag.class);
+
+ /** Manage your data and permissions in Google Cloud Storage. */
+ public static final String DEVSTORAGE_FULL_CONTROL =
+ "https://www.googleapis.com/auth/devstorage.full_control";
+
+ public static final String USAGE = "Usage: gcscreds\n" + STANDARD_OPTS + "\t<gs path>\n"
+ + "warning: the -verbose option will print the credentials. DO NOT SHARE THE OUTPUT";
+
+ public GsCredDiag() {
+ createCommandFormat(1, 1);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.isEmpty()) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+ String key;
+ final char[] password = conf.getPassword(SERVICE_ACCOUNT_PRIVATE_KEY);
+ if (password != null) {
+ key = new String(password).trim();
+ } else {
+ key = null;
+ }
+ if (key != null) {
+ if (isVerbose()) {
+ warn("The private key printed below is a critical secret. Do not share it");
+
+ // insecure
+ println("private key is <%s>", key);
+ }
+ if (key.contains("\\n")) {
+ println("key uses \\n for separator -gs connector must convert to line endings");
+ key = key.replace("\\n", System.lineSeparator());
+ if (isVerbose()) {
+
+ // insecure
+
+ println("modified private key is <%s>", key);
+ }
+ }
+ } else {
+ println("no key in %s", SERVICE_ACCOUNT_PRIVATE_KEY);
+ return LauncherExitCodes.EXIT_NOT_FOUND;
+ }
+ Reader reader = new StringReader(key);
+ PemReader.Section section = PemReader.readFirstSectionAndClose(reader, "PRIVATE KEY");
+ if (!section.isSuccess()) {
+ errorln("Failed to parse private key: %s", section.getMessage());
+ return LauncherExitCodes.EXIT_NOT_ACCEPTABLE;
+ }
+
+ println("Parsed private key -entry length %d lines", section.getLines());
+ final CredentialFactory factory =
+ HadoopCredentialConfiguration.getCredentialFactory(conf, GCS_CONFIG_PREFIX);
+ println("factory %s", factory);
+ Credential credential = factory.getCredential(ImmutableList.of(DEVSTORAGE_FULL_CONTROL));
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new GsCredDiag(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/gs/PemReader.java b/src/main/java/org/apache/hadoop/fs/gs/PemReader.java
new file mode 100644
index 0000000..86eb2c6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/gs/PemReader.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.gs;
+
+import com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.util.Base64;
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Reader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * derivative of the google pem reader which requires a section title and provides diagnostics.
+ */
+public class PemReader implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PemReader.class);
+
+ private static final String BEGIN = "-----BEGIN %s-----";
+ private static final String END = "-----END %s-----";
+
+ /** Reader. */
+ private final BufferedReader reader;
+
+ /** @param reader reader */
+ public PemReader(Reader reader) {
+ this.reader = new BufferedReader(reader);
+ }
+
+ /**
+ * Reads the next section in the PEM file, optionally based on a title to look for.
+ *
+ * @param titleToLookFor title to look for.
+ * @return next section or {@code null} for end of file
+ */
+ public Section readNextSection(String titleToLookFor) throws IOException {
+ StringBuilder keyBuilder = new StringBuilder();
+ int lines = 0;
+ String scan = String.format(BEGIN, titleToLookFor);
+ while (true) {
+ String line = reader.readLine();
+ lines++;
+ if (line == null) {
+ return new Section(titleToLookFor, lines, "no match for line '" + scan + "'");
+ }
+ if (scan.equals(line)) {
+ break;
+ }
+ }
+
+ LOG.info("title match at line {}", lines);
+
+ LOG.info("scanning for end ");
+ int start = lines;
+ scan = String.format(END, titleToLookFor);
+
+ while (true) {
+ String line = reader.readLine();
+ lines++;
+ if (line == null) {
+ break;
+ }
+ if (scan.equals(line)) {
+ return new Section(titleToLookFor, lines, Base64.decodeBase64(keyBuilder.toString()));
+ }
+
+ keyBuilder.append(line);
+ }
+ return new Section(titleToLookFor, lines - start, "no match for line <" + scan + ">");
+ }
+
+ /**
+ * Reads the first section in the PEM file, optionally based on a title to look for, and then
+ * closes the reader.
+ *
+ * @param titleToLookFor title to look for or {@code null} for any title
+ * @param reader reader
+ * @return first section found or {@code null} for none found
+ */
+ public static Section readFirstSectionAndClose(Reader reader, String titleToLookFor)
+ throws IOException {
+
+ try (PemReader pemReader = new PemReader(reader)) {
+ return pemReader.readNextSection(titleToLookFor);
+ }
+ }
+
+ /**
+ * Closes the reader.
+ *
+ * <p>
+ * To ensure that the stream is closed properly, call {@link #close()} in a finally block.
+ */
+ public void close() throws IOException {
+ reader.close();
+ }
+
+ /** Section in the PEM file. */
+ public static final class Section {
+
+ private final boolean success;
+
+ /** Title. */
+ private final String title;
+
+ private final int lines;
+
+ /** Base64-decoded bytes. */
+ private final byte[] base64decodedBytes;
+
+ private final String message;
+
+ /**
+ * @param title title
+ * @param lines lines read
+ * @param base64decodedBytes base64-decoded bytes
+ */
+ Section(String title, final int lines, byte[] base64decodedBytes) {
+ this.success = true;
+ this.title = title;
+ this.lines = lines;
+ this.base64decodedBytes = base64decodedBytes;
+ message = "";
+ }
+
+ public Section(final String title, final int lines, final String message) {
+ this.title = title;
+ this.lines = lines;
+ this.message = message;
+
+ success = false;
+ base64decodedBytes = null;
+ }
+
+ /** Returns the title. */
+ public String getTitle() {
+ return title;
+ }
+
+ /** Returns the base64-decoded bytes (modifiable array). */
+ public byte[] getBase64DecodedBytes() {
+ return base64decodedBytes;
+ }
+
+ @Override
+ public String toString() {
+ return "Section{" + "success=" + success + ", title='" + title + '\'' + ", lines=" + lines
+ + ", message='" + message + '\'' + '}';
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public int getLines() {
+ return lines;
+ }
+
+ public byte[] getBase64decodedBytes() {
+ return base64decodedBytes;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/gs/package-info.java b/src/main/java/org/apache/hadoop/fs/gs/package-info.java
new file mode 100644
index 0000000..2dac2ce
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/gs/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * GCS-specific diagnostics support for the cloudstore CLI commands.
+ */
+package org.apache.hadoop.fs.gs;
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/BucketMetadata.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/BucketMetadata.java
new file mode 100644
index 0000000..322f5f6
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/BucketMetadata.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInternals;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+
+/**
+ * Debug bucket settings; v2 sdk.
+ */
+public class BucketMetadata extends StoreEntryPoint {
+
+ public static final String USAGE = "Usage: bucketmetadata [-debug] <path>";
+
+ public BucketMetadata() {
+ createCommandFormat(1, 1);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> argList = processArgs(args, 1, 1, USAGE);
+ final Configuration conf = createPreconfiguredConfig();
+
+ // path on the CLI
+ Path path = new Path(argList.get(0));
+ heading("Getting bucket info for %s", path);
+ FileSystem fs = path.getFileSystem(conf);
+ if (!(fs instanceof S3AFileSystem)) {
+ println("Filesystem for path %s is not an S3A FileSystem %s", path, fs);
+ return -1;
+ }
+ S3AFileSystem s3a = (S3AFileSystem) fs;
+ final S3AInternals internals = s3a.getS3AInternals();
+ final HeadBucketResponse response = internals.getBucketMetadata();
+
+ println("Bucket metadata from S3");
+ println("Region %s%nLocation Name %s%nLocation Type %s%n", response.bucketRegion(),
+ response.bucketLocationName(), response.bucketLocationTypeAsString());
+
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new BucketMetadata(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/BulkDeleteCommand.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/BulkDeleteCommand.java
new file mode 100644
index 0000000..ac9fb28
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/BulkDeleteCommand.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.fs.store.CommonParameters.PAGE;
+import static org.apache.hadoop.fs.store.StoreUtils.readLines;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BulkDelete;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.service.launcher.LauncherExitCodes;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Bulk delete.
+ */
+public class BulkDeleteCommand extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BulkDelete.class);
+
+ public static final String USAGE =
+ "Usage: bulkdelete [-verbose] [-page <pagesize>] <path> <file>\n"
+ + "<file> is a text file with full/relative paths to files under <path>\n"
+ + " Empty lines and lines starting with # are ignored.\n"
+ + " As are root paths of stores\n"
+ + "<pagesize> is the page size if less than the store page size ";
+
+ public BulkDeleteCommand() {
+ createCommandFormat(2, 2);
+ addValueOptions(PAGE);
+ }
+
+ /**
+ * @param args command specific arguments.
+ * @return
+ * @throws Exception
+ */
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> argList = processArgs(args, 2, 2, USAGE);
+
+ final Configuration conf = createPreconfiguredConfig();
+
+ int pageLimit = getIntOption(PAGE, 0);
+ // path on the CLI
+ Path path = new Path(argList.get(0));
+ heading("Bulk delete under %s", path);
+
+ String filename = argList.get(1);
+ File file = new File(filename);
+ if (!file.exists()) {
+ errorln("File not found \"%s\"", filename);
+ return LauncherExitCodes.EXIT_NOT_FOUND;
+ }
+
+ // get the list of paths to delete
+ final List<String> toDelete = readLines(file);
+ final int files = toDelete.size();
+ println("%d files to delete", files);
+ if (files == 0) {
+ // no files
+ return 0;
+ }
+
+ FileSystem fs = path.getFileSystem(conf);
+ fs.setWorkingDirectory(new Path("/"));
+
+ final StoreDurationInfo executionDuration = new StoreDurationInfo();
+
+ final List<Path> pathsToDelete = toDelete.stream().map(l -> fs.makeQualified(new Path(l)))
+ .filter(p -> !p.isRoot()).collect(Collectors.toList());
+
+ final BulkDelete deleter = fs.createBulkDelete(path);
+ int pageSize = deleter.pageSize();
+ println("Store page size = %s", pageSize);
+ if (pageLimit > 0 && pageLimit < pageSize) {
+ pageSize = pageLimit;
+ println("Delete page size = %s", pageSize);
+ }
+ println();
+
+ int batches = 0;
+ int failures = 0;
+ for (int i = 0; i < files; i += pageSize) {
+ batches++;
+ int end = Math.min(files, i + pageSize);
+ List<Path> batch = pathsToDelete.subList(i, end);
+ batch.forEach(p -> println(" %s", p));
+
+ // delete one batch
+ final List<Map.Entry<Path, String>> result = deleter.bulkDelete(batch);
+
+ final int rs = result.size();
+ failures += rs;
+
+ result.forEach(entry -> {
+ println(" Failed to delete %s: ", entry.getKey(), entry.getValue());
+ });
+ }
+
+ executionDuration.finished();
+
+ heading("Summary");
+ println("Bulk delete of %d file(s) finished, duration: %s", files,
+ executionDuration.getDurationString());
+ println("Batch count: %d. Failure count: %d", batches, failures);
+ if (isVerbose()) {
+ heading("Statistics");
+ println("\nStatistics\n%s", ioStatisticsToPrettyString(retrieveIOStatistics(fs)));
+ }
+
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new BulkDeleteCommand(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/CsvVersionWriter.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/CsvVersionWriter.java
new file mode 100644
index 0000000..208d0d1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/CsvVersionWriter.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.isDirMarker;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.Date;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tools.csv.SimpleCsvWriter;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+
+/**
+ * Write to csv.
+ */
+final class CsvVersionWriter extends ListAndProcessVersionedObjects.NoopProcessor {
+
+ private final SimpleCsvWriter csv;
+
+ private final DateFormat df = new SimpleDateFormat("yyyy-MM-ddZhh:mm:ss");
+
+ private final boolean logDirs;
+
+ private final boolean logDeleted;
+
+ long index = 0;
+
+ CsvVersionWriter(final OutputStream out, final boolean closeOutput, String separator,
+ final boolean logDirs, final boolean logDeleted) throws IOException {
+ this.logDirs = logDirs;
+ this.logDeleted = logDeleted;
+ csv = new SimpleCsvWriter(out, separator, "\n", true, closeOutput);
+ csv.columns("index", "key", "path", "restore", "latest", "size", "tombstone", "directory",
+ "date", "timestamp", "version", "etag");
+ csv.newline();
+ }
+
+ @Override
+ public void close() throws IOException {
+ csv.close();
+ }
+
+ public boolean process(ObjectVersion summary, Path path, final boolean deleteMarker)
+ throws IOException {
+ final boolean dirMarker = isDirMarker(summary);
+ if (dirMarker && !logDirs) {
+ return false;
+ }
+ if (deleteMarker && !logDeleted) {
+ return false;
+ }
+ csv.columnL(++index);
+ csv.column(summary.key());
+ csv.column(path);
+ csv.columnB(!deleteMarker && !dirMarker);
+ csv.columnB(summary.isLatest());
+ csv.columnL(summary.size());
+ csv.columnB(deleteMarker);
+ csv.columnB(dirMarker);
+ final Instant lastModified = summary.lastModified();
+ csv.column(df.format(Date.from(lastModified)));
+ csv.columnL(lastModified.getEpochSecond());
+ final String versionId = summary.versionId();
+ csv.column(versionId);
+ csv.column(summary.eTag());
+ csv.newline();
+ return true;
+ }
+
+ private long getIndex() {
+ return index;
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/DeleteObject.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/DeleteObject.java
new file mode 100644
index 0000000..badb678
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/DeleteObject.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+
+/**
+ * Deletes the objects on the command line.
+ */
+public class DeleteObject extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DeleteObject.class);
+
+ public static final String USAGE = "Usage: deleteobject <S3A path>";
+
+ public DeleteObject() {
+ createCommandFormat(1, 1);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = processArgs(args, 1, 1, USAGE);
+ final Configuration conf = createPreconfiguredConfig();
+
+ final Path source = new Path(paths.get(0));
+ S3AFileSystem fs = (S3AFileSystem) source.getFileSystem(conf);
+ S3Client s3 = fs.getS3AInternals().getAmazonS3Client("DeleteObjects");
+ Invoker.once("delete", source.toString(), () -> s3.deleteObject(
+ DeleteObjectRequest.builder().bucket(fs.getBucket()).key(fs.pathToKey(source)).build()));
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new DeleteObject(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/IamPolicy.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/IamPolicy.java
new file mode 100644
index 0000000..cee9fcf
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/IamPolicy.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.util.EnumSet;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.auth.RoleModel;
+import org.apache.hadoop.fs.s3a.auth.delegation.AWSPolicyProvider;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Ask the target FS to generate its policy rules.
+ */
+public class IamPolicy extends StoreEntryPoint {
+
+ public static final String USAGE = "Usage: iampolicy <S3A path>";
+
+ public IamPolicy() {
+ createCommandFormat(1, 1);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = processArgs(args, 1, 1, USAGE);
+
+ final Configuration conf = createPreconfiguredConfig();
+
+ final Path source = new Path(paths.get(0));
+ S3AFileSystem fs = (S3AFileSystem) source.getFileSystem(conf);
+ EnumSet<AWSPolicyProvider.AccessLevel> access = EnumSet.of(AWSPolicyProvider.AccessLevel.READ,
+ AWSPolicyProvider.AccessLevel.WRITE, AWSPolicyProvider.AccessLevel.ADMIN);
+
+ List<RoleModel.Statement> rules = fs.listAWSPolicyRules(access);
+ String ruleset = new RoleModel().toJson(new RoleModel.Policy(rules));
+ println(ruleset);
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new IamPolicy(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/InternalAccess.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/InternalAccess.java
new file mode 100644
index 0000000..51480c5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/InternalAccess.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import software.amazon.awssdk.services.s3.S3Client;
+
+/**
+ * Internal accessor to S3 state.
+ */
+public class InternalAccess {
+
+ private final S3AFileSystem filesystem;
+
+ public InternalAccess(final S3AFileSystem filesystem) {
+ this.filesystem = checkNotNull(filesystem);
+ }
+
+ /**
+ * Returns the S3 client used by this filesystem. This is for internal use within the S3A code
+ * itself.
+ *
+ * @return AmazonS3Client
+ */
+ public S3Client getAmazonS3Client() {
+ return filesystem.getS3AInternals().getAmazonS3Client("Diagnostics");
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListAndProcessVersionedObjects.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListAndProcessVersionedObjects.java
new file mode 100644
index 0000000..3940ada
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListAndProcessVersionedObjects.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.createListVersionsRequest;
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.isDirMarker;
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.objectRepresentsDirectory;
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.pathToKey;
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.result;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Iterator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.diag.Printout;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteMarkerEntry;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+
+public class ListAndProcessVersionedObjects {
+
+ private final String name;
+
+ private final Printout out;
+
+ private final S3AFileSystem fs;
+
+ private final Path source;
+
+ private final SummaryProcessor processor;
+
+ private final Instant ageLimit;
+
+ private final long limit;
+
+ private long tombstones = 0;
+
+ private long fileTombstones = 0;
+
+ private long hidden = 0;
+
+ private long hiddenData = 0;
+
+ private long hiddenZeroByteFiles = 0;
+
+ private long dirMarkers = 0;
+
+ private long hiddenDirMarkers = 0;
+
+ private long objectCount = 0;
+
+ private long totalSize = 0;
+
+ private long processedCount;
+
+ public ListAndProcessVersionedObjects(final String name, final Printout out,
+ final S3AFileSystem fs, final Path source, final SummaryProcessor processor,
+ final Instant ageLimit, final long limit) {
+ this.name = requireNonNull(name);
+ this.out = requireNonNull(out);
+ this.fs = requireNonNull(fs);
+ this.source = requireNonNull(source);
+ this.processor = requireNonNull(processor);
+ this.ageLimit = requireNonNull(ageLimit);
+ this.limit = limit;
+ }
+
+ public long getTombstones() {
+ return tombstones;
+ }
+
+ public long getFileTombstones() {
+ return fileTombstones;
+ }
+
+ public long getDirTombstones() {
+ return tombstones - fileTombstones;
+ }
+
+ public long getHidden() {
+ return hidden;
+ }
+
+ public long getHiddenData() {
+ return hiddenData;
+ }
+
+ public long getHiddenZeroByteFiles() {
+ return hiddenZeroByteFiles;
+ }
+
+ public long getDirMarkers() {
+ return dirMarkers;
+ }
+
+ public long getHiddenDirMarkers() {
+ return hiddenDirMarkers;
+ }
+
+ public long execute() throws IOException {
+ if (ageLimit.toEpochMilli() > 0) {
+ out.println("Skipping entries older than %s", ageLimit);
+ if (ageLimit.isAfter(Instant.now())) {
+ // happens if millis were used or something else went wrong
+ out.warn("the filter time is greater than the current time");
+ }
+ }
+ final S3Client s3 = fs.getS3AInternals().getAmazonS3Client(name);
+ String key = pathToKey(source);
+ ListObjectVersionsRequest request =
+ createListVersionsRequest(source.toUri().getHost(), key, null);
+
+ final Iterator<ListObjectVersionsResponse> objectVersions =
+ s3.listObjectVersionsPaginator(request).iterator();
+
+ processedCount = 0;
+ out.heading("Processing %s", source);
+ try {
+
+ boolean finished = false;
+ while (!finished && objectVersions.hasNext()) {
+ final ListObjectVersionsResponse page = objectVersions.next();
+ for (ObjectVersion summary : page.versions()) {
+ // maybe skip
+ if (summary.lastModified().isBefore(ageLimit)) {
+ continue;
+ }
+ objectCount++;
+
+ final long size = summary.size();
+ totalSize += size;
+
+ final boolean isDirMarker = isDirMarker(summary);
+ dirMarkers += result(isDirMarker);
+ if (!summary.isLatest()) {
+ if (!isDirMarker) {
+ hidden++;
+ hiddenData += size;
+ hiddenZeroByteFiles += result(size == 0);
+ } else {
+ hiddenDirMarkers++;
+ }
+ }
+ if (processor.process(summary, fs.keyToQualifiedPath(summary.key()), false)) {
+ processedCount++;
+ if (limit > 0 && processedCount >= limit) {
+ finished = true;
+ break;
+ }
+ }
+ }
+ for (DeleteMarkerEntry tombstone : page.deleteMarkers()) {
+ tombstones++;
+ fileTombstones += result(!objectRepresentsDirectory(tombstone.key(), 0));
+ if (processor.processTombstone(fs.keyToQualifiedPath(tombstone.key()), tombstone)) {
+ processedCount++;
+ if (limit > 0 && processedCount >= limit) {
+ finished = true;
+ break;
+ }
+ }
+ }
+ }
+ } finally {
+ processor.close();
+ }
+ return processedCount;
+ }
+
+ public long getObjectCount() {
+ return objectCount;
+ }
+
+ public long getTotalSize() {
+ return totalSize;
+ }
+
+ static class NoopProcessor implements SummaryProcessor {
+
+ @Override
+ public boolean process(final ObjectVersion summary, final Path path,
+ final boolean isDeleteMarker) throws IOException {
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {}
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListMultiparts.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListMultiparts.java
new file mode 100644
index 0000000..97ded95
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListMultiparts.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.store.CommonParameters.LIMIT;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+import java.io.PrintStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.fs.tools.csv.SimpleCsvWriter;
+import org.apache.hadoop.util.ToolRunner;
+import software.amazon.awssdk.services.s3.model.ListPartsResponse;
+import software.amazon.awssdk.services.s3.model.MultipartUpload;
+import software.amazon.awssdk.services.s3.model.Part;
+
+public class ListMultiparts extends StoreEntryPoint {
+
+ public static final String AGE = "age";
+
+ public static final String DIRS = "dirs";
+
+ public static final String OUTPUT = "out";
+
+ public static final String QUIET = "q";
+
+ public static final String SEPARATOR = "separator";
+
+ public static final String SINCE = "since";
+
+ public static final String USAGE = "Usage: listmultiparts <path>\n" + STANDARD_OPTS
+ + optusage(LIMIT, "limit", "limit of files to list") + optusage(OUTPUT, "file", "output file")
+ + optusage(QUIET, "quiet output") + optusage(SEPARATOR, "string", "Separator if not <tab>")
+ + optusage(AGE, "seconds", "Only include versions created in this time interval")
+ + optusage(SINCE, "epoch-time", "Only include versions after this time");
+
+ public static final String NAME = "listmultiparts";
+
+ public ListMultiparts() {
+ createCommandFormat(1, 1, QUIET);
+ addValueOptions(LIMIT, OUTPUT, SEPARATOR, AGE, SINCE);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.size() < 1) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+ // stop auditing rejecting client direct calls.
+ conf.setBoolean(FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS, false);
+ int limit = getIntOption(LIMIT, 0);
+ boolean logDirs = hasOption(DIRS);
+ long since = getLongOption(SINCE, 0);
+
+ final Optional<Long> age = getOptionalLong(AGE);
+ if (age.isPresent()) {
+ if (since > 0) {
+ errorln("Only one of " + AGE + " and " + SINCE + " may be specified");
+ return EXIT_USAGE;
+ }
+ since = Instant.now().minusSeconds(age.get()).getEpochSecond();
+ }
+ Instant ageLimit = Instant.ofEpochSecond(since);
+
+ S3AFileSystem fs = null;
+ final Path source = new Path(paths.get(0));
+ final String name = NAME;
+ fs = (S3AFileSystem) source.getFileSystem(conf);
+ try (StoreDurationInfo duration = new StoreDurationInfo(getOut(), name)) {
+ SummaryProcessor processor;
+
+ final String output = getOption(OUTPUT);
+ PrintStream dest;
+ boolean closeOutput;
+ if (output != null) {
+ // writing to a dir
+ final Path destPath = new Path(output);
+ final FileSystem destFS = destPath.getFileSystem(conf);
+ final FSDataOutputStream dataOutputStream =
+ destFS.createFile(destPath).overwrite(true).recursive().build();
+ dest = new PrintStream(dataOutputStream);
+ closeOutput = true;
+ println("Saving output to %s", destPath);
+ } else {
+ dest = getOut();
+ closeOutput = false;
+ }
+ final String separator = getOptional(SEPARATOR).orElse("\t");
+ final SimpleCsvWriter csv = new SimpleCsvWriter(dest, separator, "\n", true, closeOutput);
+ final DateFormat df = new SimpleDateFormat("yyyy-MM-ddZhh:mm:ss");
+ final RemoteIterator<MultipartUpload> uploads = fs.listUploads(fs.pathToKey(source));
+ final MultipartProcessor multipartProcessor = new MultipartProcessor(fs);
+
+ int entries = 0;
+ csv.columns("index", "key", "size", "blocks", "date", "id");
+ csv.newline();
+ long totalParts = 0;
+ long totalSize = 0;
+ while (uploads.hasNext()) {
+ final MultipartUpload upload = uploads.next();
+ long bytes = 0;
+ long parts = 0;
+ final String key = upload.key();
+ final MultipartProcessor.PartIterator listing =
+ multipartProcessor.partListing(key, upload.uploadId());
+ while (listing.hasNext()) {
+ final ListPartsResponse part = listing.next();
+ for (Part summary : part.parts()) {
+ parts++;
+ bytes += summary.size();
+ }
+ csv.columns(++entries, key);
+ csv.columnL(bytes);
+ csv.columnL(parts);
+ csv.column(df.format(upload.initiated()));
+ csv.column(upload.uploadId());
+ totalParts += parts;
+ totalSize += bytes;
+ }
+ }
+ println();
+ println("Found %,d uploads under %s with total size %,d bytes in %,d parts", entries, source,
+ totalSize, totalParts);
+ println();
+ } finally {
+ maybeDumpStorageStatistics(fs);
+ }
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new ListMultiparts(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListObjects.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListObjects.java
new file mode 100644
index 0000000..c9e62d9
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListObjects.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.Invoker.once;
+import static org.apache.hadoop.fs.store.CommonParameters.LIMIT;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CommonPrefix;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public class ListObjects extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ListObjects.class);
+
+ public static final String PURGE = "purge";
+
+ public static final String DELETE = "delete";
+
+ public static final String QUIET = "q";
+
+ public static final String USAGE = "Usage: listobjects <path>\n" + STANDARD_OPTS
+ + optusage(DELETE, "delete the objects") + optusage(LIMIT, "limit", "limit of files to list")
+ + optusage(PURGE, "purge directory markers") + optusage(QUIET, "quiet output");
+
+ public ListObjects() {
+ createCommandFormat(1, 1, PURGE, DELETE, QUIET);
+ addValueOptions(LIMIT);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = processArgs(args, 1, 1, USAGE);
+
+ final Configuration conf = createPreconfiguredConfig();
+ // stop auditing rejecting client direct calls.
+ conf.setBoolean(FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS, false);
+ int limit = getOptional(LIMIT).map(Integer::valueOf).orElse(0);
+ int deletePageSize = 500;
+ boolean purge = hasOption(PURGE);
+ boolean quiet = hasOption(QUIET);
+
+ boolean delete = hasOption(DELETE);
+ String deleteType = "";
+ if (delete) {
+ deleteType = "objects";
+ println("objects will be deleted");
+ purge = false;
+ } else if (purge) {
+ deleteType = "markers";
+ println("directory markers will be purged");
+ }
+
+ List<String> markers = new ArrayList<>();
+ S3AFileSystem fs = null;
+ final Path source = new Path(paths.get(0));
+ try (StoreDurationInfo duration = new StoreDurationInfo(LOG, "listobjects")) {
+ fs = (S3AFileSystem) source.getFileSystem(conf);
+ String bucket = fs.getBucket();
+ final S3Client s3 = new InternalAccess(fs).getAmazonS3Client();
+ String key = S3ListingSupport.pathToKey(source);
+ if (!key.endsWith("/")) {
+ key += "/";
+ }
+ ListObjectsV2Request request =
+ S3ListingSupport.createListObjectsV2Request(source.toUri().getHost(), key, null);
+
+ final ObjectListingIterator objects = new ObjectListingIterator(s3, source, request);
+ List<String> prefixes = new ArrayList<>();
+ int objectCount = 0;
+ long size = 0;
+ List<ObjectIdentifier> objectsToDelete = new ArrayList<>(deletePageSize);
+
+ heading("Listing Objects under %s", source);
+ boolean finished = false;
+ while (!finished && objects.hasNext()) {
+ final ListObjectsV2Response page = objects.next();
+ for (S3Object summary : page.contents()) {
+ objectCount++;
+ if (limit > 0 && objectCount >= limit) {
+ finished = true;
+ break;
+ }
+ size += summary.size();
+ String k = summary.key();
+
+ if (!quiet) {
+
+ // print the output, if verbose call getFileStatus for the s3a view.
+ String extra;
+ if (isVerbose()) {
+ extra = "\t" + fs.getFileStatus(S3ListingSupport.keyToPath(k));
+ } else {
+ extra = "";
+ }
+ println("[%05d] %s%s", objectCount, S3ListingSupport.stringify(summary), extra);
+ }
+ if (objectRepresentsDirectory(summary)) {
+ markers.add(k);
+ }
+ if (delete) {
+ objectsToDelete.add(ObjectIdentifier.builder().key(k).build());
+ if (objectsToDelete.size() >= deletePageSize) {
+ final List<ObjectIdentifier> keyVersions = objectsToDelete;
+ objectsToDelete = new ArrayList<>(deletePageSize);
+ delete(s3, bucket, keyVersions);
+ }
+ }
+ }
+ page.commonPrefixes().stream().map(CommonPrefix::prefix).forEach(prefixes::add);
+ }
+
+ if (delete && !objectsToDelete.isEmpty()) {
+ // final batch
+ delete(s3, bucket, objectsToDelete);
+ }
+ println("");
+ String action = delete ? "Deleted" : "Found";
+ println("%s %d objects with total size %d bytes", action, objectCount, size);
+ if (!prefixes.isEmpty()) {
+ heading("%s prefixes", prefixes.size());
+ for (String prefix : prefixes) {
+ println(prefix);
+ }
+ }
+ if (!markers.isEmpty()) {
+ int markerCount = markers.size();
+ heading("Marker count: %d", markerCount);
+ if (purge) {
+ println("Purging all directory markers");
+ }
+
+ List<ObjectIdentifier> kv = new ArrayList<>(markerCount);
+ for (String marker : markers) {
+ println(marker);
+ if (purge) {
+ kv.add(ObjectIdentifier.builder().key(marker).build());
+ if (kv.size() >= deletePageSize) {
+ delete(s3, bucket, kv);
+ kv.clear();
+ }
+ }
+ }
+ if (purge) {
+ delete(s3, bucket, kv);
+ } else {
+ if (!delete) {
+ println("\nTo delete these markers, rerun with the option -%s", PURGE);
+ }
+ }
+ } else if (purge) {
+ heading("No markers found to purge");
+ }
+ } finally {
+ maybeDumpStorageStatistics(fs);
+ }
+
+ return 0;
+ }
+
+ /**
+ * Delete the entries; clears the list. No-op if empty.
+ *
+ * @param s3
+ * @param bucket
+ * @param kv
+ */
+ public void delete(S3Client s3, String bucket, List<ObjectIdentifier> kv) throws IOException {
+ if (kv.isEmpty()) {
+ return;
+ }
+ try (StoreDurationInfo d = new StoreDurationInfo(LOG, "deleting %s objects", kv.size())) {
+ final DeleteObjectsRequest request = DeleteObjectsRequest.builder().bucket(bucket)
+ .delete(Delete.builder().objects(kv).quiet(true).build()).build();
+ once("delete", kv.toString(), () -> s3.deleteObjects(request));
+ kv.clear();
+ }
+ }
+
+ /**
+ * Predicate: does the object represent a directory?.
+ *
+ * @return true if it meets the criteria for being an object
+ */
+ public boolean objectRepresentsDirectory(S3Object summary) {
+ final String name = summary.key();
+ final long size = summary.size();
+ return S3ListingSupport.objectRepresentsDirectory(name, size);
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new ListObjects(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListVersions.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListVersions.java
new file mode 100644
index 0000000..b8c993c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ListVersions.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.sdk.ListMultiparts.AGE;
+import static org.apache.hadoop.fs.store.CommonParameters.LIMIT;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS;
+
+import java.io.PrintStream;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * List versions of S3 objects.
+ */
+public class ListVersions extends StoreEntryPoint {
+
+
+ public static final String DELETED = "deleted";
+
+ public static final String DIRS = "dirs";
+
+ public static final String OUTPUT = "out";
+
+ public static final String QUIET = "q";
+
+ public static final String SEPARATOR = "separator";
+
+ public static final String SINCE = "since";
+
+ public static final String USAGE = "Usage: listversions <path>\n" + STANDARD_OPTS
+ + optusage(DELETED, "include delete markers") + optusage(DIRS, "include directory markers")
+ + optusage(LIMIT, "limit", "limit of files to list") + optusage(OUTPUT, "file", "output file")
+ + optusage(QUIET, "quiet output") + optusage(SEPARATOR, "string", "Separator if not <tab>")
+ + optusage(AGE, "seconds", "Only include versions created in this time interval")
+ + optusage(SINCE, "epoch-time", "Only include versions after this time");
+
+ public static final String NAME = "listversions";
+
+ public ListVersions() {
+ createCommandFormat(1, 1, DELETED, DIRS, QUIET);
+ addValueOptions(LIMIT, OUTPUT, SEPARATOR, AGE, SINCE);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.isEmpty()) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+ // stop auditing rejecting client direct calls.
+ conf.setBoolean(FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS, false);
+ int limit = getIntOption(LIMIT, 0);
+ boolean quiet = hasOption(QUIET);
+ boolean logDeleted = hasOption(DELETED);
+ boolean logDirs = hasOption(DIRS);
+ long since = getLongOption(SINCE, 0);
+
+ final Optional<Long> age = getOptionalLong(AGE);
+ if (age.isPresent()) {
+ if (since > 0) {
+ errorln("Only one of " + AGE + " and " + SINCE + " may be specified");
+ return EXIT_USAGE;
+ }
+ since = Instant.now().minusSeconds(age.get()).getEpochSecond();
+ }
+ Instant ageLimit = Instant.ofEpochSecond(since);
+
+ S3AFileSystem fs = null;
+ final Path source = new Path(paths.get(0));
+ final String name = NAME;
+ fs = (S3AFileSystem) source.getFileSystem(conf);
+ try (StoreDurationInfo duration = new StoreDurationInfo(getOut(), name)) {
+ SummaryProcessor processor;
+
+ if (!quiet) {
+ final String output = getOption(OUTPUT);
+ PrintStream dest;
+ boolean closeOutput;
+ if (output != null) {
+ // writing to a dir
+ final Path destPath = new Path(output);
+ final FileSystem destFS = destPath.getFileSystem(conf);
+ final FSDataOutputStream dataOutputStream =
+ destFS.createFile(destPath).overwrite(true).recursive().build();
+ dest = new PrintStream(dataOutputStream);
+ closeOutput = true;
+ println("Saving output to %s", destPath);
+ } else {
+ dest = getOut();
+ closeOutput = false;
+ }
+ final String separator = getOptional(SEPARATOR).orElse("\t");
+ processor = new CsvVersionWriter(dest, closeOutput, separator, logDirs, logDeleted);
+ } else {
+ processor = new ListAndProcessVersionedObjects.NoopProcessor();
+ }
+
+ final ListAndProcessVersionedObjects listing =
+ new ListAndProcessVersionedObjects(name, this, fs, source, processor, ageLimit, limit);
+
+ listing.execute();
+
+ println();
+ println("Found %,d objects under %s with total size %,d bytes", listing.getObjectCount(),
+ source, listing.getTotalSize());
+
+ println("Hidden file count %,d with hidden data size %,d bytes", listing.getHidden(),
+ listing.getHiddenData());
+ println("Hidden zero-byte file count %,d", listing.getHiddenZeroByteFiles());
+ println("Hidden directory markers %,d", listing.getHiddenDirMarkers());
+ println("Tombstone entries %,d comprising %,d files and %,d dir markers",
+ listing.getTombstones(), listing.getFileTombstones(), listing.getDirTombstones());
+ println();
+ } finally {
+ maybeDumpStorageStatistics(fs);
+ }
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new ListVersions(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/MkBucket.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/MkBucket.java
new file mode 100644
index 0000000..f64235a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/MkBucket.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.CreateBucketResponse;
+
+/**
+ * Create the s3 bucket.
+ */
+public class MkBucket extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MkBucket.class);
+
+ public static final String USAGE = "Usage: mkbucket <region> <S3A path>";
+
+ public MkBucket() {
+ createCommandFormat(2, 2);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.size() != 2) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+ conf.setInt(S3A_BUCKET_PROBE, 0);
+ conf.setBoolean(FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS, false);
+
+ final String region = paths.get(0);
+ final String bucketPath = paths.get(1);
+ final Path source = new Path(bucketPath);
+ S3AFileSystem fs = (S3AFileSystem) source.getFileSystem(conf);
+ final S3Client client = fs.getS3AInternals().getAmazonS3Client("mkbucket");
+ final String bucketName = source.toUri().getHost();
+ final CreateBucketRequest request =
+ CreateBucketRequest.builder().bucket(bucketName).createBucketConfiguration(
+ CreateBucketConfiguration.builder().locationConstraint(region).build()).build();
+ CreateBucketResponse bucket;
+ try (StoreDurationInfo ignored = new StoreDurationInfo(LOG, "Creating bucket %s", bucketName)) {
+ bucket = Invoker.once("delete", source.toString(), () -> client.createBucket(request));
+ }
+ println("Created bucket %s", bucket);
+
+ return 0;
+ }
+
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new MkBucket(), args);
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/MultipartProcessor.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/MultipartProcessor.java
new file mode 100644
index 0000000..1168a58
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/MultipartProcessor.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListPartsRequest;
+import software.amazon.awssdk.services.s3.model.ListPartsResponse;
+
+/**
+ * List multipart uploads.
+ */
+public class MultipartProcessor {
+
+ private final S3AFileSystem fs;
+
+ private final S3Client amazonS3;
+
+ public MultipartProcessor(final S3AFileSystem fs) {
+ this.fs = fs;
+ amazonS3 = new InternalAccess(fs).getAmazonS3Client();
+ }
+
+ public PartIterator partListing(String key, final String id) {
+ return new PartIterator(key, id);
+ }
+
+ public class PartIterator implements RemoteIterator<ListPartsResponse> {
+
+ private final String key;
+
+ private final String id;
+
+ private boolean firstListing = true;
+
+ private ListPartsResponse partListing;
+
+ private ListPartsRequest request;
+
+ PartIterator(final String key, final String id) {
+ this.key = key;
+ this.id = id;
+ }
+
+ public void listFirst() {
+ request = ListPartsRequest.builder().bucket(fs.getBucket()).key(key).uploadId(id).build();
+ partListing = amazonS3.listParts(request);
+ }
+
+ public void listNext() {
+ request = request.toBuilder().partNumberMarker(partListing.nextPartNumberMarker()).build();
+ partListing = amazonS3.listParts(request);
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return firstListing || (partListing != null && partListing.isTruncated());
+ }
+
+ @Override
+ public ListPartsResponse next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ if (firstListing) {
+ firstListing = false;
+ listFirst();
+ } else {
+ // not first listing, so there's a valid, truncated part
+ listNext();
+ }
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/ObjectListingIterator.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ObjectListingIterator.java
new file mode 100644
index 0000000..bc32362
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/ObjectListingIterator.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.io.IOException;
+import java.util.NoSuchElementException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+
+/**
+ * Wraps up AWS `ListObjects` requests in a remote iterator which will ask for more listing data if
+ * needed.
+ * <p>
+ * That is:
+ * <p>
+ * 1. The first invocation of the {@link #next()} call will return the results of the first request,
+ * the one created during the construction of the instance.
+ * <p>
+ * 2. Second and later invocations will continue the ongoing listing.
+ * <p>
+ * 3. The {@link #hasNext()} predicate returns true for the initial call, where {@link #next()} will
+ * return the initial results. It declares that it has future results iff the last executed request
+ * was truncated.
+ * <p>
+ * Thread safety: none.
+ */
+class ObjectListingIterator implements RemoteIterator<ListObjectsV2Response> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ObjectListingIterator.class);
+
+ /** The path listed. */
+ private final Path listPath;
+
+ private final S3Client s3;
+
+ private final ListObjectsV2Request request;
+
+ private String continuationToken;
+
+ /** The most recent listing results. */
+ private ListObjectsV2Response response;
+
+ /** Indicator that this is the first listing. */
+ private boolean firstListing = true;
+
+ /**
+ * Count of how many listings have been requested (including initial result).
+ */
+ private int listingCount = 1;
+
+ /**
+ * Maximum keys in a request.
+ */
+ private final int maxKeys;
+
+ /**
+ * Constructor -calls `listObjects()` on the request to populate the initial set of results/fail
+ * if there was a problem talking to the bucket.
+ *
+ * @param listPath path of the listing
+ * @param request initial request to make
+ */
+ ObjectListingIterator(S3Client s3, Path listPath, ListObjectsV2Request request) {
+ this.listPath = listPath;
+ this.maxKeys = S3ListingSupport.MAX_KEYS;
+ this.s3 = s3;
+ this.request = request;
+ this.response = s3.listObjectsV2(this.request);
+ continuationToken = response.continuationToken();
+ }
+
+ /**
+ * Declare that the iterator has data if it is either is the initial iteration or it is a later
+ * one and the last listing obtained was incomplete.
+ *
+ * @throws IOException never: there is no IO in this operation.
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ return firstListing || response.isTruncated();
+ }
+
+ /**
+ * Ask for the next listing. For the first invocation, this returns the initial set, with no
+ * remote IO. For later requests, S3 will be queried, hence the calls may block or fail.
+ *
+ * @return the next object listing.
+ * @throws IOException if a query made of S3 fails.
+ * @throws NoSuchElementException if there is no more data to list.
+ */
+ @Override
+ public ListObjectsV2Response next() throws IOException {
+ if (firstListing) {
+ // on the first listing, don't request more data.
+ // Instead just clear the firstListing flag so that it future calls
+ // will request new data.
+ firstListing = false;
+ } else {
+ Invoker.once("listObjects()", listPath.toString(), () -> {
+ if (!response.isTruncated()) {
+ // nothing more to request: fail.
+ throw new NoSuchElementException("No more results in listing of " + listPath);
+ }
+ // need to request a new set of objects.
+ LOG.debug("[{}], Requesting next {} objects under {}", listingCount, maxKeys, listPath);
+ final ListObjectsV2Request r2 =
+ request.toBuilder().continuationToken(continuationToken).build();
+ response = s3.listObjectsV2(r2);
+ continuationToken = response.continuationToken();
+ listingCount++;
+ LOG.debug("New listing status: {}", this);
+ });
+ }
+ return response;
+ }
+
+ @Override
+ public String toString() {
+ return "Object listing iterator against " + listPath + "; listing count " + listingCount
+ + "; isTruncated=" + response.isTruncated();
+ }
+
+ /**
+ * Get the path listed.
+ *
+ * @return the path used in this listing.
+ */
+ public Path getListPath() {
+ return listPath;
+ }
+
+ /**
+ * Get the count of listing requests.
+ *
+ * @return the counter of requests made (including the initial lookup).
+ */
+ public int getListingCount() {
+ return listingCount;
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/Regions.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/Regions.java
new file mode 100644
index 0000000..4c8795c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/Regions.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.regions.providers.AwsProfileRegionProvider;
+import software.amazon.awssdk.regions.providers.AwsRegionProvider;
+import software.amazon.awssdk.regions.providers.InstanceProfileRegionProvider;
+import software.amazon.awssdk.regions.providers.SystemSettingsRegionProvider;
+
+/**
+ * Debug region settings; v2 sdk.
+ */
+public class Regions extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Regions.class);
+
+ public static final String USAGE = "Usage: regions";
+
+ private String foundRegion = null;
+
+ private String regionProvider = null;
+
+ public Regions() {
+ createCommandFormat(0, 0);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ processArgs(args, 0, 0, USAGE);
+ heading("Determining AWS region for SDK clients");
+ println("This uses same region resolution chain and ordering as in the AWS client.");
+
+ printRegion("SystemSettingsRegionProvider", new SystemSettingsRegionProvider(),
+ "System property aws.region");
+ printRegion("AwsProfileRegionProvider", new AwsProfileRegionProvider(),
+ "Region info in ~/.aws/config");
+ printRegion("InstanceMetadataRegionProvider", new InstanceProfileRegionProvider(),
+ "EC2 metadata; will only work in AWS infrastructure");
+ if (foundRegion == null) {
+ heading("Region was NOT FOUND");
+ warn("AWS region was not determined through SDK region chain");
+ warn("This may not work");
+ return 50; // 500 server error
+ } else {
+
+ // all good
+ heading("Region found: \"%s\"", foundRegion);
+ println("Region was determined by %s as \"%s\"", regionProvider, foundRegion);
+ }
+ return 0;
+ }
+
+ private boolean printRegion(String name, AwsRegionProvider provider, String comment) {
+ heading("Determining region using %s", name);
+ println("%s", comment);
+ String region = null;
+ try (StoreDurationInfo ignored = new StoreDurationInfo(LOG, "%s.getRegion()", name)) {
+ region = provider.getRegion().id();
+ } catch (Exception e) {
+ warn("Provider %s raised an exception %s", name, e);
+ LOG.info("Provider {} raised an exception", name, e);
+ }
+ if (region != null) {
+ println("Region is determined as \"%s\"", region);
+ if (foundRegion == null) {
+ foundRegion = region;
+ regionProvider = name;
+ }
+ return true;
+ } else {
+ println("region is not known");
+ return false;
+ }
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new Regions(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/RestoreObject.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/RestoreObject.java
new file mode 100644
index 0000000..89b67e5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/RestoreObject.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Deletes the objects on the command line.
+ */
+public class RestoreObject extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RestoreObject.class);
+
+ public static final String USAGE = "Usage: restore [-verbose] <S3A path> <version> <dest path>";
+
+ public RestoreObject() {
+ createCommandFormat(3, 3);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.size() != 3) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+
+ final Path source = new Path(paths.get(0));
+ final String version = paths.get(1);
+ final S3AFileSystem fs = (S3AFileSystem) source.getFileSystem(conf);
+ final Path src = fs.makeQualified(source);
+ final Path dst = fs.makeQualified(new Path(paths.get(2)));
+ println("restoring %s @ %s to %s", src, version, dst);
+ long l;
+ try (VersionedFileCopier copier = new VersionedFileCopier(fs);
+ StoreDurationInfo d = new StoreDurationInfo(getOut(), "restore")) {
+ l = copier.copy(fs.pathToKey(src), version, fs.pathToKey(dst));
+ }
+ println("Restored object of size %,d bytes to %s%n", l, dst);
+
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new RestoreObject(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/S3ListingSupport.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/S3ListingSupport.java
new file mode 100644
index 0000000..994304c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/S3ListingSupport.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+/**
+ * Listing support for {@link ListObjects} and others.
+ */
+public final class S3ListingSupport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(S3ListingSupport.class);
+
+ public static final int MAX_KEYS = 5000;
+
+ private S3ListingSupport() {}
+
+ /**
+ * Turns a path (relative or otherwise) into an S3 key.
+ *
+ * @param path input path, may be relative to the working dir
+ * @return a key excluding the leading "/", or, if it is the root path, ""
+ */
+ public static String pathToKey(Path path) {
+ boolean condition = path.isAbsolute();
+ Preconditions.checkArgument(condition, "not absolute: " + path);
+ if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
+ return "";
+ }
+
+ return path.toUri().getPath().substring(1);
+ }
+
+ /**
+ * Convert a path back to a key.
+ *
+ * @param key input key
+ * @return the path from this key
+ */
+ public static Path keyToPath(String key) {
+ return new Path("/" + key);
+ }
+
+ /**
+ * Create a {@code ListObjectsRequest} request against this bucket.
+ *
+ * @param bucket bucket to listJitendra
+ * @param key key for request
+ * @param delimiter any delimiter
+ * @return the request
+ */
+ public static ListObjectsV2Request createListObjectsV2Request(final String bucket, String key,
+ String delimiter) {
+ ListObjectsV2Request.Builder request =
+ ListObjectsV2Request.builder().bucket(bucket).maxKeys(MAX_KEYS).prefix(key);
+ if (delimiter != null) {
+ request.delimiter(delimiter);
+ }
+ return request.build();
+ }
+
+ /**
+ * Create a {@code ListObjectsRequest} request against this bucket.
+ *
+ * @param bucket bucket to listJitendra
+ * @param key key for request
+ * @param delimiter any delimiter
+ * @return the request
+ */
+ public static ListObjectVersionsRequest createListVersionsRequest(final String bucket, String key,
+ String delimiter) {
+ ListObjectVersionsRequest.Builder request =
+ ListObjectVersionsRequest.builder().bucket(bucket).maxKeys(MAX_KEYS).prefix(key);
+ if (delimiter != null) {
+ request.delimiter(delimiter);
+ }
+ return request.build();
+ }
+
+ /**
+ * Predicate: does the object represent a directory?.
+ *
+ * @param name object name
+ * @param size object size
+ * @return true if it meets the criteria for being an object
+ */
+ public static boolean objectRepresentsDirectory(final String name, final long size) {
+ boolean hasDirName = !name.isEmpty() && name.charAt(name.length() - 1) == '/';
+
+ boolean isEmpty = size == 0L;
+ if (hasDirName && !isEmpty) {
+ LOG.warn("Warning: object {} has length {} so is not a directory marker", name, size);
+ }
+ return hasDirName && isEmpty;
+ }
+
+ /**
+ * String information about a summary entry.
+ *
+ * @param summary summary object
+ * @return string value
+ */
+ public static String stringify(S3Object summary) {
+ return String.format("\"%s\"\tsize: [%d]\t%s\ttag: %s", summary.key(), summary.size(),
+ summary.lastModified().toString(), summary.eTag());
+ }
+
+ static boolean isDirMarker(final ObjectVersion summary) {
+ return objectRepresentsDirectory(summary.key(), summary.size());
+ }
+
+ static int result(boolean b) {
+ return b ? 1 : 0;
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/SessionKeys.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/SessionKeys.java
new file mode 100644
index 0000000..9046fe5
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/SessionKeys.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
+import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.SESSION_TOKEN;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
+import org.apache.hadoop.fs.s3a.auth.STSClientFactory;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.fs.store.StoreUtils;
+import org.apache.hadoop.fs.store.commands.EnvEntry;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.services.sts.StsClientBuilder;
+import software.amazon.awssdk.services.sts.model.Credentials;
+
+public class SessionKeys extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SessionKeys.class);
+
+ public static final String TEMPORARY_AWSCREDENTIALS_PROVIDER =
+ "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+
+ public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID";
+
+ public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
+
+ public static final String AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN";
+
+ public static final String JSON = "json";
+
+ public static final String ROLE = "role";
+
+ public static final String DURATION = "duration";
+
+ public static final String USAGE = "Usage: sessionkeys\n" + STANDARD_OPTS
+ + optusage(DURATION, "duration",
+ "duration of token, suffix of h m s for hour, minute, second")
+ + optusage(ROLE, "arn", "Role to assume")
+ + optusage(JSON, "file", "Json file to load (only valid if -role is set") + " <S3A path>";
+
+ public SessionKeys() {
+ createCommandFormat(1, 1);
+ addValueOptions(DURATION, ROLE, JSON);
+ }
+
+ @SuppressWarnings("MagicNumber")
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.isEmpty()) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+
+ // get JSON or empty string
+ String role = getOptional(ROLE).orElse("");
+ String jsonFile = getOptional(JSON).orElse("");
+ Duration duration =
+ getOptional(DURATION).map(arg -> StoreUtils.parseDurationArgument(arg, 12, TimeUnit.HOURS))
+ .orElse(Duration.of(12, ChronoUnit.HOURS));
+ boolean hasRole = !role.isEmpty();
+ boolean hasJsonFile = !jsonFile.isEmpty();
+ String json = null;
+ if (hasJsonFile) {
+ if (!hasRole) {
+ errorln("No -role specified for JSON");
+ return EXIT_USAGE;
+ }
+ json = new String(Files.readAllBytes(Paths.get(jsonFile)));
+ }
+
+ AWSCredentialProviderList credentials = null;
+
+ final Path source = new Path(paths.get(0));
+ try (StoreDurationInfo ignored =
+ new StoreDurationInfo(LOG, "requesting %s credentials", hasRole ? "role" : "session")) {
+ S3AFileSystem fs = (S3AFileSystem) source.getFileSystem(conf);
+ credentials = fs.shareCredentials("session");
+ Configuration fsconf = fs.getConf();
+ String bucket = fs.getBucket();
+ String keyId;
+ String secretKey;
+ String sessionToken;
+ // look to see if the creds are already session credentials
+ if (credentials.resolveCredentials() instanceof AwsSessionCredentials) {
+ // already got session credentials, so just print out
+ println("Bucket credentials are already session credentials");
+ final AwsSessionCredentials session =
+ (AwsSessionCredentials) credentials.resolveCredentials();
+ keyId = session.accessKeyId();
+ secretKey = session.secretAccessKey();
+ sessionToken = session.sessionToken();
+ } else {
+
+ Credentials sessionCreds;
+
+ StsClientBuilder builder = STSClientFactory.builder(fsconf, bucket, credentials);
+ STSClientFactory.STSClient stsClient = STSClientFactory.createClientConnection(
+ builder.build(), new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT));
+
+ final long minutes = duration.toMinutes();
+ println("Session duration: %d minutes", minutes);
+ if (!hasRole) {
+ sessionCreds = stsClient.requestSessionCredentials(minutes, TimeUnit.MINUTES);
+ } else {
+ sessionCreds =
+ stsClient.requestRole(role, "role-session", json, minutes, TimeUnit.MINUTES);
+ }
+
+ keyId = sessionCreds.accessKeyId();
+ secretKey = sessionCreds.secretAccessKey();
+ sessionToken = sessionCreds.sessionToken();
+ }
+
+ String endpoint = fsconf.get("fs.s3a.endpoint");
+ String region = fsconf.get("fs.s3a.endpoint.region");
+ String endpointkey = String.format("fs.s3a.bucket.%s.endpoint", bucket);
+ String regionkey = String.format("fs.s3a.bucket.%s.endpoint.region", bucket);
+
+ List<EnvEntry> entries = new ArrayList<>();
+ entries.add(new EnvEntry(ACCESS_KEY, AWS_ACCESS_KEY_ID, keyId));
+ entries.add(new EnvEntry(SECRET_KEY, AWS_SECRET_ACCESS_KEY, secretKey));
+ entries.add(new EnvEntry(SESSION_TOKEN, AWS_SESSION_TOKEN, sessionToken));
+ entries.add(new EnvEntry(AWS_CREDENTIALS_PROVIDER, "", TEMPORARY_AWSCREDENTIALS_PROVIDER));
+ if (endpoint != null) {
+ entries.add(new EnvEntry(endpointkey, "", endpoint));
+ }
+ if (region != null) {
+ entries.add(new EnvEntry(regionkey, "AWS_REGION", region));
+ }
+
+ // =================================================
+ heading("XML settings");
+
+ StringBuilder xml = new StringBuilder();
+ xml.append("<configuration>\n\n");
+
+ entries.forEach(e -> xml.append(e.xml()));
+ xml.append("\n</configuration>\n");
+
+ println(xml.toString());
+
+ // =================================================
+ heading("Properties");
+
+ StringBuilder props = new StringBuilder();
+ entries.forEach(e -> props.append(e.property()));
+
+ println(props.toString());
+
+ // =================================================
+ heading("CLI Arguments");
+
+ StringBuilder cliprops = new StringBuilder();
+ entries.forEach(e -> cliprops.append(e.cliProperty()));
+
+ println(cliprops.toString());
+
+ // =================================================
+ heading("Spark");
+ StringBuilder spark = new StringBuilder();
+ entries.forEach(e -> spark.append(e.spark()));
+ println(spark.toString());
+
+ // =================================================
+ heading("Bash");
+
+ StringBuilder bash = new StringBuilder();
+ entries.stream().filter(EnvEntry::hasEnvVar).forEach(e -> bash.append(e.bash()));
+ println(bash.toString());
+
+ // =================================================
+ heading("Fish");
+
+ StringBuilder fish = new StringBuilder();
+ entries.stream().filter(EnvEntry::hasEnvVar).forEach(e -> fish.append(e.fish()));
+ println(fish.toString());
+
+ // =================================================
+ heading("env");
+
+ StringBuilder env = new StringBuilder();
+ entries.stream().filter(EnvEntry::hasEnvVar).forEach(e -> env.append(e.env()));
+ println(env.toString());
+
+ } finally {
+ if (credentials != null) {
+ credentials.close();
+ }
+ }
+
+ return 0;
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new SessionKeys(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/SummaryProcessor.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/SummaryProcessor.java
new file mode 100644
index 0000000..026fd9a
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/SummaryProcessor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.io.Closeable;
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import software.amazon.awssdk.services.s3.model.DeleteMarkerEntry;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+
+interface SummaryProcessor extends Closeable {
+
+ boolean process(ObjectVersion summary, Path path, boolean isDeleteMarker) throws IOException;
+
+ default boolean processTombstone(Path path, DeleteMarkerEntry tombstone) throws IOException {
+ return false;
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/Undelete.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/Undelete.java
new file mode 100644
index 0000000..cbe47a2
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/Undelete.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import static org.apache.hadoop.fs.s3a.sdk.S3ListingSupport.isDirMarker;
+import static org.apache.hadoop.fs.store.CommonParameters.LIMIT;
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import org.apache.hadoop.fs.store.StoreDurationInfo;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
+
+/**
+ * Undeleter -removes tombstone markers (possibly) on top of files.
+ */
+public class Undelete extends StoreEntryPoint implements SummaryProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Undelete.class);
+
+ public static final String AGE = "age";
+
+ public static final String DELETED = "deleted";
+
+ public static final String DIRS = "dirs";
+
+ public static final String OUTPUT = "out";
+
+ public static final String QUIET = "q";
+
+ public static final String SEPARATOR = "separator";
+
+ public static final String SINCE = "since";
+
+ public static final String USAGE = "Usage: undelete <path>\n" + STANDARD_OPTS
+ + optusage(LIMIT, "limit", "limit of files to list")
+ + optusage(AGE, "seconds", "Only include versions created in this time interval")
+ + optusage(SINCE, "epoch-time", "Only include versions after this time");
+
+ public static final String NAME = "undelete";
+
+ private final int deletePageSize;
+
+ private S3Client s3;
+
+ private S3AFileSystem fs;
+
+ private StoreContext context;
+
+ private Path source;
+
+ private List<ObjectIdentifier> deletions;
+
+ public Undelete() {
+ createCommandFormat(1, 1);
+ addValueOptions(LIMIT,
+ // OUTPUT,
+ // SEPARATOR,
+ AGE, SINCE);
+ deletePageSize = 1000;
+ }
+
+ private void newDeletions() {
+ deletions = new ArrayList<>(deletePageSize);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ List<String> paths = parseArgs(args);
+ if (paths.isEmpty()) {
+ errorln(USAGE);
+ return EXIT_USAGE;
+ }
+
+ final Configuration conf = createPreconfiguredConfig();
+ // stop auditing rejecting client direct calls.
+ conf.setBoolean(FS_S3A_AUDIT_REJECT_OUT_OF_SPAN_OPERATIONS, false);
+ int limit = getIntOption(LIMIT, 0);
+ long since = getLongOption(SINCE, 0);
+
+ final Optional<Long> age = getOptionalLong(AGE);
+ if (age.isPresent()) {
+ if (since > 0) {
+ errorln("Only one of " + AGE + " and " + SINCE + " may be specified");
+ return EXIT_USAGE;
+ }
+ since = Instant.now().minusSeconds(age.get()).getEpochSecond();
+ }
+ Instant ageLimit = Instant.ofEpochSecond(since);
+
+ source = new Path(paths.get(0));
+ fs = (S3AFileSystem) source.getFileSystem(conf);
+ context = fs.createStoreContext();
+
+ s3 = fs.getS3AInternals().getAmazonS3Client(NAME);
+ newDeletions();
+
+ try (StoreDurationInfo duration = new StoreDurationInfo(getOut(), NAME)) {
+
+ final ListAndProcessVersionedObjects listing =
+ new ListAndProcessVersionedObjects(NAME, this, fs, source, this, ageLimit, limit);
+
+ final long count = listing.execute();
+
+ // final deletion
+ if (!deletions.isEmpty()) {
+ executeDeleteOperation();
+ }
+
+ println();
+
+ println("Removed %,d tombstones\n", count);
+ }
+
+ return 0;
+ }
+
+ @Override
+ public boolean process(final ObjectVersion summary, final Path path, final boolean isDeleteMarker)
+ throws IOException {
+ // Live ObjectVersion entries are never undeleted. Only DeleteMarkerEntry
+ // tombstones, handled in processTombstone, get queued for removal.
+ return false;
+ }
+
+ @Override
+ public boolean processTombstone(final Path path,
+ final software.amazon.awssdk.services.s3.model.DeleteMarkerEntry tombstone)
+ throws IOException {
+ final boolean dirMarker = isDirMarker(tombstone.key());
+ if (dirMarker) {
+ return false;
+ }
+ println("%s @ %s", fs.keyToQualifiedPath(tombstone.key()), tombstone.versionId());
+ deletions.add(
+ ObjectIdentifier.builder().key(tombstone.key()).versionId(tombstone.versionId()).build());
+ if (deletions.size() == deletePageSize) {
+ executeDeleteOperation();
+ }
+ return true;
+ }
+
+ private static boolean isDirMarker(String key) {
+ return !key.isEmpty() && key.charAt(key.length() - 1) == '/';
+ }
+
+ private void executeDeleteOperation() throws IOException {
+ final DeleteObjectsRequest r = DeleteObjectsRequest.builder().bucket(fs.getBucket())
+ .delete(Delete.builder().objects(deletions).quiet(true).build()).build();
+ println();
+ try (StoreDurationInfo duration =
+ new StoreDurationInfo(getOut(), "deleting %,d objects", deletions.size())) {
+ context.getInvoker().retry("delete", source.toString(), true, () -> s3.deleteObjects(r));
+ }
+ newDeletions();
+ }
+
+ /**
+ * Execute the command, return the result or throw an exception, as appropriate.
+ *
+ * @param args argument varags.
+ * @return return code
+ * @throws Exception failure
+ */
+ public static int exec(String... args) throws Exception {
+ return ToolRunner.run(new Undelete(), args);
+ }
+
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/VersionedFileCopier.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/VersionedFileCopier.java
new file mode 100644
index 0000000..0cf4cca
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/VersionedFileCopier.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
+
+import java.io.Closeable;
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.api.RequestFactory;
+import org.apache.hadoop.fs.s3a.audit.AuditManagerS3A;
+import org.apache.hadoop.fs.s3a.impl.StoreContext;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
+import software.amazon.awssdk.services.s3.model.CopyObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+
+/**
+ * Support for copying versioned files to different locations in the same bucket. This code uses
+ * S3Auditing and other @private apis which came with HADOOP-17511. Add an Audit plugin point for
+ * S3A auditing/context then constructs a new TransferManager -the one in the s3afs is inaccessible.
+ */
+public class VersionedFileCopier implements Closeable {
+
+ private final Configuration conf;
+
+ private final S3AFileSystem fs;
+
+ private final RequestFactory requestFactory;
+
+ private final StoreContext storeContext;
+
+ private final AuditManagerS3A auditManager;
+
+ private final S3Client s3;
+
+ private final Invoker invoker;
+
+ public VersionedFileCopier(final S3AFileSystem fs) {
+ this.fs = fs;
+ this.s3 = new InternalAccess(fs).getAmazonS3Client();
+ this.storeContext = fs.createStoreContext();
+ this.invoker = storeContext.getInvoker();
+ this.conf = storeContext.getConfiguration();
+ this.requestFactory = storeContext.getRequestFactory();
+ this.auditManager = fs.getAuditManager();
+ }
+
+ public RequestFactory getRequestFactory() {
+ return requestFactory;
+ }
+
+ @Override
+ public void close() throws IOException {}
+
+ /**
+ * Copy an object.
+ *
+ * @param sourceKey source
+ * @param version source version
+ * @param destKey dest in same bucket
+ * @return bytes copied
+ * @throws IOException failure
+ */
+ long copy(String sourceKey, String version, String destKey) throws IOException {
+
+ String action = String.format("copy %s @ %s to %s", sourceKey, version, destKey);
+ HeadObjectRequest head =
+ requestFactory.newHeadObjectRequestBuilder(sourceKey).versionId(version).build();
+ final HeadObjectResponse srcom =
+ invoker.retry("HEAD @" + version, sourceKey, true, () -> s3.headObject(head));
+
+ CopyObjectRequest copyObjectRequest = requestFactory
+ .newCopyObjectRequestBuilder(sourceKey, destKey, srcom).sourceVersionId(version).build();
+
+ CopyObjectResponse response =
+ invoker.retry(action, sourceKey, true, () -> s3.copyObject(copyObjectRequest));
+ return srcom.contentLength();
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/s3a/sdk/package-info.java b/src/main/java/org/apache/hadoop/fs/s3a/sdk/package-info.java
new file mode 100644
index 0000000..448ee9e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/s3a/sdk/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Low level SDK code.
+ */
+package org.apache.hadoop.fs.s3a.sdk;
diff --git a/src/main/java/org/apache/hadoop/fs/store/Cloudstore.java b/src/main/java/org/apache/hadoop/fs/store/Cloudstore.java
new file mode 100644
index 0000000..d172ab4
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/Cloudstore.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.hadoop.fs.gs.GsCredDiag;
+import org.apache.hadoop.fs.s3a.sdk.BucketMetadata;
+import org.apache.hadoop.fs.s3a.sdk.BulkDeleteCommand;
+import org.apache.hadoop.fs.s3a.sdk.DeleteObject;
+import org.apache.hadoop.fs.s3a.sdk.IamPolicy;
+import org.apache.hadoop.fs.s3a.sdk.ListMultiparts;
+import org.apache.hadoop.fs.s3a.sdk.ListObjects;
+import org.apache.hadoop.fs.s3a.sdk.ListVersions;
+import org.apache.hadoop.fs.s3a.sdk.MkBucket;
+import org.apache.hadoop.fs.s3a.sdk.Regions;
+import org.apache.hadoop.fs.s3a.sdk.RestoreObject;
+import org.apache.hadoop.fs.s3a.sdk.SessionKeys;
+import org.apache.hadoop.fs.s3a.sdk.Undelete;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.store.audit.AuditTool;
+import org.apache.hadoop.fs.store.commands.Bandwidth;
+import org.apache.hadoop.fs.store.commands.BucketState;
+import org.apache.hadoop.fs.store.commands.CommitterInfo;
+import org.apache.hadoop.fs.store.commands.Constval;
+import org.apache.hadoop.fs.store.commands.EtagCommand;
+import org.apache.hadoop.fs.store.commands.ExtendedDu;
+import org.apache.hadoop.fs.store.commands.FetchTokens;
+import org.apache.hadoop.fs.store.commands.ListFiles;
+import org.apache.hadoop.fs.store.commands.LocalHost;
+import org.apache.hadoop.fs.store.commands.LocateFiles;
+import org.apache.hadoop.fs.store.commands.PathCapability;
+import org.apache.hadoop.fs.store.commands.PrintStatus;
+import org.apache.hadoop.fs.store.commands.Put;
+import org.apache.hadoop.fs.store.commands.TLSInfo;
+import org.apache.hadoop.fs.store.diag.StoreDiag;
+import org.apache.hadoop.fs.tools.cloudup.Cloudup;
+import org.apache.hadoop.fs.tools.csv.MkCSV;
+import org.apache.hadoop.service.launcher.LauncherExitCodes;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Single launcher entry point. Replaces the per-command lowercase default-package shims with a name
+ * → {@link EntryPoint} registry dispatched through {@link ToolRunner#run}.
+ *
+ * <p>
+ * Invocation, using the bundled jar:
+ *
+ * <pre>{@code
+ * hadoop jar cloudstore-1.3.jar org.apache.hadoop.fs.store.Cloudstore <command> [args...]
+ * hadoop jar cloudstore-1.3.jar org.apache.hadoop.fs.store.Cloudstore help
+ * }</pre>
+ *
+ * <p>
+ * The lowercase default-package shims (e.g. {@code dux}, {@code storediag}) are still accepted by
+ * {@code hadoop jar} for back-compat; this class is the supported single entry point going forward.
+ *
+ * <p>
+ * Each invocation instantiates the {@link Tool} subclass via its public no-arg constructor and runs
+ * it with the rest of the argv via {@link ToolRunner}. Throwables coming out of
+ * {@code ToolRunner.run} are routed through the same three-way handler used by every existing
+ * {@code main()}: usage errors map to {@link LauncherExitCodes#EXIT_USAGE},
+ * {@link ExitUtil.ExitException}s preserve their embedded code, and any other throwable produces
+ * {@link LauncherExitCodes#EXIT_FAIL}.
+ */
+@SuppressWarnings("SpellCheckingInspection")
+public final class Cloudstore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Cloudstore.class);
+
+ /** Registry value: implementing class + one-line description shown by {@code help}. */
+ static final class EntryPoint {
+
+ final Class<? extends Tool> tool;
+ final String description;
+
+ private EntryPoint(final Class<? extends Tool> tool, final String description) {
+ this.tool = tool;
+ this.description = description;
+ }
+ }
+
+ private static EntryPoint ep(final Class<? extends Tool> tool, final String description) {
+ return new EntryPoint(tool, description);
+ }
+
+ /** Command name → {@link EntryPoint}. {@link #printUsage} sorts by key at print time. */
+ private static final Map<String, EntryPoint> COMMANDS;
+
+ static {
+ Map<String, EntryPoint> m = new LinkedHashMap<>();
+ m.put("auditlogs", ep(AuditTool.class, "Audit log processings"));
+ m.put("bandwidth", ep(Bandwidth.class, "measure network bandwidth"));
+ m.put("bucketmetadata", ep(BucketMetadata.class, "retrieve bucket metadata"));
+ m.put("bucketstate", ep(BucketState.class, "prints the AWS bucket state"));
+ m.put("bulkdelete", ep(BulkDeleteCommand.class, "bulk delete objects/files"));
+ m.put("cloudup", ep(Cloudup.class, "copies to/from cloud storage"));
+ m.put("committerinfo", ep(CommitterInfo.class, "Print committer information"));
+ m.put("constval", ep(Constval.class, "look up a constant value in a class"));
+ m.put("deleteobject", ep(DeleteObject.class, "Delete an S3 object"));
+ m.put("dux", ep(ExtendedDu.class, "extended du"));
+ m.put("etag", ep(EtagCommand.class, "print the etag of an object (where supported)"));
+ m.put("fetchdt", ep(FetchTokens.class, "fetch delegation tokens"));
+ m.put("filestatus", ep(PrintStatus.class, "print file statuses"));
+ m.put("gcscreds",
+ ep(GsCredDiag.class, "credential diagnostics for GCS. Warning: logs secrets"));
+ m.put("iampolicy", ep(IamPolicy.class, "generate IAM policy"));
+ m.put("list", ep(ListFiles.class, "list files"));
+ m.put("listmultiparts", ep(ListMultiparts.class, "list multipart uploads to CSV"));
+ m.put("listobjects", ep(ListObjects.class, "list S3 objects and their translated statuses"));
+ m.put("listversions", ep(ListVersions.class, "list all versions of S3 objects under a path"));
+ m.put("localhost", ep(LocalHost.class, "print local host details"));
+ m.put("locatefiles", ep(LocateFiles.class, "locate files"));
+ m.put("mkbucket", ep(MkBucket.class, "Create an S3 bucket"));
+ m.put("mkcsv", ep(MkCSV.class, "generate CSV file"));
+ m.put("pathcapability", ep(PathCapability.class, "probe for path capabilities"));
+ m.put("put", ep(Put.class, "file upload/copy"));
+ m.put("regions", ep(Regions.class, "Emulate region lookup of AWS SDK"));
+ m.put("restore", ep(RestoreObject.class, "Restore a versioned S3 object"));
+ m.put("sessionkeys", ep(SessionKeys.class, "request STS session credentials"));
+ m.put("storediag", ep(StoreDiag.class, "store diagnostics"));
+ m.put("tlsinfo", ep(TLSInfo.class, "Print TLS information"));
+ m.put("undelete", ep(Undelete.class, "undelete s3 objects by removing tombstones"));
+ COMMANDS = Collections.unmodifiableMap(m);
+ }
+
+ private Cloudstore() {}
+
+ /**
+ * Dispatch one command. Returns the exit code from {@link ToolRunner#run} for known commands, or
+ * {@link LauncherExitCodes#EXIT_USAGE} when the command name is missing, unknown, or {@code help}
+ * / {@code -help} / {@code --help}.
+ *
+ * <p>
+ * Throwables raised by the dispatched Tool propagate up, just like the existing per-command
+ * {@code exec} methods.
+ */
+ public static int exec(String... args) throws Exception {
+ if (args == null || args.length == 0) {
+ printUsage(System.err);
+ return LauncherExitCodes.EXIT_USAGE;
+ }
+ String name = args[0];
+ if ("help".equals(name) || "-help".equals(name) || "--help".equals(name)) {
+ printUsage(System.out);
+ return 0;
+ }
+ EntryPoint entry = COMMANDS.get(name);
+ if (entry == null) {
+ System.err.println("Unknown command: " + name);
+ printUsage(System.err);
+ return LauncherExitCodes.EXIT_USAGE;
+ }
+ Tool tool = entry.tool.getDeclaredConstructor().newInstance();
+ String[] rest = Arrays.copyOfRange(args, 1, args.length);
+ return ToolRunner.run(tool, rest);
+ }
+
+ /**
+ * Process entry point. Calls {@link ExitUtil#terminate} on every exit path, mirroring the
+ * convention used by every existing {@code main()} in the project so the wrapper script semantics
+ * stay identical.
+ */
+ public static void main(String[] args) {
+ try {
+ ExitUtil.terminate(exec(args), "");
+ } catch (Throwable e) {
+ exitOnThrowable(e);
+ }
+ }
+
+ /**
+ * Map a Throwable to an exit code following the same three-way split used by
+ * {@code StoreEntryPoint.exitOnThrowable}.
+ */
+ static void exitOnThrowable(Throwable ex) {
+ if (ex instanceof CommandFormat.UnknownOptionException) {
+ // usage error
+ System.err.println(ex.getMessage());
+ ExitUtil.terminate(LauncherExitCodes.EXIT_USAGE, ex.getMessage());
+ } else if (ex instanceof ExitUtil.ExitException) {
+ // exception with explicitly declared exit code
+ LOG.debug("Command failure", ex);
+ ExitUtil.terminate((ExitUtil.ExitException) ex);
+ } else {
+ // any other failure
+ ex.printStackTrace(System.err);
+ ExitUtil.terminate(LauncherExitCodes.EXIT_FAIL, ex.toString());
+ }
+ }
+
+ /** Visible for testing. */
+ static Map<String, EntryPoint> commands() {
+ return COMMANDS;
+ }
+
+ private static void printUsage(java.io.PrintStream out) {
+ out.println("Usage: cloudstore <command> [args...]");
+ out.println();
+ out.println("Commands:");
+ Map<String, EntryPoint> sorted = new TreeMap<>(COMMANDS);
+ for (Map.Entry<String, EntryPoint> e : sorted.entrySet()) {
+ out.printf(" %-16s %s%n", e.getKey(), e.getValue().description);
+ }
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/CommonParameters.java b/src/main/java/org/apache/hadoop/fs/store/CommonParameters.java
new file mode 100644
index 0000000..7107f9f
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/CommonParameters.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import static org.apache.hadoop.fs.store.StoreEntryPoint.optusage;
+
+/**
+ * Common parameters across entry points.
+ */
+public final class CommonParameters {
+
+ /** {@value}. */
+ public static final String TOKENFILE = "tokenfile";
+
+ /** {@value}. */
+ public static final String XMLFILE = "xmlfile";
+
+ /** file for system properties {@value}. */
+ public static final String SYSPROPS = "sysprop";
+
+ /** {@value}. */
+ public static final String DEFINE = "D";
+
+ /** {@value}. */
+ public static final String VERBOSE = "verbose";
+
+ /** {@value}. */
+ public static final String DEBUG = "debug";
+
+ /** {@value}. */
+ public static final String LOG_OVERRIDES = "logoverrides";
+
+ /**
+ * Standard options of all entry points.
+ */
+ public static final String STANDARD_OPTS = optusage(DEFINE, "key=value",
+ "Define a single configuration option")
+ + optusage(SYSPROPS, "file", "Property file of system properties")
+ + optusage(TOKENFILE, "file", "Hadoop token file to load")
+ + optusage(XMLFILE, "file", "XML config file to load") + optusage(VERBOSE, "verbose output")
+ + optusage(DEBUG,
+ "enable JVM logs (ALL) and override log4j levels (DEBUG) on specified packages or classes")
+ + optusage(LOG_OVERRIDES, "file", "A newline separated list of package and class names");
+
+ /**
+ * File for log4j properties: {@value}.
+ */
+ public static final String LOGFILE = "logfile";
+
+ /** {@value}. */
+ public static final String LIMIT = "limit";
+
+ /** {@value}. */
+ public static final String OPTIONS = "options";
+
+ /** {@value}. */
+ public static final String THREADS = "threads";
+
+ public static final String BFS = "bfs";
+
+ public static final String BLOCK = "block";
+
+ public static final String CSVFILE = "csv";
+
+ public static final String FLUSH = "flush";
+
+ public static final String HFLUSH = "hflush";
+
+ public static final String IGNORE = "ignore";
+
+ public static final String LARGEST = "largest";
+
+ public static final String OVERWRITE = "overwrite";
+
+ public static final String PAGE = "page";
+
+ public static final String UPDATE = "update";
+
+ private CommonParameters() {}
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/MinMeanMax.java b/src/main/java/org/apache/hadoop/fs/store/MinMeanMax.java
new file mode 100644
index 0000000..45ee204
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/MinMeanMax.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Simple min/mean/max statistics.
+ */
+public class MinMeanMax {
+
+ final String name;
+ final AtomicLong min = new AtomicLong(Long.MAX_VALUE);
+ final AtomicLong max = new AtomicLong(Long.MIN_VALUE);
+ final AtomicInteger samples = new AtomicInteger(0);
+ final AtomicLong sum = new AtomicLong(0);
+
+ public MinMeanMax(final String name) {
+ this.name = name;
+ }
+
+ /**
+ * copilot made this up for us.
+ *
+ * @param value new value
+ */
+ public synchronized void add(final long value) {
+ min.accumulateAndGet(value, Math::min);
+ max.accumulateAndGet(value, Math::max);
+ sum.addAndGet(value);
+ samples.incrementAndGet();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long min() {
+ return min.get();
+ }
+
+ public long max() {
+ return max.get();
+ }
+
+ public int samples() {
+ return samples.get();
+ }
+
+ public long sum() {
+ return sum.get();
+ }
+
+ public double mean() {
+ final int sam = samples.get();
+ return sam > 0 ? ((double) sum.get()) / sam : 0;
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/StoreDiagConstants.java b/src/main/java/org/apache/hadoop/fs/store/StoreDiagConstants.java
new file mode 100644
index 0000000..df9c2d1
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/StoreDiagConstants.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+public final class StoreDiagConstants {
+
+ public static final String IOSTATISTICS_LOGGING_LEVEL = "fs.iostatistics.logging.level";
+
+ public static final boolean DEFAULT_HIDE_ALL_SENSITIVE_CHARS = true;
+
+ private StoreDiagConstants() {}
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/StoreDurationInfo.java b/src/main/java/org/apache/hadoop/fs/store/StoreDurationInfo.java
new file mode 100644
index 0000000..f8eea6e
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/StoreDurationInfo.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import java.io.PrintStream;
+import java.time.Duration;
+import java.util.function.Supplier;
+import org.slf4j.Logger;
+
+/**
+ * A duration with logging of final state at info in the {@code close()} call. This allows it to be
+ * used in a try-with-resources clause, and have the duration automatically logged.
+ * <p>
+ * Based on the hadoop one; adds an empty constructor which doesn't do any logging.
+ */
+public class StoreDurationInfo implements AutoCloseable {
+
+ private final long started;
+
+ private long finished;
+
+ private boolean isFinished;
+
+ private final Supplier<String> text;
+
+ public static final Supplier<String> EMPTY_TEXT = () -> "";
+
+ private String textStr;
+
+ private final Logger log;
+
+ /**
+ * Should the log be at INFO rather than DEBUG.
+ */
+ private final boolean logAtInfo;
+
+ private final PrintStream out;
+
+ /**
+ * Create the duration text from a {@code String.format()} code call.
+ *
+ * @param log log to write to
+ * @param format format string
+ * @param args list of arguments
+ */
+ public StoreDurationInfo(Logger log, String format, Object... args) {
+
+ this(log, true, format, args);
+ }
+
+ /**
+ * Create the duration text from a {@code String.format()} code call and log either at info or
+ * debug.
+ *
+ * @param log log to write to
+ * @param logAtInfo should the log be at info, rather than debug
+ * @param format format string
+ * @param args list of arguments
+ */
+ public StoreDurationInfo(Logger log, boolean logAtInfo, String format, Object... args) {
+ this.started = time();
+ this.finished = started;
+ this.text = () -> String.format(format, args);
+ this.log = log;
+ this.logAtInfo = logAtInfo;
+ this.out = null;
+ if (log != null) {
+ if (logAtInfo) {
+ log.info("Starting: {}", getFormattedText());
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("Starting: {}", getFormattedText());
+ }
+ }
+ }
+ }
+
+ /**
+ * Create the duration text from a {@code String.format()} code call.
+ *
+ * @param out log to write to
+ * @param format format string
+ * @param args list of arguments
+ */
+ public StoreDurationInfo(PrintStream out, String format, Object... args) {
+ started = time();
+ finished = started;
+ this.text = () -> String.format(format, args);
+ this.out = out;
+ this.log = null;
+ this.logAtInfo = false;
+
+ if (out != null) {
+ out.printf("[%s] Starting: %s%n", humanTime(started), getFormattedText());
+ }
+ }
+
+ /**
+ * Create the duration with no output printed.
+ */
+ public StoreDurationInfo() {
+ this.text = EMPTY_TEXT;
+ this.log = null;
+ this.out = null;
+ this.logAtInfo = false;
+ started = time();
+ finished = started;
+ }
+
+ private String getFormattedText() {
+ if (textStr == null) {
+ textStr = text.get();
+ }
+ return textStr;
+ }
+
+ private long time() {
+ return System.currentTimeMillis();
+ }
+
+ /**
+ * Finish the operation; only valid once.
+ */
+ public synchronized void finished() {
+ if (!isFinished) {
+ finished = time();
+ isFinished = true;
+ }
+ }
+
+ public String getDurationString() {
+ return humanTime(value());
+ }
+
+ public static String humanTime(long time) {
+ long millis = time % 1000;
+ long seconds = (time / 1000);
+ long minutes = (seconds / 60);
+ long hours = (minutes / 60);
+ return String.format("%02d:%02d:%02d.%03d", hours % 24, minutes % 60, seconds % 60, millis);
+ }
+
+ /**
+ * The duration in seconds; only valid once finished/closed.
+ *
+ * @return time in milliseconds
+ */
+ public long value() {
+ return finished - started;
+ }
+
+ /**
+ * Get as a java time duration; only valid once finished/closed.
+ *
+ * @return duration.
+ */
+ public Duration asDuration() {
+ return Duration.ofMillis(value());
+ }
+
+ @Override
+ public String toString() {
+ return getDurationString();
+ }
+
+ /**
+ * Close the duration, invoke {@link #finished()}. Log the final state if a log or output stream
+ * was provided.
+ */
+ @Override
+ public void close() {
+ finished();
+ if (log != null) {
+ if (logAtInfo) {
+ log.info("Duration of {}: {}", getFormattedText(), this);
+ } else {
+ log.debug("Duration of {}: {}", getFormattedText(), this);
+ }
+ }
+ if (out != null) {
+ out.printf("Duration of %s: %s%n", getFormattedText(), this);
+ }
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/StoreEntryPoint.java b/src/main/java/org/apache/hadoop/fs/store/StoreEntryPoint.java
new file mode 100644
index 0000000..4467c6c
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/StoreEntryPoint.java
@@ -0,0 +1,885 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import static java.util.logging.Level.ALL;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+import static org.apache.hadoop.fs.store.CommonParameters.DEBUG;
+import static org.apache.hadoop.fs.store.CommonParameters.DEFINE;
+import static org.apache.hadoop.fs.store.CommonParameters.LOG_OVERRIDES;
+import static org.apache.hadoop.fs.store.CommonParameters.SYSPROPS;
+import static org.apache.hadoop.fs.store.CommonParameters.TOKENFILE;
+import static org.apache.hadoop.fs.store.CommonParameters.VERBOSE;
+import static org.apache.hadoop.fs.store.CommonParameters.XMLFILE;
+import static org.apache.hadoop.fs.store.StoreUtils.split;
+import static org.apache.hadoop.fs.store.diag.OptionSets.CLOUD_CONNECTOR_LOGS;
+import static org.apache.hadoop.fs.store.diag.OptionSets.JAVAX_NET_DEBUG;
+import static org.apache.hadoop.fs.store.diag.OptionSets.NET_DEBUG_SSL_HANDSHAKE;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.DIRECTORY_MARKER_RETENTION;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_CONNECTION_MAXIMUM;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.FS_S3A_THREADS_MAX;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.INPUT_FADVISE;
+import static org.apache.hadoop.fs.store.diag.S3ADiagnosticsInfo.INPUT_FADV_NORMAL;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.MalformedURLException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.LogManager;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.statistics.IOStatistics;
+import org.apache.hadoop.fs.store.diag.Printout;
+import org.apache.hadoop.fs.store.diag.StdoutPrintout;
+import org.apache.hadoop.fs.store.logging.LogControl;
+import org.apache.hadoop.fs.store.logging.LogControllerFactory;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.service.launcher.LauncherExitCodes;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Tool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Entry point for store applications.
+ */
+@SuppressWarnings({"UseOfSystemOutOrSystemErr", "SpellCheckingInspection",
+ "OverloadedVarargsMethod"})
+public class StoreEntryPoint extends Configured implements Tool, Closeable, Printout {
+
+ protected static final int MB_1 = (1024 * 1024);
+
+ private static final Logger LOG = LoggerFactory.getLogger(StoreEntryPoint.class);
+
+ /**
+ * Exit code when a usage message was printed: {@value}.
+ */
+ public static final int EXIT_USAGE = LauncherExitCodes.EXIT_USAGE;
+
+ private final LogExactlyOnce logJceksFailureOnce = new LogExactlyOnce(LOG);
+
+ /**
+ * Hide all sensitive data.
+ */
+ private boolean hideAllSensitiveChars = StoreDiagConstants.DEFAULT_HIDE_ALL_SENSITIVE_CHARS;
+
+ protected CommandFormat commandFormat;
+
+ private PrintStream out = System.out;
+
+ /**
+ * Active sink for all {@link Printout} method calls. Tests can swap this via
+ * {@link #setActivePrintout(Printout)} to capture output (e.g. into a
+ * {@link org.apache.hadoop.fs.store.diag.StringBuilderPrintout}). The field is intentionally
+ * global; cloudstore tests run serialised.
+ */
+ private static volatile Printout activePrintout = new StdoutPrintout();
+
+ /**
+ * Get the current active Printout sink.
+ *
+ * @return the sink
+ */
+ public static Printout getActivePrintout() {
+ return activePrintout;
+ }
+
+ /**
+ * Install a new active Printout sink, returning the previous one.
+ *
+ * @param newSink replacement (null restores the default stdout sink)
+ * @return the previous sink
+ */
+ public static Printout setActivePrintout(Printout newSink) {
+ Printout prev = activePrintout;
+ activePrintout = newSink != null ? newSink : new StdoutPrintout();
+ return prev;
+ }
+
+ /**
+ * Reset the active sink to a fresh stdout-backed default.
+ */
+ public static void resetActivePrintout() {
+ activePrintout = new StdoutPrintout();
+ }
+
+ private final AtomicInteger headingCounter = new AtomicInteger(1);
+ private final AtomicInteger subheadingCounter = new AtomicInteger(1);
+
+ /**
+ * Print a usage line for an option without a parameter.
+ *
+ * @param opt option option
+ * @param description description
+ */
+ public static String optusage(String opt, String description) {
+ return String.format("\t-%s\t%s%n", opt, description);
+ }
+
+ /**
+ * Print a usage line for an option with a parameter.
+ *
+ * @param opt option option
+ * @param second parameter name
+ * @param description description
+ */
+ public static String optusage(String opt, String second, String description) {
+ return String.format("\t-%s <%s>\t%s%n", opt, second, description);
+ }
+
+ /**
+ * Dump token info from the credentials, with resilience to failure.
+ *
+ * @param cred credentials
+ * @return the tokens
+ */
+ protected Collection<Token<? extends TokenIdentifier>> dumpTokens(final Credentials cred) {
+ final Collection<Token<? extends TokenIdentifier>> tokens = cred.getAllTokens();
+ for (Token<?> token : tokens) {
+ try {
+ println("Fetched token: %s", token);
+ } catch (Exception e) {
+ warn("Failed to unmarshall token %s", e);
+ LOG.warn("exception", e);
+ }
+ }
+ return tokens;
+ }
+
+ protected final String plural(int n) {
+ return n == 1 ? "" : "s";
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ return 0;
+ }
+
+ @Override
+ public void close() throws IOException {}
+
+ public final PrintStream getOut() {
+ return out;
+ }
+
+ public final void setOut(PrintStream out) {
+ this.out = out;
+ }
+
+ @Override
+ public final void println() {
+ activePrintout.println();
+ }
+
+ @Override
+ public void println(String format) {
+ activePrintout.println(format);
+ }
+
+ @Override
+ public void println(String format, Object... args) {
+ activePrintout.println(format, args);
+ }
+
+ @Override
+ public void flush() {
+ activePrintout.flush();
+ }
+
+ @Override
+ public final void print(String format, Object... args) {
+ activePrintout.print(format, args);
+ }
+
+ @Override
+ public final void warn(String format, Object... args) {
+ activePrintout.warn(format, args);
+ }
+
+ @Override
+ public final void advise(String format, Object... args) {
+ activePrintout.advise(format, args);
+ }
+
+ @Override
+ public final void error(String format, Object... args) {
+ activePrintout.error(format, args);
+ }
+
+ @Override
+ public final void exception(Throwable thrown, String format, Object... args) {
+ activePrintout.exception(thrown, format, args);
+ }
+
+ public static void errorln(String format, Object... args) {
+ System.err.printf(format + "%n", args);
+ System.err.println();
+ System.err.flush();
+ }
+
+ @Override
+ public void heading(String format, Object... args) {
+
+ final int hc = headingCounter.getAndIncrement();
+ subheadingCounter.set(1);
+ String text = String.format("%d. ", hc) + String.format(format, args);
+ println();
+ println(text);
+ println(underline('=', text.length()));
+ println();
+ }
+
+ protected static String underline(final char c, final int len) {
+ StringBuilder sb = new StringBuilder(len);
+ for (int i = 0; i < len; i++) {
+ sb.append(c);
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void subheading(String format, Object... args) {
+ // the current heading number, not the next one
+ final int hc = headingCounter.get() - 1;
+ final int shc = subheadingCounter.getAndIncrement();
+
+ String prefix = String.format("%d.%d ", hc, shc);
+ final String text = String.format(prefix + format, args);
+
+ println();
+ println(text);
+ println(underline('-', text.length()));
+ }
+
+ /**
+ * Debug message.
+ *
+ * @param format Log4J format string
+ * @param args arguments.
+ */
+ @Override
+ public final void debug(String format, Object... args) {
+ activePrintout.debug(format, args);
+ }
+
+ protected static void exit(int status, String text) {
+ ExitUtil.terminate(status, text);
+ }
+
+ protected static void exit(ExitUtil.ExitException ex) {
+ ExitUtil.terminate(ex.getExitCode(), ex.getMessage());
+ }
+
+ public final CommandFormat getCommandFormat() {
+ return commandFormat;
+ }
+
+ public final void setCommandFormat(CommandFormat commandFormat) {
+ this.commandFormat = commandFormat;
+ }
+
+ /**
+ * Create the command format. Use {@link #addValueOptions(String...)} to declare the value options
+ * afterwards. automatically add the standard opts
+ *
+ * @param min minimum number of non-option arguments.
+ * @param max max number of non-option arguments.
+ * @param options simple options.
+ */
+ protected final void createCommandFormat(int min, int max, String... options) {
+ List<String> ol = new ArrayList<>(Arrays.asList(options));
+ ol.add(DEBUG);
+ ol.add(VERBOSE);
+ setCommandFormat(new CommandFormat(min, max, ol.toArray(new String[0])));
+ addStandardValueOptions();
+ }
+
+ /**
+ * Add the standard value options; subclasses can remove any.
+ */
+ protected void addStandardValueOptions() {
+ addValueOptions(DEFINE, SYSPROPS, TOKENFILE, XMLFILE, LOG_OVERRIDES);
+ }
+
+ /**
+ * Add a list of value options.
+ *
+ * @param names option names.
+ */
+ protected final void addValueOptions(String... names) {
+ for (String s : names) {
+ commandFormat.addOptionWithValue(s);
+ }
+ }
+
+ /**
+ * Parse CLI arguments and returns the position arguments. The options are stored in
+ * {@link #commandFormat}.
+ *
+ * This is also where java debug logging is enabled if the {@code -debug} option is set.
+ *
+ * @param args command line arguments.
+ * @param min min number of args if positive
+ * @param max max number of args if positive
+ * @param error error text
+ * @return the position arguments from CLI.
+ * @throws IOException failure to load token from -tokenfile
+ * @throws ExitUtil.ExitException if the number of arguments is wrong.
+ */
+ protected List<String> processArgs(String[] args, int min, final int max, String error)
+ throws IOException {
+ final List<String> parsed = parseArgs(args);
+
+ if ((min >= 0 && parsed.size() < min) || (max >= 0 && parsed.size() > max)) {
+ errorln(error);
+ parsed.forEach(s -> errorln(" %s", s));
+ throw new ExitUtil.ExitException(EXIT_USAGE, "invalid argment count: expected between " + min
+ + " and " + max + " but got " + parsed.size());
+ }
+ maybeEnableDebugLogging();
+ maybeAddTokens(TOKENFILE);
+ maybeAddJvmOptions();
+ return parsed;
+ }
+
+ /**
+ * Parse CLI arguments and returns the position arguments. The options are stored in
+ * {@link #commandFormat}.
+ *
+ * This is also where java debug logging is enabled if the {@code -debug} option is set.
+ *
+ * @param args command line arguments.
+ * @return the position arguments from CLI.
+ */
+ protected List<String> parseArgs(String[] args) {
+ return args.length > 0 ? getCommandFormat().parse(args, 0) : new ArrayList<>(0);
+ }
+
+ /**
+ * Maybe enable JVM and cloud connector debug logging.
+ */
+ protected void maybeEnableDebugLogging() {
+ if (hasOption(DEBUG)) {
+ println("Enabling debug logging");
+ enableJvmLogging();
+ enableSSLLogging();
+ enableCloudConnectorLogging(getLogOverrides(), LogControl.LogLevel.DEBUG);
+ }
+ }
+
+ /**
+ * Reads the logoverrides files if specified, fallbacks to CLOUD_CONNECTOR_LOGS otherwise.
+ *
+ * @return a list of package and class names
+ */
+ protected List<String> getLogOverrides() {
+ String customLogLevelFile = getOption(LOG_OVERRIDES);
+ if (customLogLevelFile != null) {
+ try (Stream<String> stream = Files.lines(Paths.get(customLogLevelFile))) {
+ return stream.map(String::trim).filter(line -> !line.startsWith("#"))
+ .filter(line -> !line.isEmpty()).collect(Collectors.toList());
+ } catch (IOException e) {
+ println("could not read logoverrides file='%s' error='%s'", customLogLevelFile,
+ e.getMessage());
+ }
+ }
+ return Arrays.asList(CLOUD_CONNECTOR_LOGS);
+ }
+
+ /**
+ * Enable JVM logging.
+ */
+ protected void enableJvmLogging() {
+ println("Enabling JVM logging");
+ ConsoleHandler handler = new ConsoleHandler();
+ handler.setLevel(ALL);
+ java.util.logging.Logger log = LogManager.getLogManager().getLogger("");
+ log.addHandler(handler);
+ log.setLevel(ALL);
+ }
+
+ protected void enableSSLLogging() {
+ // javax.net.debug=ssl:handshake
+ System.setProperty(JAVAX_NET_DEBUG, NET_DEBUG_SSL_HANDSHAKE);
+ }
+
+ /**
+ * Enable cloud connector logging.
+ *
+ * @param level desired level
+ */
+ protected void enableCloudConnectorLogging(List<String> customLogs, LogControl.LogLevel level) {
+ final Optional<LogControl> control =
+ LogControllerFactory.createController(LogControllerFactory.LOG4J);
+ control.ifPresent(c -> customLogs.forEach(log -> c.setLogLevel(log, level)));
+ }
+
+ /**
+ * Get the value of a key-val option.
+ *
+ * @param opt option.
+ * @return the value or null
+ */
+ protected final String getOption(String opt) {
+ return getCommandFormat().getOptValue(opt);
+ }
+
+ /**
+ * Get the value of a key-val option.
+ *
+ * @param opt option.
+ * @param defval default value
+ * @return the value or the default.
+ */
+ protected final String getOption(String opt, String defval) {
+ return hasOption(opt) ? getCommandFormat().getOptValue(opt) : defval;
+ }
+
+ /**
+ * Did the command line have a specific option.
+ *
+ * @param opt option.
+ * @return true iff it was set.
+ */
+ protected final boolean hasOption(String opt) {
+ return getCommandFormat().getOpt(opt);
+ }
+
+ /**
+ * Get the value of a key-val option.
+ *
+ * @param opt option.
+ * @return the value or the empty option.
+ */
+ protected final Optional<String> getOptional(String opt) {
+ return Optional.ofNullable(getCommandFormat().getOptValue(opt));
+ }
+
+ /**
+ * get an integer option.
+ *
+ * @param opt option
+ * @param defval default value
+ * @return the value to use
+ */
+ protected final int getIntOption(String opt, int defval) {
+ return getOptional(opt).map(Integer::valueOf).orElse(defval);
+ }
+
+ /**
+ * Add all the various configuration files.
+ */
+ protected final void addAllDefaultXMLFiles() {
+ addDefaultResources("hdfs-default.xml", "hdfs-site.xml",
+ // this order is what JobConf does via
+ // org.apache.hadoop.mapreduce.util.ConfigUtil.loadResources()
+ "mapred-default.xml", "mapred-site.xml", "yarn-default.xml", "yarn-site.xml",
+ "hive-site.xml", "hive-default.xml", "hbase-default.xml", "hbase-site.xml");
+ }
+
+ protected final void addDefaultResources(String... resources) {
+ for (String resource : resources) {
+ Configuration.addDefaultResource(resource);
+ }
+ }
+
+ /**
+ * For subclasses: exit after a throwable was raised.
+ *
+ * @param ex exception caught
+ */
+ protected static void exitOnThrowable(Throwable ex) {
+ if (ex instanceof CommandFormat.UnknownOptionException) {
+ errorln(ex.getMessage());
+ exit(EXIT_USAGE, ex.getMessage());
+ } else if (ex instanceof ExitUtil.ExitException) {
+ LOG.debug("Command failure", ex);
+ exit((ExitUtil.ExitException) ex);
+ } else {
+ ex.printStackTrace(System.err);
+ exit(LauncherExitCodes.EXIT_FAIL, ex.toString());
+ }
+ }
+
+ protected void maybeAddXMLFileOption(final Configuration conf)
+ throws FileNotFoundException, MalformedURLException {
+ String xmlfile = getOption(CommonParameters.XMLFILE);
+ if (xmlfile != null) {
+ if (xmlfile.isEmpty()) {
+ throw new ExitUtil.ExitException(LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
+ "XML file option " + CommonParameters.XMLFILE + " found but no value was provided");
+ }
+ File f = new File(xmlfile);
+ if (!f.exists()) {
+ throw new FileNotFoundException(f.toString());
+ }
+ println("Adding XML configuration file %s", f);
+ conf.addResource(f.toURI().toURL());
+ }
+ }
+
+ /**
+ * Add a token file from the command line.
+ *
+ * @param opt option
+ * @throws IOException failures
+ */
+ protected void maybeAddTokens(String opt) throws IOException {
+ String tokenfile = getOption(opt);
+ if (tokenfile != null) {
+ heading("Adding tokenfile %s", tokenfile);
+ Credentials credentials = Credentials.readTokenStorageFile(new File(tokenfile), getConf());
+ Collection<Token<? extends TokenIdentifier>> tokens = credentials.getAllTokens();
+ println("Loaded %d token(s)", tokens.size());
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ println("%s/%s", token.getKind(), token.getService());
+ }
+ UserGroupInformation.getCurrentUser().addCredentials(credentials);
+ }
+ }
+
+ /**
+ * Add read jvm option properties file, set the values.
+ *
+ * @throws IOException failures
+ */
+ protected void maybeAddJvmOptions() throws IOException {
+ String propertyFile = getOption(SYSPROPS);
+ if (propertyFile != null) {
+ Properties properties = new Properties();
+ properties.load(FileUtils.openInputStream(new File(propertyFile)));
+ properties.forEach((k, v) -> {
+ System.setProperty((String) k, (String) v);
+ debug("Set system property %s=%s", k, v);
+ });
+ }
+ }
+
+ protected void maybePatchDefined(final Configuration conf, final String opt) {
+ getOptional(opt).ifPresent(d -> {
+ Map.Entry<String, String> pair = split(d, "true");
+ println("Patching configuration with \"%s\"=\"%s\"", pair.getKey(), pair.getValue());
+ conf.set(pair.getKey(), pair.getValue());
+ });
+ }
+
+ protected void printStatus(final int index, final FileStatus status) {
+ println("[%04d]\t%s\t%,d\t(%s)\t%s\t%s\t%s%s", index, status.getPath(), status.getLen(),
+ FileUtils.byteCountToDisplaySize(status.getLen()), status.getLen(), status.getOwner(),
+ status.getGroup(), status.isEncrypted() ? "\t[encrypted]" : "",
+ isVerbose() ? ("\t" + status) : "");
+ }
+
+ /**
+ * Did this command have the verbose option?
+ *
+ * @return true if -verbose was on the command line
+ */
+ protected boolean isVerbose() {
+ return hasOption(VERBOSE);
+ }
+
+ /**
+ * Dump the filesystem Storage Statistics iff the verbose flag was set.
+ *
+ * @param fs filesystem; can be null
+ */
+ protected void maybeDumpStorageStatistics(final FileSystem fs) {
+ if (isVerbose()) {
+ dumpFileSystemStatistics(fs);
+ }
+ }
+
+ /**
+ * Dump the filesystem Storage Statistics.
+ *
+ * @param fs filesystem; can be null
+ */
+ protected void dumpFileSystemStatistics(FileSystem fs) {
+ if (fs == null) {
+ return;
+ }
+ final IOStatistics iostats = retrieveIOStatistics(fs);
+ String report = iostats != null ? ioStatisticsToPrettyString(iostats) : "";
+ if (!report.isEmpty()) {
+ heading("IO Statistics");
+
+ println("%s", report);
+ } else {
+ // fall back
+ StorageStatistics st = fs.getStorageStatistics();
+
+ Iterator<StorageStatistics.LongStatistic> it = st.getLongStatistics();
+ if (it.hasNext()) {
+ // there is data
+ heading("Storage Statistics");
+ while (it.hasNext()) {
+ StorageStatistics.LongStatistic next = it.next();
+ println("%s\t%s", next.getName(), next.getValue());
+ }
+ }
+ }
+ }
+
+ protected void printIfVerbose(String format, Object o) {
+ if (isVerbose()) {
+ println(format, o);
+ }
+ }
+
+ protected void maybeClose(Object o) {
+ if (o instanceof Closeable) {
+ IOUtils.closeStreams((Closeable) o);
+ }
+ }
+
+ /**
+ * Set up the config with CLI config options. XML file, -D and abfs/s3a to log their IOStats at
+ * debug.
+ *
+ * @return a new config.
+ * @throws FileNotFoundException XML file was requested but not found.
+ * @throws MalformedURLException problems setting up default XML files.
+ */
+ protected Configuration createPreconfiguredConfig()
+ throws FileNotFoundException, MalformedURLException {
+ addAllDefaultXMLFiles();
+
+ final Configuration conf = new Configuration(getConf());
+
+ maybeAddXMLFileOption(conf);
+ maybePatchDefined(conf, DEFINE);
+ // conf.set(IOSTATISTICS_LOGGING_LEVEL, "info");
+
+ return conf;
+ }
+
+ /**
+ * Patch the configuration for maximum S3A performance.
+ *
+ * @param conf config
+ * @return the now updated config
+ */
+ protected Configuration patchForMaxS3APerformance(Configuration conf) {
+ conf.set(DIRECTORY_MARKER_RETENTION, "keep");
+ final int workers = 256;
+ conf.setInt(FS_S3A_CONNECTION_MAXIMUM, workers * 2);
+ conf.setInt(FS_S3A_THREADS_MAX, workers);
+ conf.set(INPUT_FADVISE, INPUT_FADV_NORMAL);
+ return conf;
+ }
+
+ protected void printFSInfoInVerbose(FileSystem fs) {
+ if (isVerbose()) {
+ println();
+
+ println("FileSystem %s", fs.getUri());
+ println();
+ println("%s", fs);
+ println();
+ }
+ }
+
+ /**
+ * @param operation
+ * @param tracker
+ * @param sizeBytes
+ * @param blockName
+ * @param blockSummary
+ */
+ protected void summarize(String operation, StoreDurationInfo tracker, long sizeBytes,
+ final String blockName, final MinMeanMax blockSummary) {
+ heading("%s Summary", operation);
+ println("Data size %,d bytes", sizeBytes);
+ println("%s duration %s", operation, tracker.getDurationString());
+ println();
+ final long durationMillis = tracker.value();
+ // now calculated it in MBits/GBits
+ double seconds = durationMillis / 1000.0;
+ if (seconds < 1) {
+ seconds = 1;
+ }
+ double bitsPerSecond = sizeBytes * 8.0 / seconds;
+ double megabitsPerSecond = bitsPerSecond / MB_1;
+ double megabytesPerSecond = megabitsPerSecond / 8;
+
+ println("%s bandwidth in Megabits/second %,.3f Mbit/s", operation, megabitsPerSecond);
+ println("%s bandwidth in Megabytes/second %,.3f MB/s", operation, megabytesPerSecond);
+ if (blockSummary != null) {
+ println("%s %d: min %.3f seconds, max %.3f seconds, mean %.3f seconds,", blockName,
+ blockSummary.samples(), blockSummary.min() / 1000.0, blockSummary.max() / 1000.0,
+ blockSummary.mean() / 1000.0);
+ }
+ println();
+ }
+
+ protected Optional<Long> getOptionalLong(final String s) {
+ return getOptional(s).map(Long::valueOf);
+ }
+
+ protected long getLongOption(final String s, long def) {
+ return getOptional(s).map(Long::valueOf).orElse(def);
+ }
+
+ /**
+ * Print the selected options in a config. This is an array of (name, secret, obfuscate) entries.
+ *
+ * @param title heading to print
+ * @param conf source configuration
+ * @param options map of options
+ */
+ @Override
+ public final void printOptions(String title, Configuration conf, Object[][] options)
+ throws IOException {
+ int index = 0;
+ if (options.length > 0) {
+ heading(title);
+ for (final Object[] option : options) {
+ printOption(conf, ++index, (String) option[0], (Boolean) option[1], (Boolean) option[2]);
+ }
+ }
+ }
+
+ /**
+ * Sanitize a value if needed.
+ *
+ * @param value option value.
+ * @param obfuscate should it be obfuscated?
+ * @return string safe to log; in quotes
+ */
+ @Override
+ public String maybeSanitize(String value, boolean obfuscate) {
+ return obfuscate ? StoreUtils.sanitize(value, isHideAllSensitiveChars())
+ : ("\"" + value + "\"");
+ }
+
+ @Override
+ public void printOption(Configuration conf, final int index, final String key,
+ final boolean secret, boolean obfuscate) throws IOException {
+ if (key.isEmpty()) {
+ return;
+ }
+ String source = "";
+ String option;
+ String suffix = "";
+ if (secret) {
+ // secret: look up the password.
+ try {
+ final char[] password = conf.getPassword(key);
+ if (password != null) {
+ option = new String(password).trim();
+ source = "<credentials>";
+ } else {
+ option = null;
+ }
+ } catch (IOException e) {
+ // can be triggered by jceks
+ logJceksFailureOnce.warn("Failed to read key {}", key, e);
+ option = "failed: " + e;
+ obfuscate = false;
+ }
+ } else {
+ // not secret. Simple get and look for and print the raw value if different.
+ option = conf.get(key);
+ final String raw = conf.getRaw(key);
+ if (option != null && !option.equals(raw)) {
+ suffix = String.format("; (\"%s\")", raw);
+ }
+ }
+ String full;
+ if (option == null) {
+ full = "(unset)";
+ } else {
+ option = maybeSanitize(option, obfuscate);
+ source = getOrigins(conf, key, source);
+ full = option + " " + source;
+ }
+ // pretty inefficient to do this every option, but
+ // it's a bit late to fix.
+ final Set<String> finalParameters = conf.getFinalParameters();
+ if (finalParameters.contains(key)) {
+ full = full + "[final]";
+ }
+ println("[%03d] %s = %s%s", index, key, full, suffix);
+ }
+
+ /**
+ * Get the origin of a config as a string.
+ *
+ * @param conf configuration
+ * @param key key
+ * @param sourceDefault default string.
+ * @return a source string
+ */
+ public static String getOrigins(final Configuration conf, final String key,
+ String sourceDefault) {
+ String source = sourceDefault;
+ String[] origins = conf.getPropertySources(key);
+ if (origins != null && origins.length != 0) {
+ source = "[" + StringUtils.join(",", origins) + "]";
+ }
+ return source;
+ }
+
+ @Override
+ public boolean isHideAllSensitiveChars() {
+ return hideAllSensitiveChars;
+ }
+
+ @Override
+ public void setHideAllSensitiveChars(boolean hideAllSensitiveChars) {
+ this.hideAllSensitiveChars = hideAllSensitiveChars;
+ }
+
+ protected static final class LimitReachedException extends IOException {
+
+ private static final long serialVersionUID = -8594688586071585301L;
+
+ public LimitReachedException() {
+ super("Limit reached");
+ }
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/StoreUtils.java b/src/main/java/org/apache/hadoop/fs/store/StoreUtils.java
new file mode 100644
index 0000000..cfa3419
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/StoreUtils.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store;
+
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString;
+import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ExitUtil;
+
+/** General utils, many here to avoid adding external dependencies. */
+public final class StoreUtils {
+
+ private StoreUtils() {}
+
+
+ /** {@value}. */
+ private static final int HIDE_PREFIX = 2;
+
+ /** {@value}. */
+ private static final int HIDE_SUFFIX = 4;
+
+ /** Shorter strings are hidden completely: {@value}. */
+ private static final int HIDE_THRESHOLD = 12;
+
+ /**
+ * Take an exception from a Future and convert to an IOE.
+ *
+ * @param ex exception
+ * @return the extracted or wrapped exception
+ */
+ public static IOException uprate(ExecutionException ex) {
+ Throwable cause = ex.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ if (cause instanceof IOException) {
+ return (IOException) cause;
+ }
+ return new IOException(cause.toString(), cause);
+ }
+
+ /**
+ * Await a future completing; uprate failures.
+ *
+ * @param future future to exec
+ * @param <T> return type
+ * @return the result
+ * @throws IOException failure
+ * @throws InterruptedException work interrupted
+ */
+ public static <T> T await(Future<T> future) throws IOException, InterruptedException {
+ try {
+ return future.get();
+ } catch (ExecutionException e) {
+ throw uprate(e);
+ }
+ }
+
+ /**
+ * split a key=value pair. Why not return a Pair class? Commons-lang versions: don't want to
+ * commit.
+ *
+ * @param param param to split
+ * @return a param split key = val,
+ */
+ public static Map.Entry<String, String> split(String param, String defVal) {
+ int split = param.indexOf('=');
+ int len = param.length();
+ if (split == 0 || split + 1 == len) {
+ throw new ExitUtil.ExitException(StoreEntryPoint.EXIT_USAGE,
+ "Unable to parse argument " + param);
+ }
+ String key = split > 0 ? param.substring(0, split) : param;
+ String value = split > 0 ? param.substring(split + 1, len) : defVal;
+ return new StringPair(key, value);
+ }
+
+ /**
+ * Concatenate two arrays into a new one. If either array is empty, the other array is returned;
+ * no new array is created. see
+ * https://stackoverflow.com/questions/80476/how-can-i-concatenate-two-arrays-in-java
+ *
+ * @param left left side
+ * @param right right side
+ * @param <T> type of arrays
+ * @return the expanded array.
+ */
+ public static <T> T[] cat(T[] left, T[] right) {
+ int aLen = left.length;
+ int bLen = right.length;
+ if (aLen == 0) {
+ return right;
+ }
+ if (bLen == 0) {
+ return left;
+ }
+
+ T[] dest = (T[]) Array.newInstance(left.getClass().getComponentType(), aLen + bLen);
+ System.arraycopy(left, 0, dest, 0, aLen);
+ System.arraycopy(right, 0, dest, aLen, bLen);
+
+ return dest;
+ }
+
+ /**
+ * Get the storage size from a string, uses M, G, T etc.
+ *
+ * @param size data size
+ * @param storageUnit desired size unit.
+ * @return size as a double.
+ */
+ public static double getDataSize(final String size, StorageUnit storageUnit) {
+ double uploadSize;
+
+ String s = size.trim().toUpperCase(Locale.ROOT);
+ try {
+ // look for a long value,
+ uploadSize = Long.parseLong(s);
+ } catch (NumberFormatException e) {
+ // parse the size values via Configuration
+ // this is only possible on hadoop 3.1+.
+ if (!s.endsWith("B")) {
+ s = s + "B";
+ }
+ final Configuration sizeConf = new Configuration(false);
+
+ // upload in MB.
+ uploadSize = sizeConf.getStorageSize("size", s, storageUnit);
+ }
+ return uploadSize;
+ }
+
+ /**
+ * Parse a duration argument.
+ *
+ * @param argument arg value
+ * @param defaultValue default value if none is provided
+ * @param defaultUnit default unit if no value is given
+ */
+ public static Duration parseDurationArgument(String argument, long defaultValue,
+ TimeUnit defaultUnit) {
+ final Configuration conf = new Configuration(false);
+ conf.set("key", argument);
+ final long duration = conf.getTimeDuration("key", defaultValue, defaultUnit, TimeUnit.SECONDS);
+ return Duration.of(duration, ChronoUnit.SECONDS);
+ }
+
+ /**
+ * Create a list of star characters.
+ *
+ * @param n number to create.
+ * @return a string of stars
+ */
+ public static String stars(int n) {
+ StringBuilder b = new StringBuilder(n);
+ for (int i = 0; i < n; i++) {
+ b.append('*');
+ }
+ return b.toString();
+ }
+
+ /**
+ * Sanitize a sensitive option.
+ *
+ * @param value option value.
+ * @param hide flag to hide everything
+ * @return sanitized value.
+ */
+ public static String sanitize(final String value, boolean hide) {
+ String safe;
+ int len = value.length();
+ if (!hide && len > HIDE_THRESHOLD) {
+ StringBuilder b = new StringBuilder(len);
+ int prefix = HIDE_PREFIX;
+ int suffix = HIDE_SUFFIX;
+ b.append(value, 0, prefix);
+ b.append(stars(len - prefix - suffix));
+ b.append(value, len - suffix, len);
+ safe = b.toString();
+ } else {
+ // short values get special treatment
+ safe = stars(HIDE_THRESHOLD);
+ }
+ return String.format("\"%s\" [%d]", safe, len);
+ }
+
+ /**
+ * Get any IOStatistics from an object.
+ *
+ * @param source source
+ * @return any IOStats, prettified -else ""
+ */
+ public static String prettyIOStatistics(final Object source) {
+ return ioStatisticsToPrettyString(retrieveIOStatistics(source));
+ }
+
+ /**
+ * String tuple; value is mutable.
+ */
+ public static final class StringPair implements Map.Entry<String, String> {
+
+ private final String key;
+
+ private String value;
+
+ private StringPair(final String left, final String right) {
+ this.key = left;
+ this.value = right;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ @Override
+ public String setValue(String newValue) {
+ this.value = newValue;
+ return newValue;
+ }
+ }
+
+ /**
+ * Read lines, skipping those with # or blank. (generated by copilot.)
+ *
+ * @param file path to file
+ * @return the list of lines
+ * @throws IOException failure to read
+ */
+ public static List<String> readLines(File file) throws IOException {
+ List<String> lines = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.trim().isEmpty() && !line.trim().startsWith("#")) {
+ lines.add(line);
+ }
+ }
+ }
+ return lines;
+ }
+
+ public static boolean isNullOrEmpty(CharSequence seq) {
+ return seq == null || seq.length() == 0;
+ }
+
+
+ /**
+ * Probe to determine if a path is an ancestor of another path. The root path is not considered a
+ * parent for the purposes of this check.
+ *
+ * @param parent the candidate parent path.
+ * @param child the candidate child path.
+ * @return {@code true} if {@code parent} is an ancestor of {@code child}, excluding the root
+ * path.
+ */
+ public static boolean isParentOf(Path parent, Path child) {
+ // root is not considered a valid parent
+ if (parent.isRoot()) {
+ return false;
+ }
+ Path childParent = child.getParent();
+ if (childParent == null || childParent.isRoot()) {
+ return false;
+ }
+ if (childParent.equals(parent)) {
+ return true;
+ }
+ return isParentOf(parent, childParent);
+
+ }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/audit/AuditLogProcessor.java b/src/main/java/org/apache/hadoop/fs/store/audit/AuditLogProcessor.java
new file mode 100644
index 0000000..5e26b99
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/audit/AuditLogProcessor.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store.audit;
+
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.AWS_LOG_REGEXP_GROUPS;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.BYTESSENT_GROUP;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.LOG_ENTRY_PATTERN;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.OBJECTSIZE_GROUP;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.TOTALTIME_GROUP;
+import static org.apache.hadoop.fs.s3a.audit.S3LogParser.TURNAROUNDTIME_GROUP;
+import static org.apache.hadoop.fs.store.StoreUtils.isNullOrEmpty;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.LineRecordReader;
+import org.apache.hadoop.util.DurationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Merge all the audit logs present in a directory of multiple audit log files into a single audit
+ * log file.
+ */
+public class AuditLogProcessor {
+
+ /**
+ * Max length of a line in the audit log: {@value}.
+ */
+ public static final int MAX_LINE_LENGTH = 32000;
+
+ /**
+ * List of fields in a log record which are of type long.
+ */
+ public static final List<String> FIELDS_OF_TYPE_LONG =
+ Arrays.asList(TURNAROUNDTIME_GROUP, BYTESSENT_GROUP, OBJECTSIZE_GROUP, TOTALTIME_GROUP);
+
+ private static final Logger LOG = LoggerFactory.getLogger(AuditLogProcessor.class);
+
+ /**
+ * Referrer header key from the regexp, not the actual header name which is spelled differently:
+ * {@value}.
+ */
+ private static final String REFERRER_HEADER_KEY = "referrer";
+
+ /**
+ * Timestamp key.
+ */
+ private static final String TIMESTAMP_KEY = "timestamp";
+
+ private static final String AVRO_TIMESTAMP_KEY = "tstamp";
+
+ /**
+ * Value to use when a long value cannot be parsed: {@value}.
+ */
+ public static final long FAILED_TO_PARSE_LONG = -1L;
+
+ /**
+ * Key for the referrer map in the Avro record: {@value}.
+ */
+ public static final String REFERRER_MAP = "audit";
+
+ /**
+ * Format of S3 log timestamps like "13/May/2021:11:26:06 +0000".
+ */
+ private static final DateTimeFormatter TS_FORMATTER =
+ DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH);
+
+ public static final Instant EPOCH_START = Instant.ofEpochMilli(0);
+
+ private final Configuration conf;
+
+ /**
+ * Number of records to process before giving a status update.
+ */
+ private final int sample;
+
+ // Basic parsing counters.
+
+ /**
+ * Number of audit log files parsed.
+ */
+ private long logFilesParsed = 0;
+
+ /**
+ * Number of log entries parsed.
+ */
+ private long logRecordsProcessed = 0;
+
+ /**
+ * How many referrer headers were parsed.
+ */
+ private long referrerHeadersParsed = 0;
+
+ /**
+ * How many records were skipped due to lack of referrer or other reason.
+ */
+ private long recordsSkipped = 0;
+
+ public AuditLogProcessor(final Configuration conf, final int sample) {
+ this.conf = conf;
+
+ this.sample = sample;
+ }
+
+ /**
+ * parseAuditLog method helps in parsing the audit log into key-value pairs using regular
+ * expressions.
+ *
+ * @param singleAuditLog this is single audit log from merged audit log file
+ * @return it returns a map i.e, auditLogMap which contains key-value pairs of a single audit log
+ */
+ public HashMap<String, String> parseAuditLog(String singleAuditLog) {
+ HashMap<String, String> auditLogMap = new HashMap<>();
+ if (singleAuditLog == null || singleAuditLog.isEmpty()) {
+ LOG.debug("This is an empty string or null string, expected a valid string to parse");
+ return auditLogMap;
+ }
+ final Matcher matcher = LOG_ENTRY_PATTERN.matcher(singleAuditLog);
+ boolean patternMatched = matcher.matches();
+ if (patternMatched) {
+ for (String key : AWS_LOG_REGEXP_GROUPS) {
+ try {
+ final String value = matcher.group(key);
+ auditLogMap.put(key, value);
+ } catch (IllegalStateException e) {
+ LOG.debug("Skipping key :{} due to no match with the audit log " + "pattern :", key);
+ LOG.debug(String.valueOf(e));
+ }
+ }
+ }
+ return auditLogMap;
+ }
+
+ /**
+ * Parses the http referrer header. which is one of the key-value pair of audit log.
+ *
+ * @param referrerHeader this is the http referrer header of a particular audit log.
+ * @return returns a map which contains key-value pairs of referrer headers; an empty map there
+ * was no referrer header or parsing failed
+ */
+ public static HashMap<String, String> parseAuditHeader(String referrerHeader) {
+ HashMap<String, String> referrerHeaderMap = new HashMap<>();
+
+ if (isNullOrEmpty(referrerHeader) || referrerHeader.equals("-")) {
+ LOG.debug("This is an empty string or null string, expected a valid string to parse");
+ return referrerHeaderMap;
+ }
+
+ // '?' used as the split point between the headers and the url. This
+ // returns the first occurrence of '?'
+ int indexOfQuestionMark = referrerHeader.indexOf("?");
+ if (indexOfQuestionMark < 0 || indexOfQuestionMark + 1 == referrerHeader.length()) {
+ LOG.debug("no ? in the string/args after the ?");
+ return referrerHeaderMap;
+ }
+ String httpReferrer =
+ referrerHeader.substring(indexOfQuestionMark + 1, referrerHeader.length() - 1);
+
+ int lengthOfReferrer = httpReferrer.length();
+ int start = 0;
+ LOG.debug("HttpReferrer headers string: {}", httpReferrer);
+ while (start < lengthOfReferrer) {
+ // splits "key" and "value" of each header
+ int equals = httpReferrer.indexOf("=", start);
+ // no match : break, no header left
+ if (equals == -1) {
+ break;
+ }
+ // key represents the string between "start" and index of "=".
+ String key = httpReferrer.substring(start, equals);
+ // splits between different headers, this also helps in ignoring "="
+ // inside values since we set the end till we find '&'
+ int end = httpReferrer.indexOf("&", equals);
+ // or end of string
+ if (end == -1) {
+ end = lengthOfReferrer;
+ }
+ // value represents the string between index of "=" + 1 and the "end"
+ String value = httpReferrer.substring(equals + 1, end);
+ referrerHeaderMap.put(key, value);
+ start = end + 1;
+ }
+
+ return referrerHeaderMap;
+ }
+
+ /**
+ * Merge and parse all the audit log files and convert data into avro file.
+ *
+ * @param logsPath source path of logs
+ * @param destFile destination path of merged log file
+ * @param overwrite overwrite the dest
+ * @param processed processing callback.
+ * @return number of records parsed.
+ * @throws IOException on any failure
+ */
+ public long mergeAndParseAuditLogFiles(final Path logsPath, final Path destFile,
+ boolean overwrite, Processed processed) throws IOException {
+
+ logRecordsProcessed = 0;
+ logFilesParsed = 0;
+ // List source log files
+ final FileSystem sourceFS = logsPath.getFileSystem(conf);
+ RemoteIterator<LocatedFileStatus> listOfLogFiles = sourceFS.listFiles(logsPath, true);
+
+ final FileSystem destFS = destFile.getFileSystem(conf);
+ final FSDataOutputStreamBuilder builder =
+ destFS.createFile(destFile).recursive().overwrite(overwrite);
+
+ FSDataOutputStream fsDataOutputStream = builder.build();
+
+ // Instantiate DatumWriter class
+ DatumWriter<AvroS3LogEntryRecord> datumWriter =
+ new SpecificDatumWriter<>(AvroS3LogEntryRecord.class);
+ boolean continueProcessing = true;
+
+ try (DataFileWriter<AvroS3LogEntryRecord> dataFileWriter = new DataFileWriter<>(datumWriter);
+ DataFileWriter<AvroS3LogEntryRecord> avroWriter =
+ dataFileWriter.create(AvroS3LogEntryRecord.getClassSchema(), fsDataOutputStream);) {
+
+ // Iterating over the list of files to merge and parse
+ while (continueProcessing && listOfLogFiles.hasNext()) {
+ FileStatus fileStatus = listOfLogFiles.next();
+ int fileLength = (int) fileStatus.getLen();
+
+ try (
+ DurationInfo duration = new DurationInfo(LOG, "[%05d] Processing %s",
+ getLogFilesParsed() + 1, fileStatus.getPath());
+ FSDataInputStream fsDataInputStream =
+ awaitFuture(sourceFS.openFile(fileStatus.getPath()).withFileStatus(fileStatus)
+ .opt(FS_OPTION_OPENFILE_READ_POLICY, FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+ .build())) {
+ logFilesParsed++;
+
+ // Reading the file data using LineRecordReader
+ LineRecordReader lineRecordReader =
+ new LineRecordReader(fsDataInputStream, 0L, fileLength, MAX_LINE_LENGTH);
+ LongWritable longWritable = new LongWritable();
+ Text singleAuditLog = new Text();
+
+ // Parse each and every audit log from list of logs
+ while (continueProcessing && lineRecordReader.next(longWritable, singleAuditLog)) {
+ // Parse audit log
+ HashMap<String, String> auditLogMap = parseAuditLog(singleAuditLog.toString());
+ logRecordsProcessed++;
+
+ // Insert data according to schema
+
+ // Instantiating generated AvroDataRecord class
+ AvroS3LogEntryRecord avroDataRecord = buildLogRecord(auditLogMap);
+ if (!avroDataRecord.getAudit().isEmpty()) {
+ referrerHeadersParsed++;
+ }
+ avroWriter.append(avroDataRecord);
+ continueProcessing = processed.processed(avroDataRecord);
+ }
+ dataFileWriter.flush();
+ }
+ }
+ }
+
+ LOG.info("Successfully parsed: {} records with {} referrer headers " + "in the logs",
+ logRecordsProcessed, referrerHeadersParsed);
+ return logRecordsProcessed;
+ }
+
+ public long getLogRecordsProcessed() {
+ return logRecordsProcessed;
+ }
+
+ public long getLogFilesParsed() {
+ return logFilesParsed;
+ }
+
+ public long getReferrerHeadersParsed() {
+ return referrerHeadersParsed;
+ }
+
+ /**
+ * Build log record from a parsed audit log entry.
+ *
+ * @param auditLogMap parsed audit log entry.
+ * @return the Avro record.
+ */
+ public AvroS3LogEntryRecord buildLogRecord(Map<String, String> auditLogMap) {
+
+ // Instantiating generated AvroDataRecord class
+ AvroS3LogEntryRecord avroDataRecord = new AvroS3LogEntryRecord();
+
+ // special timestamp handling. This is a mandatory field, but we're being
+ // cautious to keep the audit tools happy.
+ String timestamp = auditLogMap.get(TIMESTAMP_KEY);
+ timestamp = timestamp.substring(1, timestamp.length() - 1);
+ try {
+ avroDataRecord.setEvent(parseToInstant(timestamp).orElseGet(() -> EPOCH_START));
+ } catch (DateTimeParseException e) {
+ avroDataRecord.setEvent(EPOCH_START);
+ }
+ avroDataRecord.setTstamp(timestamp);
+
+ for (Map.Entry<String, String> entry : auditLogMap.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue().trim();
+
+ // fixup
+ if (TIMESTAMP_KEY.equals(key)) {
+ // timestamp handler.
+ continue;
+ }
+
+ // if value == '-' and key is not in arraylist then put '-' or else '-1'
+ // if key is in arraylist of long values then parse the long value
+ // while parsing do it in try-catch block,
+ // in catch block need to log exception and set value as '-1'
+ try {
+ if (FIELDS_OF_TYPE_LONG.contains(key)) {
+ if (value.equals("-")) {
+ avroDataRecord.put(key, null);
+ } else {
+ try {
+ avroDataRecord.put(key, Long.parseLong(value));
+ } catch (NumberFormatException e) {
+ // failed to parse the long value.
+ LOG.debug("Failed to parse long value for key {} : {}", key, value);
+ avroDataRecord.put(key, FAILED_TO_PARSE_LONG);
+ }
+ }
+ } else {
+ avroDataRecord.put(key, value);
+ }
+ } catch (Exception e) {
+ avroDataRecord.put(key, null);
+ }
+ }
+
+ // Parse the audit header
+ HashMap<String, String> referrerHeaderMap =
+ parseAuditHeader(auditLogMap.get(REFERRER_HEADER_KEY));
+ avroDataRecord.put(REFERRER_MAP, referrerHeaderMap);
+
+ return avroDataRecord;
+
+ // build the event header
+
+ }
+
+ @Override
+ public String toString() {
+ String sb = "AuditLogProcessor{" + "logFilesParsed=" + logFilesParsed + ", logRecordsProcessed="
+ + logRecordsProcessed + ", referrerHeadersParsed=" + referrerHeadersParsed
+ + ", recordsSkipped=" + recordsSkipped + '}';
+ return sb;
+ }
+
+ /**
+ * Parse the input string to an Instant (UTC).
+ *
+ * @throws DateTimeParseException if the input cannot be parsed.
+ */
+ public static Optional<Instant> parseToInstant(String input) {
+ if (!isNullOrEmpty(input)) {
+ OffsetDateTime odt = OffsetDateTime.parse(input, TS_FORMATTER);
+ return Optional.of(odt.toInstant());
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * Parse the input string to the epoch millis; fallback is 0.
+ *
+ * @param input input string.
+ * @return parsed value or 0 on a failure.
+ */
+ public static long parseToEpochMillis(String input) {
+ if (isNullOrEmpty(input)) {
+ return 0;
+ }
+ try {
+ return parseToInstant(input).map(Instant::toEpochMilli).orElse(0L);
+ } catch (DateTimeParseException e) {
+ return 0;
+ }
+ }
+
+ @FunctionalInterface
+ public interface Processed {
+ // callback on process; return true to continue processing
+ boolean processed(AvroS3LogEntryRecord record);
+ }
+
+ public static final Processed PROCESS_ALL = record -> true;
+}
diff --git a/src/main/java/org/apache/hadoop/fs/store/audit/AuditTool.java b/src/main/java/org/apache/hadoop/fs/store/audit/AuditTool.java
new file mode 100644
index 0000000..89bf6d8
--- /dev/null
+++ b/src/main/java/org/apache/hadoop/fs/store/audit/AuditTool.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.store.audit;
+
+import static org.apache.hadoop.fs.store.CommonParameters.STANDARD_OPTS;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_FAIL;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_SUCCESS;
+
+import java.util.List;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.store.StoreEntryPoint;
+import org.apache.hadoop.util.DurationInfo;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * AuditTool is a Command Line Interface. Its functionality is to parse the audit log files and
+ * generate avro file.
+ */
+public class AuditTool extends StoreEntryPoint {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AuditTool.class);
+
+ /**
+ * Name of audit tool: {@value}.
+ */
+ public static final String AUDIT = "audit";
+
+ /**
+ * Name of this tool: {@value}.
+ */
+ public static final String AUDIT_TOOL = "auditlogs";
+
+ /**
+ * Purpose of this tool: {@value}.
+ */
+ public static final String PURPOSE =
+ "\n\nUSAGE:\nMerge and parse audit log files and convert into avro files "
+ + "for better visualization";
+
+ // Exit codes
+ private static final int SUCCESS = EXIT_SUCCESS;
+
+ private static final int FAILURE = EXIT_FAIL;
+
+ private static final int INVALID_ARGUMENT = EXIT_COMMAND_ARGUMENT_ERROR;
+
+ private static final int SAMPLE = 500;
+
+ public static final String OVERWRITE = "overwrite";
+
+ public static final String USAGE =
+ "Usage: auditlogs\n" + STANDARD_OPTS + " <path of source files>" + " <path of output file>"
+ + optusage(OVERWRITE, "overwrite the output file");
+
+ public AuditTool() {
+ createCommandFormat(2, 2, OVERWRITE);
+ }
+
+ /**
+ * Tells us the usage of the AuditTool by commands.
+ *
+ * @return the string USAGE
+ */
+ public String getUsage() {
+ return USAGE + PURPOSE;
+ }
+
+ public String getName() {
+ return AUDIT_TOOL;
+ }
... 20994 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org