You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fa...@apache.org on 2026/03/26 21:06:42 UTC
(hadoop) branch branch-3.5 updated: HADOOP-18878: deprecate fs.s3a.multipart.purge (#8353)
This is an automated email from the ASF dual-hosted git repository.
fabbri pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 5a8abd3d96a HADOOP-18878: deprecate fs.s3a.multipart.purge (#8353)
5a8abd3d96a is described below
commit 5a8abd3d96ab491a7391ff315019310bca14b2db
Author: Aaron Fabbri <fa...@apache.org>
AuthorDate: Thu Mar 26 13:45:22 2026 -0700
HADOOP-18878: deprecate fs.s3a.multipart.purge (#8353)
---
.../hadoop-common/src/main/resources/core-default.xml | 4 ++--
.../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 2 ++
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 11 ++++++-----
.../site/markdown/tools/hadoop-aws/third_party_stores.md | 6 ++----
hadoop-tools/hadoop-aws/src/test/resources/core-site.xml | 14 --------------
5 files changed, 12 insertions(+), 25 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9859b8b94a4..21446d52138 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1766,7 +1766,7 @@
<property>
<name>fs.s3a.multipart.purge</name>
<value>false</value>
- <description>True if you want to purge existing multipart uploads that may not have been
+ <description>Deprecated. True if you want to purge existing multipart uploads that may not have been
completed/aborted correctly. The corresponding purge age is defined in
fs.s3a.multipart.purge.age.
If set, when the filesystem is instantiated then all outstanding uploads
@@ -1780,7 +1780,7 @@
<property>
<name>fs.s3a.multipart.purge.age</name>
<value>24h</value>
- <description>Minimum age in seconds of multipart uploads to purge
+ <description>Deprecated. Minimum age in seconds of multipart uploads to purge
on startup if "fs.s3a.multipart.purge" is true
</description>
</property>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 75db90092d1..271e9302a3d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -704,6 +704,7 @@ private Constants() {
public static final String STORAGE_CLASS_GLACIER_INSTANT_RETRIEVAL = "glacier_ir";
// should we try to purge old multipart uploads when starting up
+ @Deprecated
public static final String PURGE_EXISTING_MULTIPART =
"fs.s3a.multipart.purge";
public static final boolean DEFAULT_PURGE_EXISTING_MULTIPART = false;
@@ -711,6 +712,7 @@ private Constants() {
/**
* purge any multipart uploads older than this number of seconds.
*/
+ @Deprecated
public static final String PURGE_EXISTING_MULTIPART_AGE =
"fs.s3a.multipart.purge.age";
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 3ea65e4f9bb..88fcd2db7fb 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -585,7 +585,7 @@ Here are some the S3A properties for use in production.
<property>
<name>fs.s3a.multipart.purge</name>
<value>false</value>
- <description>True if you want to purge existing multipart uploads that may not have been
+ <description>Deprecated. True if you want to purge existing multipart uploads that may not have been
completed/aborted correctly. The corresponding purge age is defined in
fs.s3a.multipart.purge.age.
If set, when the filesystem is instantiated then all outstanding uploads
@@ -599,7 +599,7 @@ Here are some the S3A properties for use in production.
<property>
<name>fs.s3a.multipart.purge.age</name>
<value>86400</value>
- <description>Minimum age in seconds of multipart uploads to purge
+ <description>Deprecated. Minimum age in seconds of multipart uploads to purge
on startup if "fs.s3a.multipart.purge" is true
</description>
</property>
@@ -1676,14 +1676,15 @@ intermediate partitions uploaded to S3 —data which will be billed for.
If an S3A committer job is halted partway through, again, there may be
many incomplete multipart uploads in the output directory.
-These charges can be reduced by enabling `fs.s3a.multipart.purge`,
-and setting a purge time in seconds, such as 24 hours.
+These charges can be reduced by enabling `fs.s3a.multipart.purge`
+(deprecated), and setting a purge time in seconds, such as 24 hours.
When an S3A FileSystem instance is instantiated with the purge time greater
than zero, it will, on startup, delete all outstanding partition requests
older than this time. However, this makes filesystem instantiate slow, especially
against very large buckets, as a full scan is made.
-Consider avoiding this in future.
+For this reason, the option is deprecated and will be removed in a future
+release.
```xml
<property>
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
index 0336efa677c..64096188924 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/third_party_stores.md
@@ -113,7 +113,6 @@ of incomplete uploads.
This can be addressed in two ways
* Command line: `hadoop s3guard uploads -abort -force \<path>`.
-* With `fs.s3a.multipart.purge` and a purge age set in `fs.s3a.multipart.purge.age`
* In rename/delete `fs.s3a.directory.operations.purge.uploads = true`.
#### S3Guard uploads command
@@ -126,7 +125,7 @@ hadoop s3guard uploads -abort -force s3a://bucket/
Consult the [S3Guard documentation](s3guard.html) for the full set of parameters.
-#### In startup: `fs.s3a.multipart.purge`
+#### Deprecated: `fs.s3a.multipart.purge`
This lists all uploads in a bucket when a filesystem is created and deletes
all of those above a certain age.
@@ -135,8 +134,7 @@ This can hurt performance on a large bucket, as the purge scans the entire tree,
and is executed whenever a filesystem is created -which can happen many times during
hive, spark, distcp jobs.
-For this reason, this option may be deleted in future, however it has long been
-available in the S3A client and so guaranteed to work across versions.
+For this reason, this option is deprecated and will be deleted in future.
#### During rename and delete: `fs.s3a.directory.operations.purge.uploads`
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index 1abec3e0706..b6d1604b94f 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -58,13 +58,6 @@
<value>us-east-1</value>
</property>
- <property>
- <name>fs.s3a.bucket.noaa-isd-pds.multipart.purge</name>
- <value>false</value>
- <description>Don't try to purge uploads in the read-only bucket, as
- it will only create log noise.</description>
- </property>
-
<property>
<name>fs.s3a.bucket.noaa-isd-pds.probe</name>
<value>0</value>
@@ -101,13 +94,6 @@
<description>usgs-landsat requires requester pays enabled</description>
</property>
- <property>
- <name>fs.s3a.bucket.usgs-landsat.multipart.purge</name>
- <value>false</value>
- <description>Don't try to purge uploads in the read-only bucket, as
- it will only create log noise.</description>
- </property>
-
<property>
<name>fs.s3a.bucket.usgs-landsat.audit.add.referrer.header</name>
<value>false</value>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org