You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2026/08/08 11:43:32 UTC
(asterixdb) branch master updated: [ASTERIXDB-3801][EXT] Cap COPY TO max-file-size at 5GB
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 927438c233 [ASTERIXDB-3801][EXT] Cap COPY TO max-file-size at 5GB
927438c233 is described below
commit 927438c233aee2f06d8967dec9ff6a9584ae885a
Author: rithwik.koul <ri...@couchbase.com>
AuthorDate: Thu Aug 6 19:14:54 2026 +0530
[ASTERIXDB-3801][EXT] Cap COPY TO max-file-size at 5GB
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Lower the upper bound of the max-file-size WITH option from 5TB to 5GB,
The lower bound of 5MB is unchanged.
Ext-ref: MB-70477
Change-Id: I6acc84ee6dae890dc5266b68678ef8d6a6e9e1e0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21498
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Rithwik Koul <ri...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../runtimets/testsuite_external_dataset_azure_blob_storage.xml | 2 +-
.../src/test/resources/runtimets/testsuite_external_dataset_s3.xml | 2 +-
.../src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml | 2 +-
asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md | 6 +++---
.../org/apache/asterix/external/util/ExternalDataConstants.java | 2 +-
.../asterix/external/util/WriterMaxFileSizeValidationTest.java | 4 ++--
6 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 0efb85a3ca..b0feb80066 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -283,7 +283,7 @@
<output-dir compare="Text">bad-max-file-size</output-dir>
<expected-error>Minimum value allowed for 'max-file-size' is 5 MiB. Found 1MB</expected-error>
<expected-error>ASX1206: Storage units expected for the field 'max-file-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
- <expected-error>Maximum value allowed for 'max-file-size' is 5 TiB. Found 6TB</expected-error>
+ <expected-error>Maximum value allowed for 'max-file-size' is 5 GiB. Found 6TB</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="copy-to/negative">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index c4c395d2ea..33875b28ea 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -392,7 +392,7 @@
<output-dir compare="Text">bad-max-file-size</output-dir>
<expected-error>Minimum value allowed for 'max-file-size' is 5 MiB. Found 1MB</expected-error>
<expected-error>ASX1206: Storage units expected for the field 'max-file-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
- <expected-error>Maximum value allowed for 'max-file-size' is 5 TiB. Found 6TB</expected-error>
+ <expected-error>Maximum value allowed for 'max-file-size' is 5 GiB. Found 6TB</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="copy-to/negative">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
index f50169c16e..2d192470de 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
@@ -302,7 +302,7 @@
<output-dir compare="Text">bad-max-file-size</output-dir>
<expected-error>Minimum value allowed for 'max-file-size' is 5 MiB. Found 1MB</expected-error>
<expected-error>ASX1206: Storage units expected for the field 'max-file-size' (e.g., 0.1KB, 100kb, 1mb, 3MB, 8.5GB ...). Provided 'random'</expected-error>
- <expected-error>Maximum value allowed for 'max-file-size' is 5 TiB. Found 6TB</expected-error>
+ <expected-error>Maximum value allowed for 'max-file-size' is 5 GiB. Found 6TB</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index c149d5aeb4..ffc61bcdcb 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@ -789,9 +789,9 @@ Each partition itself will also be sorted by the `name` field, and compressed wi
objects or fewer per file.
A file is also started afresh once it reaches `max-file-size`, so whichever of the two limits is reached first ends the
-current file. `max-file-size` accepts a storage unit string such as `5MB` or `1GB`, has a minimum of `5MB`, and is
+current file. `max-file-size` accepts a storage unit string such as `5MB` or `1GB`, ranges from `5MB` to `5GB`, and is
unbounded when it is not specified. It is measured on the bytes written to the target after compression and is
approximate: an object is never split across files, so the object that reaches the limit takes the file past it. For
`parquet` the size is only observable at row group boundaries, so `max-file-size` should be set comfortably above
-`row-group-size`. The largest file the target itself accepts is generally lower than the maximum accepted here, as it
-depends on the target's multipart upload limits.
+`row-group-size`. The largest file a target itself accepts may be lower still, as it depends on the target's multipart
+upload limits.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index d115e2f005..2b9e29de7c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -388,7 +388,7 @@ public class ExternalDataConstants {
public static final long WRITER_MAX_FILE_SIZE_MINIMUM =
StorageUtil.getLongSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE);
public static final long WRITER_MAX_FILE_SIZE_MAXIMUM =
- StorageUtil.getLongSizeInBytes(5, StorageUtil.StorageUnit.TERABYTE);
+ StorageUtil.getLongSizeInBytes(5, StorageUtil.StorageUnit.GIGABYTE);
public static final Set<String> WRITER_SUPPORTED_FORMATS;
public static final Set<String> WRITER_SUPPORTED_ADAPTERS;
public static final Set<String> TEXTUAL_WRITER_SUPPORTED_COMPRESSION;
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/WriterMaxFileSizeValidationTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/WriterMaxFileSizeValidationTest.java
index 13966afbe1..0c9f9ee50d 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/WriterMaxFileSizeValidationTest.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/util/WriterMaxFileSizeValidationTest.java
@@ -44,8 +44,8 @@ public class WriterMaxFileSizeValidationTest {
validate("5MB");
validate("128MB");
validate("1GB");
- validate("8.5GB");
- validate("5TB");
+ validate("4.5GB");
+ validate("5GB");
validate("10485760");
}