You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by li...@apache.org on 2025/08/18 03:53:38 UTC
(texera) branch master updated: feat: enforce S3 multipart-upload limits in configuration (#3664)
This is an automated email from the ASF dual-hosted git repository.
linxinyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/master by this push:
new 53bdcce600 feat: enforce S3 multipart-upload limits in configuration (#3664)
53bdcce600 is described below
commit 53bdcce600c10898e8c34438739e121b4b4bb792
Author: Xuan Gu <16...@users.noreply.github.com>
AuthorDate: Sun Aug 17 20:53:33 2025 -0700
feat: enforce S3 multipart-upload limits in configuration (#3664)
---
.../admin/settings/admin-settings.component.html | 27 ++++++++++++++--------
.../admin/settings/admin-settings.component.ts | 18 ++++++++++++++-
2 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
index a158389102..c6dfa05701 100644
--- a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
+++ b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.html
@@ -245,32 +245,41 @@
<nz-input-number
[(ngModel)]="maxFileSizeMB"
[nzMin]="1"
- [nzStep]="10">
+ [nzMax]="MAX_FILE_SIZE_MB"
+ [nzStep]="10"
+ [nzPrecision]="0">
</nz-input-number>
</div>
- <div class="help-text-number">Maximum size allowed for individual file uploads (MB).</div>
+ <div class="help-text-number">
+ Maximum size allowed for individual file uploads (MB). (Range: 1 MB - {{ MAX_FILE_SIZE_MB }} MB (5 TiB))
+ </div>
<div class="settings-row">
<span>Concurrent Parts:</span>
<nz-input-number
[(ngModel)]="maxConcurrentChunks"
[nzMin]="1"
- [nzStep]="1">
+ [nzMax]="MAX_TOTAL_PARTS"
+ [nzStep]="1"
+ [nzPrecision]="0">
</nz-input-number>
</div>
- <div class="help-text-number">
- Number of file chunks that can be uploaded simultaneously. Higher values use more memory.
- </div>
+ <div class="help-text-number">Number of file chunks that can be uploaded simultaneously. (Range: 1 - 10000)</div>
<div class="settings-row">
<span>Part Size:</span>
<nz-input-number
[(ngModel)]="chunkSizeMB"
- [nzMin]="1"
- [nzStep]="10">
+ [nzMin]="MIN_PART_SIZE_MB"
+ [nzMax]="MAX_PART_SIZE_MB"
+ [nzStep]="10"
+ [nzPrecision]="0">
</nz-input-number>
</div>
- <div class="help-text-number">Size of each chunk during multipart upload in MB. Larger chunks use more memory.</div>
+ <div class="help-text-number">
+ Size of each chunk during multipart upload in MB. (Range: {{ MIN_PART_SIZE_MB }} MB - {{ MAX_PART_SIZE_MB }} MB (5
+ GiB))
+ </div>
<div class="button-row">
<button
diff --git a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
index 9633bb19b6..863ea67b81 100644
--- a/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
+++ b/core/gui/src/app/dashboard/component/admin/settings/admin-settings.component.ts
@@ -52,6 +52,12 @@ export class AdminSettingsComponent implements OnInit {
maxConcurrentChunks: number = 10;
chunkSizeMB: number = 50;
+ // S3 Multipart Upload Constraints
+ readonly MIN_PART_SIZE_MB = 5; // 5 MiB minimum for parts (except last part)
+ readonly MAX_PART_SIZE_MB = 5120; // 5 GiB maximum per part (5 * 1024 MiB)
+ readonly MAX_FILE_SIZE_MB = 5242880; // 5 TiB maximum object size (5 * 1024 * 1024 MiB)
+ readonly MAX_TOTAL_PARTS = 10000; // S3 maximum parts per upload
+
constructor(
private adminSettingsService: AdminSettingsService,
private message: NzMessageService
@@ -205,7 +211,17 @@ export class AdminSettingsComponent implements OnInit {
saveDatasetSettings(): void {
if (this.maxFileSizeMB < 1 || this.maxConcurrentChunks < 1 || this.chunkSizeMB < 1) {
- this.message.info("Value must be at least 1.");
+ this.message.error("Please enter valid integer values.");
+ return;
+ }
+
+ const partsAtMax = Math.ceil(this.maxFileSizeMB / this.chunkSizeMB);
+ if (partsAtMax > this.MAX_TOTAL_PARTS) {
+ const requiredMin = Math.max(this.MIN_PART_SIZE_MB, Math.ceil(this.maxFileSizeMB / this.MAX_TOTAL_PARTS));
+ this.message.error(
+ `This setting would create ${partsAtMax.toLocaleString()} parts (>10,000). ` +
+ `Increase "Part Size" to at least ${requiredMin} MB or reduce "File Size".`
+ );
return;
}