You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@amoro.apache.org by xu...@apache.org on 2026/07/31 07:18:24 UTC
(amoro) 01/01: Merge remote-tracking branch 'origin/0.9.0-rc4' into 0.9.x
This is an automated email from the ASF dual-hosted git repository.
xxubai pushed a commit to branch 0.9.x
in repository https://gitbox.apache.org/repos/asf/amoro.git
commit 30580a3aee87265ec5d97192e7c60f91ef0b58ef
Merge: 7a65790f4 a93a32b4d
Author: xuba <xu...@cisco.com>
AuthorDate: Fri Jul 31 14:48:36 2026 +0800
Merge remote-tracking branch 'origin/0.9.0-rc4' into 0.9.x
# Conflicts:
# amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
# amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConfValidator.java
# amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/DanglingDeleteFilesCleaningProcess.java
# amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/DataExpiringProcess.java
# amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/IcebergProcessFactory.java
# amoro-ams/src/main/java/org/apache/amoro/server/process/iceberg/OrphanFilesCleaningProcess.java
# amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/InlineTableExecutors.java
# amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java
# amoro-ams/src/test/java/org/apache/amoro/server/AmsEnvironment.java
# amoro-ams/src/test/java/org/apache/amoro/server/TestAmoroManagementConfValidator.java
# amoro-ams/src/test/java/org/apache/amoro/server/process/iceberg/TestIcebergProcessFactory.java
# amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/PeriodicTableSchedulerTestBase.java
# amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestConfigurableIntervalExecutors.java
# amoro-ams/src/test/java/org/apache/amoro/server/scheduler/inline/TestPeriodicTableSchedulerCleanup.java
# amoro-common/src/main/java/org/apache/amoro/IcebergActions.java
# amoro-common/src/test/java/org/apache/amoro/process/TestLocalExecutionEngine.java
# charts/amoro/templates/amoro-configmap.yaml
# dist/src/main/amoro-bin/conf/config.yaml
# dist/src/main/amoro-bin/conf/plugins/execute-engines.yaml
# dist/src/main/amoro-bin/conf/plugins/process-factories.yaml
# docs/admin-guides/deployment.md
.github/workflows/core-hadoop3-ci.yml | 7 +-
README.md | 2 +-
amoro-ams/pom.xml | 36 ++
.../apache/amoro/server/AmoroManagementConf.java | 48 ---
.../amoro/server/AmoroManagementConfValidator.java | 8 -
.../apache/amoro/server/AmoroServiceContainer.java | 15 +-
.../amoro/server/DefaultOptimizingService.java | 28 +-
.../amoro/server/catalog/CatalogBuilder.java | 12 +-
.../amoro/server/dashboard/DashboardServer.java | 4 +-
.../dashboard/MixedAndIcebergTableDescriptor.java | 88 ++++-
.../server/dashboard/ServerTableDescriptor.java | 14 +-
.../dashboard/controller/CatalogController.java | 20 +-
.../controller/OptimizerGroupController.java | 18 +-
.../dashboard/controller/TableController.java | 25 +-
.../server/manager/AbstractOptimizerContainer.java | 5 +
.../server/manager/FlinkOptimizerContainer.java | 6 +-
.../manager/KubernetesOptimizerContainer.java | 46 ++-
.../amoro/server/optimizing/TaskRuntime.java | 98 +++++-
.../optimizing/dra/DynamicAllocationConfig.java | 362 ++++++++++++++++++++
.../persistence/mapper/TableProcessMapper.java | 2 +
.../server/process/DefaultTableProcessStore.java | 50 +--
.../amoro/server/process/TableProcessMeta.java | 4 +-
.../DanglingDeleteFilesCleaningProcess.java | 6 +-
.../process/iceberg/DataExpiringProcess.java | 6 +-
.../iceberg/HiveCommitSyncProcess.java} | 66 ++--
.../process/iceberg/IcebergProcessFactory.java | 57 +++-
.../iceberg/OrphanFilesCleaningProcess.java | 6 +-
.../process/iceberg/SnapshotsExpiringProcess.java | 10 +-
...ngProcess.java => TagsAutoCreatingProcess.java} | 18 +-
.../server/scheduler/PeriodicTableScheduler.java | 121 +------
.../scheduler/inline/InlineTableExecutors.java | 22 --
.../inline/SnapshotsExpiringExecutor.java | 86 -----
.../scheduler/inline/TagsAutoCreatingExecutor.java | 76 -----
.../amoro/server/table/DefaultTableRuntime.java | 27 --
.../amoro/server/utils/IcebergTableUtil.java | 3 +-
.../amoro/server/utils/SnowflakeIdGenerator.java | 3 +
.../apache/amoro/server/AMSServiceTestBase.java | 3 +
.../org/apache/amoro/server/AmsEnvironment.java | 1 -
.../amoro/server/TestAmoroManagementConf.java | 5 -
.../server/TestAmoroManagementConfValidator.java | 16 -
.../amoro/server/TestDefaultOptimizingService.java | 47 ++-
.../amoro/server/TestOptimizerGroupController.java | 44 +++
.../amoro/server/TestOptimizerGroupKeeper.java | 61 ++++
.../TestIcebergServerTableDescriptor.java | 117 ++++++-
.../manager/TestFlinkOptimizerContainer.java | 23 ++
.../manager/TestKubernetesOptimizerContainer.java | 277 +++++++++++++++-
.../server/optimizing/BaseOptimizingChecker.java | 6 +-
.../server/optimizing/TestOptimizingQueue.java | 115 +++++++
.../dra/TestDynamicAllocationConfig.java | 245 ++++++++++++++
.../process/iceberg/TestIcebergProcessFactory.java | 29 ++
.../inline/PeriodicTableSchedulerTestBase.java | 82 -----
.../inline/TestConfigurableIntervalExecutors.java | 34 +-
.../inline/TestPeriodicTableSchedulerCleanup.java | 250 --------------
.../inline/TestProcessDataExpiringExecutor.java | 3 +-
.../src/test/resources/config-with-units.yaml | 22 --
.../src/test/resources/config-without-units.yaml | 3 -
.../org/apache/amoro/CommonUnifiedCatalog.java | 6 +-
.../main/java/org/apache/amoro/IcebergActions.java | 3 +-
.../java/org/apache/amoro/OptimizerProperties.java | 60 +++-
.../main/java/org/apache/amoro/TableFormat.java | 1 +
.../apache/amoro/maintainer/TableMaintainer.java | 14 +-
.../apache/amoro/process/StagedTaskDescriptor.java | 4 +
.../table/descriptor/FormatTableDescriptor.java | 21 +-
.../amoro/table/descriptor/ProcessCategory.java | 55 +++
.../apache/amoro/MockAmoroManagementServer.java | 15 +
.../amoro/process/TestLocalExecutionEngine.java | 4 +
.../org/apache/amoro/utils/JacksonUtilTest.java | 6 +-
amoro-format-hudi/pom.xml | 2 -
.../amoro/formats/hudi/HudiTableDescriptor.java | 25 +-
.../iceberg/maintainer/IcebergTableMaintainer.java | 101 ++++--
.../iceberg/maintainer/MixedTableMaintainer.java | 74 ++++-
.../formats/iceberg/utils/RollingFileCleaner.java | 55 +--
.../optimizing/AbstractRewriteFilesExecutor.java | 2 +-
.../amoro/optimizing/IcebergRewriteExecutor.java | 358 ++++++++++++++++++++
.../amoro/optimizing/ParquetFileMergeRunner.java | 198 +++++++++++
.../apache/amoro/optimizing/TaskProperties.java | 7 +
.../org/apache/amoro/table/TableProperties.java | 12 +
.../org/apache/amoro/utils/IcebergThreadPools.java | 153 ++++++---
.../java/org/apache/amoro/utils/TableFileUtil.java | 6 +
.../apache/iceberg/parquet/ParquetIOBridge.java | 27 +-
.../optimizing/IcebergRewriteExecutorTest.java | 369 +++++++++++++++++++++
.../apache/amoro/utils/TestIcebergThreadPools.java | 65 ++++
.../pom.xml | 63 ++--
.../amoro/formats/lance/LanceCatalogFactory.java | 58 ++++
.../formats/lance/LanceDirectoryV1Catalog.java | 215 ++++++++++++
.../org/apache/amoro/formats/lance/LanceTable.java | 71 ++++
.../amoro/formats/lance/LanceTableDescriptor.java | 329 ++++++++++++++++++
.../services/org.apache.amoro.FormatCatalogFactory | 19 ++
...he.amoro.table.descriptor.FormatTableDescriptor | 18 +
.../amoro-mixed-flink-common/pom.xml | 10 +-
.../interceptor/KerberosInvocationHandler.java | 1 -
.../hybrid/reader/MixedFormatSourceReader.java | 38 ++-
.../org/apache/amoro/flink/table/FlinkSource.java | 35 +-
.../flink/table/KerberosAwareInputFormat.java | 165 +++++++++
.../amoro/flink/table/MixedFormatTableLoader.java | 247 +++++++++++++-
.../table/UnkeyedInputFormatOperatorFactory.java | 8 +-
.../table/UnkeyedStreamingReaderOperator.java | 175 ++++++++++
.../amoro/flink/util/FlinkClassReflectionUtil.java | 65 ----
.../apache/amoro/flink/util/IcebergClassUtil.java | 119 +++----
.../org/apache/amoro/flink/write/FlinkSink.java | 8 +-
.../apache/amoro/flink/table/TestWatermark.java | 66 +---
.../amoro/flink/util/TestIcebergClassUtil.java | 75 +++++
.../amoro/flink/write/TestAutomaticLogWriter.java | 14 +-
.../flink/write/TestMixedFormatFileWriter.java | 11 +-
amoro-format-mixed/amoro-mixed-trino/pom.xml | 1 -
amoro-format-paimon/pom.xml | 9 +
.../formats/paimon/PaimonTableDescriptor.java | 34 +-
.../common/AbstractOptimizerOperator.java | 44 ++-
.../apache/amoro/optimizer/common/Optimizer.java | 88 ++++-
.../amoro/optimizer/common/OptimizerConfig.java | 18 +
.../amoro/optimizer/common/OptimizerExecutor.java | 39 ++-
.../amoro/optimizer/common/OptimizerToucher.java | 55 ++-
.../amoro/optimizer/common/OptimizerTestBase.java | 1 +
.../common/TestAbstractOptimizerOperator.java | 57 ++++
.../amoro/optimizer/common/TestOptimizer.java | 139 ++++++++
.../optimizer/common/TestOptimizerExecutor.java | 135 ++++++++
.../optimizer/common/TestOptimizerToucher.java | 32 ++
.../amoro/optimizer/flink/FlinkExecutor.java | 94 +++++-
.../amoro/optimizer/flink/FlinkOptimizer.java | 3 +-
.../optimizer/flink/FlinkOptimizerExecutor.java | 21 ++
.../amoro/optimizer/flink/TestFlinkExecutor.java | 178 ++++++++++
.../flink/TestFlinkOptimizerExecutor.java | 66 ++++
.../amoro/optimizer/spark/SparkOptimizer.java | 26 ++
.../optimizer/standalone/StandaloneOptimizer.java | 24 ++
amoro-web/mock/modules/table.js | 33 +-
amoro-web/src/assets/icons/svg/lance.svg | 25 ++
amoro-web/src/language/en.ts | 5 +
amoro-web/src/language/zh.ts | 5 +
amoro-web/src/services/table.service.ts | 14 +-
amoro-web/src/types/common.type.ts | 1 +
amoro-web/src/views/tables/components/Cleanup.vue | 26 ++
.../src/views/tables/components/Optimizing.vue | 51 ++-
.../src/views/tables/components/Profiling.vue | 26 ++
amoro-web/src/views/tables/index.vue | 6 +
charts/amoro/templates/amoro-configmap.yaml | 13 -
charts/amoro/templates/amoro-httproute.yaml | 54 +++
charts/amoro/tests/amoro-httproute_test.yaml | 192 +++++++++++
charts/amoro/values.yaml | 48 +++
dev/deps/dependencies-hadoop-2-spark-3.3 | 13 +-
dev/deps/dependencies-hadoop-3-spark-3.5 | 13 +-
dist/src/main/amoro-bin/bin/optimizer.sh | 2 +-
dist/src/main/amoro-bin/conf/config.yaml | 13 -
.../amoro-bin/conf/plugins/execute-engines.yaml | 4 +-
.../amoro-bin/conf/plugins/process-factories.yaml | 4 +
docs/admin-guides/deployment.md | 10 +-
docs/admin-guides/managing-optimizers.md | 4 +
docs/configuration/ams-config.md | 8 -
docs/user-guides/configurations.md | 51 +--
pom.xml | 81 ++++-
149 files changed, 6515 insertions(+), 1492 deletions(-)