You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/06/30 05:04:33 UTC
(doris) branch branch-3.0 updated (fef6c7df5b8 -> 3df52cbf761)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from fef6c7df5b8 [Fix](Variant) fix variant partial update with row store enabled (#36793)
new f04c185cc6c [Refactor](meta) Delete useless classes and write methods (#36894)
new 5b68de56cd1 [refactor](nereids) New distribute planner (#36531)
new 018ff0ae94c [feature](insert)support external hive truncate table DDL (#36801)
new a89a43c2a97 [Fix](hive-writer) Fixed the issue where `uncompletedMpuPendingUploads` did not remove objects correctly. (#36905)
new 05f694a9405 [Fix](multi-catalog) Fix the transaction is not removed in abnormal situations by removing transaction in finally block. (#36705)
new 15f162f6905 [Fix](variant) ignore serialization of nothing type (#36997)
new 13e2a19bbb5 [chore](profile) rm useless code of profile (#36915)
new fee6a24ef64 [chore](upgrade) turn off fallback_to_original_planner when upgrade (#37005)
new c0591389cc7 [enhancement](cloud) batching get visible version from MetaService (#34615)
new 039d4f722e3 [fix](regression test) fix unstable single compaction test p2 (#36881)
new 1cbc322d0c5 [improvement](jdbc catalog) Modify the maximum number of connections in the connection pool to 30 by default (#36720)
new 74eaf05cb85 [fix](statistics)Escape fetch partition stats sql. (#36941)
new 05ca6c66f51 [fix](mtmv)fix mtmv dead lock (#37009)
new b2f90a1ce4f [Fix](autoinc) try fix concurrent load problem with auto inc column (#36421)
new 66ca7e94c83 [bugfix](testcase)add java error log output (#35998)
new 82e4633053f Revert "[feature](cloud) support file cache only cache index pages (#36273) (#36797)
new 9255c366ec1 (cloud-merge) Get fileCacheSize by RPC always (#36857)
new 3666c75576f [performance](load) do not copy input_block in memtable (#36939)
new 900976cf44a Revert "[enhancement](compaction) optimizing memory usage for compaction (#36492)" (#37032)
new 848fda6afd3 [fix](cgroup memory) Correct cgroup mem info cache (#36966)
new 3df52cbf761 [improvement](segmentcache) limit segment cache by memory or segment num (#37026)
The 21 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/cloud/cloud_base_compaction.cpp | 10 -
be/src/cloud/cloud_cumulative_compaction.cpp | 13 +-
be/src/common/config.cpp | 10 +-
be/src/common/config.h | 8 -
be/src/olap/base_compaction.cpp | 10 -
be/src/olap/base_tablet.h | 5 -
be/src/olap/compaction.cpp | 15 +-
be/src/olap/compaction.h | 2 -
be/src/olap/cumulative_compaction.cpp | 15 +-
be/src/olap/iterators.h | 15 +-
be/src/olap/lru_cache.cpp | 5 +
be/src/olap/lru_cache.h | 4 +-
be/src/olap/memtable.cpp | 14 +-
be/src/olap/merger.cpp | 67 +---
be/src/olap/merger.h | 6 +-
be/src/olap/rowset/rowset_meta.h | 15 -
be/src/olap/rowset/segcompaction.cpp | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 4 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 3 +-
be/src/olap/segment_loader.h | 13 +-
be/src/olap/tablet_reader.h | 2 -
be/src/runtime/exec_env_init.cpp | 8 +-
be/src/util/cgroup_util.cpp | 2 +-
be/src/util/mem_info.cpp | 8 +-
be/src/vec/core/block.cpp | 25 +-
be/src/vec/core/block.h | 5 +-
be/src/vec/data_types/data_type_object.cpp | 14 +-
be/src/vec/olap/vertical_block_reader.cpp | 18 +-
be/src/vec/olap/vertical_block_reader.h | 3 +-
be/src/vec/olap/vertical_merge_iterator.cpp | 29 +-
be/src/vec/olap/vertical_merge_iterator.h | 25 +-
be/src/vec/sink/autoinc_buffer.cpp | 150 +++++----
be/src/vec/sink/autoinc_buffer.h | 35 ++-
be/test/olap/base_compaction_test.cpp | 84 ------
be/test/olap/rowid_conversion_test.cpp | 6 +-
be/test/testutil/run_all_tests.cpp | 2 +-
be/test/vec/olap/vertical_compaction_test.cpp | 14 +-
cloud/src/meta-service/meta_service.cpp | 3 +
cloud/src/meta-service/meta_service_txn.cpp | 6 +-
.../apache/doris/jdbc/JdbcDataSourceConfig.java | 2 +-
.../java/org/apache/doris/common/io/DeepCopy.java | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../main/java/org/apache/doris/alter/Alter.java | 4 +-
.../apache/doris/analysis/TruncateTableStmt.java | 3 -
.../java/org/apache/doris/backup/BackupMeta.java | 1 +
.../doris/catalog/AutoIncrementGenerator.java | 5 +-
.../apache/doris/catalog/CatalogRecycleBin.java | 2 +
.../main/java/org/apache/doris/catalog/Column.java | 30 +-
.../java/org/apache/doris/catalog/ColumnStats.java | 26 +-
.../org/apache/doris/catalog/DistributionInfo.java | 1 +
.../main/java/org/apache/doris/catalog/Env.java | 17 +-
.../apache/doris/catalog/HashDistributionInfo.java | 1 +
.../org/apache/doris/catalog/JdbcResource.java | 2 +-
.../apache/doris/catalog/ListPartitionInfo.java | 1 +
.../apache/doris/catalog/MaterializedIndex.java | 11 +-
.../java/org/apache/doris/catalog/OlapTable.java | 18 +-
.../java/org/apache/doris/catalog/Partition.java | 9 +
.../org/apache/doris/catalog/PartitionInfo.java | 20 +-
.../doris/catalog/RandomDistributionInfo.java | 1 +
.../apache/doris/catalog/RangePartitionInfo.java | 1 +
.../java/org/apache/doris/catalog/Replica.java | 22 +-
.../apache/doris/catalog/SinglePartitionInfo.java | 1 +
.../main/java/org/apache/doris/catalog/Tablet.java | 12 +-
.../org/apache/doris/catalog/TempPartitions.java | 24 +-
.../apache/doris/cloud/CacheHotspotManager.java | 39 ++-
.../apache/doris/cloud/catalog/CloudPartition.java | 33 +-
.../apache/doris/cloud/catalog/CloudReplica.java | 17 +-
.../cloud/datasource/CloudInternalCatalog.java | 4 -
.../transaction/CloudGlobalTransactionMgr.java | 4 +-
.../src/main/java/org/apache/doris/common/Id.java | 7 +-
.../doris/common/NereidsSqlCacheManager.java | 9 +-
.../java/org/apache/doris/common/TreeNode.java | 25 ++
.../doris/common/profile/ExecutionProfile.java | 23 --
.../org/apache/doris/common/profile/Profile.java | 10 +
.../doris/common/profile/SummaryProfile.java | 13 +
.../org/apache/doris/datasource/CatalogIf.java | 3 +
.../apache/doris/datasource/ExternalCatalog.java | 23 ++
.../doris/datasource/hive/HMSCachedClient.java | 2 +
.../doris/datasource/hive/HMSTransaction.java | 18 ++
.../doris/datasource/hive/HiveMetadataOps.java | 16 +
.../hive/PostgreSQLJdbcHMSCachedClient.java | 4 +
.../datasource/hive/ThriftHMSCachedClient.java | 19 +-
.../datasource/iceberg/IcebergMetadataOps.java | 5 +
.../datasource/operations/ExternalMetadataOps.java | 8 +
.../org/apache/doris/nereids/NereidsPlanner.java | 60 +++-
.../org/apache/doris/nereids/SqlCacheContext.java | 1 -
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +
.../properties/ChildrenPropertiesRegulator.java | 52 ++--
.../functions/table/TableValuedFunction.java | 4 +
.../doris/nereids/trees/plans/PlaceholderId.java | 7 +-
.../trees/plans/commands/ExplainCommand.java | 1 +
.../plans/distribute/BucketSpecifyInstances.java} | 27 +-
.../plans/distribute/DefaultSpecifyInstances.java} | 27 +-
.../trees/plans/distribute/DistributePlanner.java | 66 ++++
.../trees/plans/distribute/DistributedPlan.java | 57 ++++
.../trees/plans/distribute/FragmentIdMapping.java | 71 +++++
.../plans/distribute/NereidsSpecifyInstances.java | 59 ++++
.../plans/distribute/PipelineDistributedPlan.java | 72 +++++
.../BackendDistributedPlanWorkerManager.java | 63 ++++
.../plans/distribute/worker/BackendWorker.java | 74 +++++
.../distribute/worker/DistributedPlanWorker.java} | 33 +-
.../worker/DistributedPlanWorkerManager.java} | 26 +-
.../worker/LoadBalanceScanWorkerSelector.java | 336 +++++++++++++++++++++
.../distribute/worker/ScanWorkerSelector.java | 62 ++++
.../plans/distribute/worker/WorkerScanRanges.java} | 28 +-
.../trees/plans/distribute/worker/Workload.java} | 25 +-
.../worker/job/AbstractUnassignedJob.java | 74 +++++
.../worker/job/AbstractUnassignedScanJob.java | 202 +++++++++++++
.../plans/distribute/worker/job/AssignedJob.java} | 34 +--
.../distribute/worker/job/AssignedJobBuilder.java | 64 ++++
.../distribute/worker/job/BucketScanSource.java | 148 +++++++++
.../worker/job/CustomAssignmentJob.java} | 26 +-
.../distribute/worker/job/DefaultScanSource.java | 111 +++++++
.../worker/job/LocalShuffleAssignedJob.java | 43 +++
.../plans/distribute/worker/job/ScanRange.java} | 24 +-
.../plans/distribute/worker/job/ScanRanges.java | 109 +++++++
.../plans/distribute/worker/job/ScanSource.java} | 34 +--
.../plans/distribute/worker/job/Splittable.java | 69 +++++
.../distribute/worker/job/StaticAssignedJob.java | 106 +++++++
.../UnassignedGatherScanMultiRemoteTablesJob.java | 87 ++++++
.../plans/distribute/worker/job/UnassignedJob.java | 52 ++++
.../worker/job/UnassignedJobBuilder.java | 266 ++++++++++++++++
.../worker/job/UnassignedQueryConstantJob.java | 50 +++
.../job/UnassignedScanBucketOlapTableJob.java | 310 +++++++++++++++++++
.../job/UnassignedScanSingleOlapTableJob.java | 82 +++++
.../job/UnassignedScanSingleRemoteTableJob.java | 53 ++++
.../worker/job/UnassignedShuffleJob.java | 124 ++++++++
.../worker/job/UnassignedSpecifyInstancesJob.java | 49 +++
.../worker/job/UninstancedScanSource.java} | 29 +-
.../distribute/worker/job/WorkerScanSource.java} | 28 +-
.../org/apache/doris/nereids/util/JoinUtils.java | 22 +-
.../java/org/apache/doris/nereids/util/Utils.java | 14 +
.../apache/doris/persist/BackendIdsUpdateInfo.java | 63 ----
.../java/org/apache/doris/persist/ClusterInfo.java | 114 -------
.../org/apache/doris/planner/DataGenScanNode.java | 4 +
.../org/apache/doris/planner/OlapScanNode.java | 37 ++-
.../org/apache/doris/planner/PlanFragment.java | 25 ++
.../java/org/apache/doris/planner/PlanNode.java | 25 ++
.../java/org/apache/doris/planner/ScanNode.java | 9 +-
.../java/org/apache/doris/qe/ConnectContext.java | 6 +
.../main/java/org/apache/doris/qe/Coordinator.java | 51 ++--
.../org/apache/doris/qe/NereidsCoordinator.java | 194 ++++++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 54 +++-
.../java/org/apache/doris/qe/StmtExecutor.java | 12 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 11 +-
.../apache/doris/service/FrontendServiceImpl.java | 10 +
.../apache/doris/statistics/AnalysisManager.java | 2 +-
.../PartitionColumnStatisticCacheLoader.java | 2 +-
.../doris/statistics/StatisticsRepository.java | 6 +-
.../main/java/org/apache/doris/system/Backend.java | 10 -
.../doris/transaction/HiveTransactionManager.java | 7 +-
.../transaction/IcebergTransactionManager.java | 7 +-
.../apache/doris/transaction/TabletCommitInfo.java | 11 +-
.../org/apache/doris/catalog/ColumnStatTest.java | 21 +-
.../java/org/apache/doris/catalog/ColumnTest.java | 18 +-
.../org/apache/doris/catalog/JdbcResourceTest.java | 4 +-
.../doris/catalog/MaterializedIndexTest.java | 6 +-
.../doris/catalog/RangePartitionInfoTest.java | 6 +-
.../java/org/apache/doris/catalog/ReplicaTest.java | 12 +-
.../java/org/apache/doris/catalog/TabletTest.java | 6 +-
.../apache/doris/catalog/TempPartitionTest.java | 4 +-
.../doris/datasource/TestHMSCachedClient.java | 3 +
.../org/apache/doris/qe/HmsQueryCacheTest.java | 14 +-
gensrc/proto/cloud.proto | 3 +
gensrc/thrift/FrontendService.thrift | 1 +
.../unique/test_unique_auto_inc_concurrent.out | 10 +
.../hive/ddl/test_hive_truncate_table.out | 24 ++
.../distribute/colocate_union_numbers.out | 10 +
.../nereids_syntax_p0/distribute/local_shuffle.out | 36 +++
.../prune_bucket_with_bucket_shuffle_join.out | 5 +
.../distribute/query_constant.out | 12 +
.../data/nereids_syntax_p0/distribute/shuffle.out | 8 +
.../distribute/shuffle_left_join.out | 9 +
regression-test/data/variant_p0/load.out | 12 +-
.../data/variant_p0/test_sub_path_pruning.out | 10 +-
.../compaction_width_array_column.groovy | 137 ---------
.../test_single_compaction_fault_injection.groovy | 68 +++--
.../test_single_replica_compaction.groovy | 72 +++--
.../spark_connector/spark_connector.groovy | 10 +-
.../spark_connector/spark_connector_arrow.groovy | 10 +-
.../spark_connector_read_type.groovy | 10 +-
.../unique/test_unique_auto_inc_concurrent.groovy | 59 ++++
.../hive/ddl/test_hive_truncate_table.groovy | 100 ++++++
.../test_forbid_unknown_col_stats.groovy | 2 +-
.../distribute/colocate_union_numbers.groovy | 70 +++++
.../distribute/local_shuffle.groovy | 187 ++++++++++++
.../prune_bucket_with_bucket_shuffle_join.groovy | 98 ++++++
.../distribute/query_constant.groovy | 38 ++-
.../nereids_syntax_p0/distribute/shuffle.groovy | 35 +--
.../distribute/shuffle_left_join.groovy | 117 +++++++
.../suites/statistics/test_partition_stats.groovy | 27 ++
regression-test/suites/variant_p0/load.groovy | 8 +-
.../suites/variant_p0/test_sub_path_pruning.groovy | 20 +-
193 files changed, 5016 insertions(+), 1483 deletions(-)
delete mode 100644 be/test/olap/base_compaction_test.cpp
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/BucketSpecifyInstances.java} (51%)
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/DefaultSpecifyInstances.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/NereidsSpecifyInstances.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/DistributedPlanWorker.java} (52%)
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/WorkerScanRanges.java} (52%)
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/Workload.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/AssignedJob.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/CustomAssignmentJob.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/ScanRange.java} (51%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRanges.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/ScanSource.java} (52%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/Splittable.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/UninstancedScanSource.java} (52%)
copy fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java => nereids/trees/plans/distribute/worker/job/WorkerScanSource.java} (51%)
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/BackendIdsUpdateInfo.java
delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/ClusterInfo.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
create mode 100644 regression-test/data/data_model_p0/unique/test_unique_auto_inc_concurrent.out
create mode 100644 regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/colocate_union_numbers.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/local_shuffle.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/query_constant.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/shuffle.out
create mode 100644 regression-test/data/nereids_syntax_p0/distribute/shuffle_left_join.out
delete mode 100644 regression-test/suites/compaction/compaction_width_array_column.groovy
create mode 100644 regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
create mode 100644 regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
create mode 100644 regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
create mode 100644 regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy
create mode 100644 regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
copy fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java => regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy (51%)
copy fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java => regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy (51%)
create mode 100644 regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 16/21: Revert "[feature](cloud) support file cache only cache index pages (#36273) (#36797)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 82e4633053f6e7d1789069685b6031c93d28927c
Author: zhengyu <fr...@gmail.com>
AuthorDate: Sat Jun 29 02:13:28 2024 +0800
Revert "[feature](cloud) support file cache only cache index pages (#36273) (#36797)
This reverts commit f5c40a5c3bac3091707c8324c422889fa7f030c8.
This is a experimental (and buggy) commit. I hava found that caching
index only helps little. With that be the result, I think it is a good
time to
revert it.
---
be/src/common/config.cpp | 2 --
be/src/common/config.h | 2 --
be/src/olap/rowset/segment_v2/column_reader.cpp | 4 +---
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp | 3 +--
4 files changed, 2 insertions(+), 9 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 91707b5a8a2..3e948f4cca2 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1003,8 +1003,6 @@ DEFINE_Bool(enable_file_cache, "false");
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
DEFINE_String(file_cache_path, "");
DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB
-// only cache index pages (prerequisite: enable_file_cache = true)
-DEFINE_Bool(file_cache_index_only, "false");
DEFINE_Bool(clear_file_cache, "false");
DEFINE_Bool(enable_file_cache_query_limit, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index f35aeb61747..6f0065e2fe3 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1047,8 +1047,6 @@ DECLARE_Bool(enable_file_cache);
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,"normal_percent":85, "disposable_percent":10, "index_percent":5}]
DECLARE_String(file_cache_path);
DECLARE_Int64(file_cache_each_block_size);
-// only cache index pages (prerequisite: enable_file_cache = true)
-DECLARE_Bool(file_cache_index_only);
DECLARE_Bool(clear_file_cache);
DECLARE_Bool(enable_file_cache_query_limit);
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 7396bce1a55..392917e0d83 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -342,11 +342,9 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag
PageHandle* handle, Slice* page_body, PageFooterPB* footer,
BlockCompressionCodec* codec) const {
iter_opts.sanity_check();
- bool use_page_cache = iter_opts.use_page_cache &&
- (!config::file_cache_index_only || iter_opts.type == INDEX_PAGE);
PageReadOptions opts {
.verify_checksum = _opts.verify_checksum,
- .use_page_cache = use_page_cache,
+ .use_page_cache = iter_opts.use_page_cache,
.kept_in_memory = _opts.kept_in_memory,
.type = iter_opts.type,
.file_reader = iter_opts.file_reader,
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 94429920877..59251b5595d 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -115,9 +115,8 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle,
PageFooterPB* footer, PageTypePB type,
BlockCompressionCodec* codec, bool pre_decode) const {
OlapReaderStatistics tmp_stats;
- bool use_page_cache = _use_page_cache && (!config::file_cache_index_only || type == INDEX_PAGE);
PageReadOptions opts {
- .use_page_cache = use_page_cache,
+ .use_page_cache = _use_page_cache,
.kept_in_memory = _kept_in_memory,
.pre_decode = pre_decode,
.type = type,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 20/21: [fix](cgroup memory) Correct cgroup mem info cache (#36966)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 848fda6afd31e94f906ddba90625fc1d6e391cf2
Author: Hongkun Xu <xu...@163.com>
AuthorDate: Sun Jun 30 12:56:14 2024 +0800
[fix](cgroup memory) Correct cgroup mem info cache (#36966)
## Proposed changes
After upgrading to Doris 2.1.3, we noticed that the "sys available
memory" in be.INFO continuously decreases until it falls below the
warning water mark, leading to persistent garbage collection (GC)
despite the actual memory usage being very low. And The cache in cgroup
mem info is always 0. Consequently, I identified an error in the
calculation of available memory in cgroup memory:
1. The memory information for cgroup memory is stored in the file
"memory.stat" rather than "memory.meminfo" (in fact, the
"memory.meminfo" file does not exist). You can see the files under the
cgroup path in the attached screenshot1.
2. The output content of "memory.stat" is shown in the screenshot1
below.
<img width="1720" alt="image"
src="https://github.com/apache/doris/assets/38196564/e654322e-9bf4-4f5e-951f-99e101ebbf47">
<img width="1364" alt="image"
src="https://github.com/apache/doris/assets/38196564/02cf8899-7618-4d5f-bf59-68fa0c90ebf2">
<!--Describe your changes.-->
My change is about two steps:
1. Modified the file name for mem info in cgroup.
2. Modified the process for extracting the cache from cgroup.
Co-authored-by: Xinyi Zou <zo...@gmail.com>
---
be/src/util/cgroup_util.cpp | 2 +-
be/src/util/mem_info.cpp | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/be/src/util/cgroup_util.cpp b/be/src/util/cgroup_util.cpp
index a2c3e294e66..9ad78696a6f 100644
--- a/be/src/util/cgroup_util.cpp
+++ b/be/src/util/cgroup_util.cpp
@@ -184,7 +184,7 @@ Status CGroupUtil::find_cgroup_mem_info(std::string* file_path) {
}
string cgroup_path;
RETURN_IF_ERROR(find_abs_cgroup_path("memory", &cgroup_path));
- *file_path = cgroup_path + "/memory.meminfo";
+ *file_path = cgroup_path + "/memory.stat";
return Status::OK();
}
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index fc5d5512f1a..45e609d7100 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -154,7 +154,7 @@ void MemInfo::refresh_proc_meminfo() {
if (fields.size() < 2) {
continue;
}
- std::string key = fields[0].substr(0, fields[0].size() - 1);
+ std::string key = fields[0].substr(0, fields[0].size());
StringParser::ParseResult result;
auto mem_value = StringParser::string_to_int<int64_t>(fields[1].data(),
@@ -180,19 +180,19 @@ void MemInfo::refresh_proc_meminfo() {
// https://serverfault.com/questions/902009/the-memory-usage-reported-in-cgroup-differs-from-the-free-command
// memory.usage_in_bytes ~= free.used + free.(buff/cache) - (buff)
// so, memory.usage_in_bytes - memory.meminfo["Cached"]
- _s_cgroup_mem_usage = cgroup_mem_usage - _s_cgroup_mem_info_bytes["Cached"];
+ _s_cgroup_mem_usage = cgroup_mem_usage - _s_cgroup_mem_info_bytes["cache"];
// wait 10s, 100 * 100ms, avoid too frequently.
_s_cgroup_mem_refresh_wait_times = -100;
LOG(INFO) << "Refresh cgroup memory win, refresh again after 10s, cgroup mem limit: "
<< _s_cgroup_mem_limit << ", cgroup mem usage: " << _s_cgroup_mem_usage
- << ", cgroup mem info cached: " << _s_cgroup_mem_info_bytes["Cached"];
+ << ", cgroup mem info cached: " << _s_cgroup_mem_info_bytes["cache"];
} else {
// find cgroup failed, wait 300s, 1000 * 100ms.
_s_cgroup_mem_refresh_wait_times = -3000;
LOG(INFO)
<< "Refresh cgroup memory failed, refresh again after 300s, cgroup mem limit: "
<< _s_cgroup_mem_limit << ", cgroup mem usage: " << _s_cgroup_mem_usage
- << ", cgroup mem info cached: " << _s_cgroup_mem_info_bytes["Cached"];
+ << ", cgroup mem info cached: " << _s_cgroup_mem_info_bytes["cache"];
}
} else {
if (config::enable_use_cgroup_memory_info) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 18/21: [performance](load) do not copy input_block in memtable (#36939)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3666c75576f75ebfc86117beb2ccc24359dff743
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Sat Jun 29 16:06:35 2024 +0800
[performance](load) do not copy input_block in memtable (#36939)
Pass `column_offset` to `Block::clone_without_columns()` and
`MutableBlock::add_rows()`,
so we do not need to copy `input_block` in `MemTable::insert()`.
To optimize performance in cases with many (e.g. 12k+) tablets:
Before:
```
- DeltaWriterWriteTime: 1m58s
- MemTableWriterFlushTime: 0ns
- MemTableWriterLockTime: 1s41ms
- MemTableWriterShrinkTime: 0ns
- MemTableWriterWriteTime: 1m53s
- MemTableCopyCount: 12.044003M (12044003)
- MemTableCopyTime: 42s857ms
- MemTableCopyTime0: 11s866ms
- MemTableInitTime: 13.384ms
- MemTableInsertTime: 51s543ms
```
After:
```
- DeltaWriterWriteTime: 1m
- MemTableWriterFlushTime: 0ns
- MemTableWriterLockTime: 971.481ms
- MemTableWriterShrinkTime: 0ns
- MemTableWriterWriteTime: 55s274ms
- MemTableCopyCount: 0
- MemTableCopyTime: 0ns
- MemTableCopyTime0: 0ns
- MemTableInitTime: 18.746ms
- MemTableInsertTime: 53s772ms
```
---
be/src/olap/memtable.cpp | 14 ++++++--------
be/src/vec/core/block.cpp | 25 ++++++++++++++++++-------
be/src/vec/core/block.h | 5 +++--
3 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 750026c289d..e55fd678bd2 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -180,16 +180,14 @@ int RowInBlockComparator::operator()(const RowInBlock* left, const RowInBlock* r
Status MemTable::insert(const vectorized::Block* input_block,
const std::vector<uint32_t>& row_idxs) {
- vectorized::Block target_block = *input_block;
- target_block = input_block->copy_block(_column_offset);
if (_is_first_insertion) {
_is_first_insertion = false;
- auto cloneBlock = target_block.clone_without_columns();
+ auto cloneBlock = input_block->clone_without_columns(&_column_offset);
_input_mutable_block = vectorized::MutableBlock::build_mutable_block(&cloneBlock);
_vec_row_comparator->set_block(&_input_mutable_block);
_output_mutable_block = vectorized::MutableBlock::build_mutable_block(&cloneBlock);
if (_keys_type != KeysType::DUP_KEYS) {
- _init_agg_functions(&target_block);
+ _init_agg_functions(input_block);
}
if (_tablet_schema->has_sequence_col()) {
if (_is_partial_update) {
@@ -210,11 +208,11 @@ Status MemTable::insert(const vectorized::Block* input_block,
auto num_rows = row_idxs.size();
size_t cursor_in_mutableblock = _input_mutable_block.rows();
auto block_size0 = _input_mutable_block.allocated_bytes();
- RETURN_IF_ERROR(_input_mutable_block.add_rows(&target_block, row_idxs.data(),
- row_idxs.data() + num_rows));
+ RETURN_IF_ERROR(_input_mutable_block.add_rows(input_block, row_idxs.data(),
+ row_idxs.data() + num_rows, &_column_offset));
auto block_size1 = _input_mutable_block.allocated_bytes();
g_memtable_input_block_allocated_size << block_size1 - block_size0;
- auto input_size = size_t(target_block.bytes() * num_rows / target_block.rows() *
+ auto input_size = size_t(input_block->bytes() * num_rows / input_block->rows() *
config::memtable_insert_memory_ratio);
_mem_usage += input_size;
_insert_mem_tracker->consume(input_size);
@@ -348,7 +346,7 @@ Status MemTable::_sort_by_cluster_keys() {
row_pos_vec.emplace_back(row_in_blocks[i]->_row_pos);
}
return _output_mutable_block.add_rows(&in_block, row_pos_vec.data(),
- row_pos_vec.data() + in_block.rows());
+ row_pos_vec.data() + in_block.rows(), &_column_offset);
}
void MemTable::_sort_one_column(std::vector<RowInBlock*>& row_in_blocks, Tie& tie,
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index 676674e8ec0..22062dc5310 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -653,12 +653,19 @@ Block Block::clone_with_columns(const Columns& columns) const {
return res;
}
-Block Block::clone_without_columns() const {
+Block Block::clone_without_columns(const std::vector<int>* column_offset) const {
Block res;
- size_t num_columns = data.size();
- for (size_t i = 0; i < num_columns; ++i) {
- res.insert({nullptr, data[i].type, data[i].name});
+ if (column_offset != nullptr) {
+ size_t num_columns = column_offset->size();
+ for (size_t i = 0; i < num_columns; ++i) {
+ res.insert({nullptr, data[(*column_offset)[i]].type, data[(*column_offset)[i]].name});
+ }
+ } else {
+ size_t num_columns = data.size();
+ for (size_t i = 0; i < num_columns; ++i) {
+ res.insert({nullptr, data[i].type, data[i].name});
+ }
}
return res;
}
@@ -1008,14 +1015,18 @@ void MutableBlock::add_row(const Block* block, int row) {
}
Status MutableBlock::add_rows(const Block* block, const uint32_t* row_begin,
- const uint32_t* row_end) {
+ const uint32_t* row_end, const std::vector<int>* column_offset) {
RETURN_IF_CATCH_EXCEPTION({
DCHECK_LE(columns(), block->columns());
+ if (column_offset != nullptr) {
+ DCHECK_EQ(columns(), column_offset->size());
+ }
const auto& block_data = block->get_columns_with_type_and_name();
for (size_t i = 0; i < _columns.size(); ++i) {
- DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name());
+ const auto& src_col = column_offset ? block_data[(*column_offset)[i]] : block_data[i];
+ DCHECK_EQ(_data_types[i]->get_name(), src_col.type->get_name());
auto& dst = _columns[i];
- const auto& src = *block_data[i].column.get();
+ const auto& src = *src_col.column.get();
DCHECK_GE(src.size(), row_end - row_begin);
dst->insert_indices_from(src, row_begin, row_end);
}
diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h
index 3611252ea59..6f50ff0035a 100644
--- a/be/src/vec/core/block.h
+++ b/be/src/vec/core/block.h
@@ -223,7 +223,7 @@ public:
void set_columns(const Columns& columns);
Block clone_with_columns(const Columns& columns) const;
- Block clone_without_columns() const;
+ Block clone_without_columns(const std::vector<int>* column_offset = nullptr) const;
/** Get empty columns with the same types as in block. */
MutableColumns clone_empty_columns() const;
@@ -607,7 +607,8 @@ public:
void add_row(const Block* block, int row);
// Batch add row should return error status if allocate memory failed.
- Status add_rows(const Block* block, const uint32_t* row_begin, const uint32_t* row_end);
+ Status add_rows(const Block* block, const uint32_t* row_begin, const uint32_t* row_end,
+ const std::vector<int>* column_offset = nullptr);
Status add_rows(const Block* block, size_t row_begin, size_t length);
Status add_rows(const Block* block, std::vector<int64_t> rows);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 19/21: Revert "[enhancement](compaction) optimizing memory usage for compaction (#36492)" (#37032)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 900976cf44a88ef4447cba19b776aceba1c5010e
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Sun Jun 30 11:08:22 2024 +0800
Revert "[enhancement](compaction) optimizing memory usage for compaction (#36492)" (#37032)
This reverts commit 99901814d8b90887f54b1768b98b4f0b78fab376.
---
be/src/cloud/cloud_base_compaction.cpp | 10 --
be/src/cloud/cloud_cumulative_compaction.cpp | 13 +-
be/src/common/config.cpp | 6 -
be/src/common/config.h | 6 -
be/src/olap/base_compaction.cpp | 10 --
be/src/olap/base_tablet.h | 5 -
be/src/olap/compaction.cpp | 15 +--
be/src/olap/compaction.h | 2 -
be/src/olap/cumulative_compaction.cpp | 15 +--
be/src/olap/iterators.h | 15 +--
be/src/olap/merger.cpp | 67 +---------
be/src/olap/merger.h | 6 +-
be/src/olap/rowset/rowset_meta.h | 15 ---
be/src/olap/rowset/segcompaction.cpp | 2 +-
be/src/olap/tablet_reader.h | 2 -
be/src/vec/olap/vertical_block_reader.cpp | 18 +--
be/src/vec/olap/vertical_block_reader.h | 3 +-
be/src/vec/olap/vertical_merge_iterator.cpp | 29 ++---
be/src/vec/olap/vertical_merge_iterator.h | 25 +---
be/test/olap/base_compaction_test.cpp | 84 -------------
be/test/olap/rowid_conversion_test.cpp | 6 +-
be/test/vec/olap/vertical_compaction_test.cpp | 14 +--
.../compaction_width_array_column.groovy | 137 ---------------------
23 files changed, 42 insertions(+), 463 deletions(-)
diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp
index 4ceab8eb6e3..d4a86743a48 100644
--- a/be/src/cloud/cloud_base_compaction.cpp
+++ b/be/src/cloud/cloud_base_compaction.cpp
@@ -163,16 +163,6 @@ Status CloudBaseCompaction::pick_rowsets_to_compact() {
return Status::Error<BE_NO_SUITABLE_VERSION>("no suitable versions for compaction");
}
- int score = 0;
- int rowset_cnt = 0;
- while (rowset_cnt < _input_rowsets.size()) {
- score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score();
- if (score > config::base_compaction_max_compaction_score) {
- break;
- }
- }
- _input_rowsets.resize(rowset_cnt);
-
// 1. cumulative rowset must reach base_compaction_min_rowset_num threshold
if (_input_rowsets.size() > config::base_compaction_min_rowset_num) {
VLOG_NOTICE << "satisfy the base compaction policy. tablet=" << _tablet->tablet_id()
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp
index 2a26b1b294b..de318f979a5 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -354,20 +354,11 @@ Status CloudCumulativeCompaction::pick_rowsets_to_compact() {
return st;
}
- int64_t max_score = config::cumulative_compaction_max_deltas;
- auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage();
- bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 0.8;
- if (cloud_tablet()->last_compaction_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>() ||
- memory_usage_high) {
- max_score = std::max(config::cumulative_compaction_max_deltas /
- config::cumulative_compaction_max_deltas_factor,
- config::cumulative_compaction_min_deltas + 1);
- }
-
size_t compaction_score = 0;
auto compaction_policy = cloud_tablet()->tablet_meta()->compaction_policy();
_engine.cumu_compaction_policy(compaction_policy)
- ->pick_input_rowsets(cloud_tablet(), candidate_rowsets, max_score,
+ ->pick_input_rowsets(cloud_tablet(), candidate_rowsets,
+ config::cumulative_compaction_max_deltas,
config::cumulative_compaction_min_deltas, &_input_rowsets,
&_last_delete_version, &compaction_score);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 3e948f4cca2..580793d36ab 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -385,7 +385,6 @@ DEFINE_mInt32(max_single_replica_compaction_threads, "-1");
DEFINE_Bool(enable_base_compaction_idle_sched, "true");
DEFINE_mInt64(base_compaction_min_rowset_num, "5");
-DEFINE_mInt64(base_compaction_max_compaction_score, "20");
DEFINE_mDouble(base_compaction_min_data_ratio, "0.3");
DEFINE_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
@@ -416,7 +415,6 @@ DEFINE_mInt64(compaction_min_size_mbytes, "64");
// cumulative compaction policy: min and max delta file's number
DEFINE_mInt64(cumulative_compaction_min_deltas, "5");
DEFINE_mInt64(cumulative_compaction_max_deltas, "1000");
-DEFINE_mInt32(cumulative_compaction_max_deltas_factor, "10");
// This config can be set to limit thread number in multiget thread pool.
DEFINE_mInt32(multi_get_max_threads, "10");
@@ -1315,10 +1313,6 @@ DEFINE_Bool(enable_file_logger, "true");
// The minimum row group size when exporting Parquet files. default 128MB
DEFINE_Int64(min_row_group_size, "134217728");
-DEFINE_mInt64(compaction_memory_bytes_limit, "1073741824");
-
-DEFINE_mInt64(compaction_batch_size, "-1");
-
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 6f0065e2fe3..9920b65fe52 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -438,7 +438,6 @@ DECLARE_mInt32(max_single_replica_compaction_threads);
DECLARE_Bool(enable_base_compaction_idle_sched);
DECLARE_mInt64(base_compaction_min_rowset_num);
-DECLARE_mInt64(base_compaction_max_compaction_score);
DECLARE_mDouble(base_compaction_min_data_ratio);
DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes);
@@ -469,7 +468,6 @@ DECLARE_mInt64(compaction_min_size_mbytes);
// cumulative compaction policy: min and max delta file's number
DECLARE_mInt64(cumulative_compaction_min_deltas);
DECLARE_mInt64(cumulative_compaction_max_deltas);
-DECLARE_mInt32(cumulative_compaction_max_deltas_factor);
// This config can be set to limit thread number in multiget thread pool.
DECLARE_mInt32(multi_get_max_threads);
@@ -1401,10 +1399,6 @@ DECLARE_Bool(enable_file_logger);
// The minimum row group size when exporting Parquet files.
DECLARE_Int64(min_row_group_size);
-DECLARE_mInt64(compaction_memory_bytes_limit);
-
-DECLARE_mInt64(compaction_batch_size);
-
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index 8be29383c1e..436180c78ca 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -151,16 +151,6 @@ Status BaseCompaction::pick_rowsets_to_compact() {
"situation, no need to do base compaction.");
}
- int score = 0;
- int rowset_cnt = 0;
- while (rowset_cnt < _input_rowsets.size()) {
- score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score();
- if (score > config::base_compaction_max_compaction_score) {
- break;
- }
- }
- _input_rowsets.resize(rowset_cnt);
-
// 1. cumulative rowset must reach base_compaction_num_cumulative_deltas threshold
if (_input_rowsets.size() > config::base_compaction_min_rowset_num) {
VLOG_NOTICE << "satisfy the base compaction policy. tablet=" << _tablet->tablet_id()
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 4852a6cba9b..dc5f488e044 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -22,7 +22,6 @@
#include <string>
#include "common/status.h"
-#include "olap/iterators.h"
#include "olap/partial_update_info.h"
#include "olap/rowset/segment_v2/segment.h"
#include "olap/tablet_fwd.h"
@@ -300,10 +299,6 @@ public:
std::atomic<int64_t> read_block_count = 0;
std::atomic<int64_t> write_count = 0;
std::atomic<int64_t> compaction_count = 0;
-
- std::mutex sample_info_lock;
- std::vector<CompactionSampleInfo> sample_infos;
- Status last_compaction_status = Status::OK();
};
} /* namespace doris */
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index b42c23f1874..37dcac5283e 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -149,15 +149,6 @@ void Compaction::init_profile(const std::string& label) {
_merge_rowsets_latency_timer = ADD_TIMER(_profile, "merge_rowsets_latency");
}
-int64_t Compaction::merge_way_num() {
- int64_t way_num = 0;
- for (auto&& rowset : _input_rowsets) {
- way_num += rowset->rowset_meta()->get_merge_way_num();
- }
-
- return way_num;
-}
-
Status Compaction::merge_input_rowsets() {
std::vector<RowsetReaderSharedPtr> input_rs_readers;
input_rs_readers.reserve(_input_rowsets.size());
@@ -179,23 +170,19 @@ Status Compaction::merge_input_rowsets() {
_stats.rowid_conversion = &_rowid_conversion;
}
- int64_t way_num = merge_way_num();
-
Status res;
{
SCOPED_TIMER(_merge_rowsets_latency_timer);
if (_is_vertical) {
res = Merger::vertical_merge_rowsets(_tablet, compaction_type(), *_cur_tablet_schema,
input_rs_readers, _output_rs_writer.get(),
- get_avg_segment_rows(), way_num, &_stats);
+ get_avg_segment_rows(), &_stats);
} else {
res = Merger::vmerge_rowsets(_tablet, compaction_type(), *_cur_tablet_schema,
input_rs_readers, _output_rs_writer.get(), &_stats);
}
}
- _tablet->last_compaction_status = res;
-
if (!res.ok()) {
LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
<< ", tablet=" << _tablet->tablet_id()
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index 8e0c1099a20..9ec1297c69c 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -81,8 +81,6 @@ protected:
void _load_segment_to_cache();
- int64_t merge_way_num();
-
// the root tracker for this compaction
std::shared_ptr<MemTrackerLimiter> _mem_tracker;
diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
index 2c7e654787a..1e0f338da23 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -134,20 +134,11 @@ Status CumulativeCompaction::pick_rowsets_to_compact() {
<< ", tablet=" << _tablet->tablet_id();
}
- int64_t max_score = config::cumulative_compaction_max_deltas;
- auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage();
- bool memory_usage_high = process_memory_usage > MemInfo::soft_mem_limit() * 0.8;
- if (tablet()->last_compaction_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>() || memory_usage_high) {
- max_score = std::max(config::cumulative_compaction_max_deltas /
- config::cumulative_compaction_max_deltas_factor,
- config::cumulative_compaction_min_deltas + 1);
- }
-
size_t compaction_score = 0;
tablet()->cumulative_compaction_policy()->pick_input_rowsets(
- tablet(), candidate_rowsets, max_score, config::cumulative_compaction_min_deltas,
- &_input_rowsets, &_last_delete_version, &compaction_score,
- _allow_delete_in_cumu_compaction);
+ tablet(), candidate_rowsets, config::cumulative_compaction_max_deltas,
+ config::cumulative_compaction_min_deltas, &_input_rowsets, &_last_delete_version,
+ &compaction_score, _allow_delete_in_cumu_compaction);
// Cumulative compaction will process with at least 1 rowset.
// So when there is no rowset being chosen, we should return Status::Error<CUMULATIVE_NO_SUITABLE_VERSION>():
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index cbf8f1eca65..330aa9e3475 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -17,7 +17,6 @@
#pragma once
-#include <cstddef>
#include <memory>
#include "common/status.h"
@@ -122,12 +121,6 @@ public:
size_t topn_limit = 0;
};
-struct CompactionSampleInfo {
- int64_t bytes = 0;
- int64_t rows = 0;
- int64_t group_data_size;
-};
-
class RowwiseIterator;
using RowwiseIteratorUPtr = std::unique_ptr<RowwiseIterator>;
class RowwiseIterator {
@@ -140,13 +133,7 @@ public:
// Input options may contain scan range in which this scan.
// Return Status::OK() if init successfully,
// Return other error otherwise
- virtual Status init(const StorageReadOptions& opts) {
- return Status::NotSupported("to be implemented");
- }
-
- virtual Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) {
- return Status::NotSupported("to be implemented");
- }
+ virtual Status init(const StorageReadOptions& opts) = 0;
// If there is any valid data, this function will load data
// into input batch with Status::OK() returned
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index 4c620d30252..cecbeb163dd 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -24,7 +24,6 @@
#include <algorithm>
#include <iterator>
#include <memory>
-#include <mutex>
#include <numeric>
#include <ostream>
#include <shared_mutex>
@@ -34,9 +33,7 @@
#include "common/config.h"
#include "common/logging.h"
-#include "common/status.h"
#include "olap/base_tablet.h"
-#include "olap/iterators.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowid_conversion.h"
@@ -46,7 +43,6 @@
#include "olap/rowset/segment_v2/segment_writer.h"
#include "olap/storage_engine.h"
#include "olap/tablet.h"
-#include "olap/tablet_fwd.h"
#include "olap/tablet_reader.h"
#include "olap/utils.h"
#include "util/slice.h"
@@ -245,8 +241,7 @@ Status Merger::vertical_compact_one_group(
vectorized::RowSourcesBuffer* row_source_buf,
const std::vector<RowsetReaderSharedPtr>& src_rowset_readers,
RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, Statistics* stats_output,
- std::vector<uint32_t> key_group_cluster_key_idxes, int64_t batch_size,
- CompactionSampleInfo* sample_info) {
+ std::vector<uint32_t> key_group_cluster_key_idxes) {
// build tablet reader
VLOG_NOTICE << "vertical compact one group, max_rows_per_segment=" << max_rows_per_segment;
vectorized::VerticalBlockReader reader(row_source_buf);
@@ -284,8 +279,7 @@ Status Merger::vertical_compact_one_group(
reader_params.return_columns = column_group;
reader_params.origin_return_columns = &reader_params.return_columns;
- reader_params.batch_size = batch_size;
- RETURN_IF_ERROR(reader.init(reader_params, sample_info));
+ RETURN_IF_ERROR(reader.init(reader_params));
if (reader_params.record_rowids) {
stats_output->rowid_conversion->set_dst_rowset_id(dst_rowset_writer->rowset_id());
@@ -391,55 +385,6 @@ Status Merger::vertical_compact_one_group(int64_t tablet_id, ReaderType reader_t
return Status::OK();
}
-int64_t estimate_batch_size(int group_index, BaseTabletSPtr tablet, int64_t way_cnt) {
- std::unique_lock<std::mutex> lock(tablet->sample_info_lock);
- CompactionSampleInfo info = tablet->sample_infos[group_index];
- if (way_cnt <= 0) {
- LOG(INFO) << "estimate batch size for vertical compaction, tablet id: "
- << tablet->tablet_id() << " way cnt: " << way_cnt;
- return 4096 - 32;
- }
- int64_t block_mem_limit = config::compaction_memory_bytes_limit / way_cnt;
- if (tablet->last_compaction_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>()) {
- block_mem_limit /= 4;
- }
-
- int64_t group_data_size = 0;
- if (info.group_data_size > 0 && info.bytes > 0 && info.rows > 0) {
- float smoothing_factor = 0.5;
- group_data_size = int64_t(info.group_data_size * (1 - smoothing_factor) +
- info.bytes / info.rows * smoothing_factor);
- tablet->sample_infos[group_index].group_data_size = group_data_size;
- } else if (info.group_data_size > 0 && (info.bytes <= 0 || info.rows <= 0)) {
- group_data_size = info.group_data_size;
- } else if (info.group_data_size <= 0 && info.bytes > 0 && info.rows > 0) {
- group_data_size = info.bytes / info.rows;
- tablet->sample_infos[group_index].group_data_size = group_data_size;
- } else {
- LOG(INFO) << "estimate batch size for vertical compaction, tablet id: "
- << tablet->tablet_id() << " group data size: " << info.group_data_size
- << " row num: " << info.rows << " consume bytes: " << info.bytes;
- return 1024 - 32;
- }
-
- if (group_data_size <= 0) {
- LOG(WARNING) << "estimate batch size for vertical compaction, tablet id: "
- << tablet->tablet_id() << " unexpected group data size: " << group_data_size;
- return 4096 - 32;
- }
-
- tablet->sample_infos[group_index].bytes = 0;
- tablet->sample_infos[group_index].rows = 0;
-
- int64_t batch_size = block_mem_limit / group_data_size;
- int64_t res = std::max(std::min(batch_size, int64_t(4096 - 32)), 32L);
- LOG(INFO) << "estimate batch size for vertical compaction, tablet id: " << tablet->tablet_id()
- << " group data size: " << info.group_data_size << " row num: " << info.rows
- << " consume bytes: " << info.bytes << " way cnt: " << way_cnt
- << " batch size: " << res;
- return res;
-}
-
// steps to do vertical merge:
// 1. split columns into column groups
// 2. compact groups one by one, generate a row_source_buf when compact key group
@@ -449,7 +394,7 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t
const TabletSchema& tablet_schema,
const std::vector<RowsetReaderSharedPtr>& src_rowset_readers,
RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment,
- int64_t merge_way_num, Statistics* stats_output) {
+ Statistics* stats_output) {
LOG(INFO) << "Start to do vertical compaction, tablet_id: " << tablet->tablet_id();
std::vector<std::vector<uint32_t>> column_groups;
vertical_split_columns(tablet_schema, &column_groups);
@@ -460,18 +405,14 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr tablet, ReaderType reader_t
vectorized::RowSourcesBuffer row_sources_buf(
tablet->tablet_id(), dst_rowset_writer->context().tablet_path, reader_type);
- tablet->sample_infos.resize(column_groups.size(), {0, 0, 0});
// compact group one by one
for (auto i = 0; i < column_groups.size(); ++i) {
VLOG_NOTICE << "row source size: " << row_sources_buf.total_size();
bool is_key = (i == 0);
- int64_t batch_size = config::compaction_batch_size != -1
- ? config::compaction_batch_size
- : estimate_batch_size(i, tablet, merge_way_num);
RETURN_IF_ERROR(vertical_compact_one_group(
tablet, reader_type, tablet_schema, is_key, column_groups[i], &row_sources_buf,
src_rowset_readers, dst_rowset_writer, max_rows_per_segment, stats_output,
- key_group_cluster_key_idxes, batch_size, &(tablet->sample_infos[i])));
+ key_group_cluster_key_idxes));
if (is_key) {
RETURN_IF_ERROR(row_sources_buf.flush());
}
diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h
index 7513c90fbd1..5749f518136 100644
--- a/be/src/olap/merger.h
+++ b/be/src/olap/merger.h
@@ -21,7 +21,6 @@
#include "common/status.h"
#include "io/io_common.h"
-#include "olap/iterators.h"
#include "olap/rowset/rowset_fwd.h"
#include "olap/tablet_fwd.h"
@@ -60,7 +59,7 @@ public:
static Status vertical_merge_rowsets(
BaseTabletSPtr tablet, ReaderType reader_type, const TabletSchema& tablet_schema,
const std::vector<RowsetReaderSharedPtr>& src_rowset_readers,
- RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, int64_t merge_way_num,
+ RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment,
Statistics* stats_output);
// for vertical compaction
@@ -72,8 +71,7 @@ public:
vectorized::RowSourcesBuffer* row_source_buf,
const std::vector<RowsetReaderSharedPtr>& src_rowset_readers,
RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, Statistics* stats_output,
- std::vector<uint32_t> key_group_cluster_key_idxes, int64_t batch_size,
- CompactionSampleInfo* sample_info);
+ std::vector<uint32_t> key_group_cluster_key_idxes);
// for segcompaction
static Status vertical_compact_one_group(int64_t tablet_id, ReaderType reader_type,
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index aa20b5b1ef1..90b2ce48a0a 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -269,21 +269,6 @@ public:
return score;
}
- uint32_t get_merge_way_num() const {
- uint32_t way_num = 0;
- if (!is_segments_overlapping()) {
- if (num_segments() == 0) {
- way_num = 0;
- } else {
- way_num = 1;
- }
- } else {
- way_num = num_segments();
- CHECK(way_num > 0);
- }
- return way_num;
- }
-
void get_segments_key_bounds(std::vector<KeyBoundsPB>* segments_key_bounds) const {
for (const KeyBoundsPB& key_range : _rowset_meta_pb.segments_key_bounds()) {
segments_key_bounds->push_back(key_range);
diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp
index 95f2a945134..22a7049aa8f 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -101,7 +101,7 @@ Status SegcompactionWorker::_get_segcompaction_reader(
reader_params.tablet = tablet;
reader_params.return_columns = return_columns;
reader_params.is_key_column_group = is_key;
- return (*reader)->init(reader_params, nullptr);
+ return (*reader)->init(reader_params);
}
std::unique_ptr<segment_v2::SegmentWriter> SegcompactionWorker::_create_segcompaction_writer(
diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h
index c257ba007f5..a3cd3bd4a49 100644
--- a/be/src/olap/tablet_reader.h
+++ b/be/src/olap/tablet_reader.h
@@ -183,8 +183,6 @@ public:
void check_validation() const;
std::string to_string() const;
-
- int64_t batch_size = -1;
};
TabletReader() = default;
diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp
index 872836c91cd..c4dda20f40f 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -25,8 +25,6 @@
#include <ostream>
#include "cloud/config.h"
-#include "olap/compaction.h"
-#include "olap/iterators.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/rowset/rowset.h"
@@ -110,8 +108,7 @@ Status VerticalBlockReader::_get_segment_iterators(const ReaderParams& read_para
return Status::OK();
}
-Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params,
- CompactionSampleInfo* sample_info) {
+Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params) {
std::vector<bool> iterator_init_flag;
std::vector<RowsetId> rowset_ids;
std::vector<RowwiseIteratorUPtr>* segment_iters_ptr = read_params.segment_iters_ptr;
@@ -160,8 +157,7 @@ Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params,
// init collect iterator
StorageReadOptions opts;
opts.record_rowids = read_params.record_rowids;
- opts.block_row_max = read_params.batch_size;
- RETURN_IF_ERROR(_vcollect_iter->init(opts, sample_info));
+ RETURN_IF_ERROR(_vcollect_iter->init(opts));
// In agg keys value columns compact, get first row for _init_agg_state
if (!read_params.is_key_column_group && read_params.tablet->keys_type() == KeysType::AGG_KEYS) {
@@ -208,17 +204,13 @@ void VerticalBlockReader::_init_agg_state(const ReaderParams& read_params) {
}
Status VerticalBlockReader::init(const ReaderParams& read_params) {
- return init(read_params, nullptr);
-}
-
-Status VerticalBlockReader::init(const ReaderParams& read_params,
- CompactionSampleInfo* sample_info) {
StorageReadOptions opts;
- _reader_context.batch_size = read_params.batch_size;
+ _reader_context.batch_size = opts.block_row_max;
RETURN_IF_ERROR(TabletReader::init(read_params));
_arena = std::make_unique<Arena>();
- auto status = _init_collect_iter(read_params, sample_info);
+
+ auto status = _init_collect_iter(read_params);
if (!status.ok()) [[unlikely]] {
if (!config::is_cloud_mode()) {
static_cast<Tablet*>(_tablet.get())->report_error(status);
diff --git a/be/src/vec/olap/vertical_block_reader.h b/be/src/vec/olap/vertical_block_reader.h
index e1e8cfa1239..81ef8d79100 100644
--- a/be/src/vec/olap/vertical_block_reader.h
+++ b/be/src/vec/olap/vertical_block_reader.h
@@ -56,7 +56,6 @@ public:
// Initialize VerticalBlockReader with tablet, data version and fetch range.
Status init(const ReaderParams& read_params) override;
- Status init(const ReaderParams& read_params, CompactionSampleInfo* sample_info);
Status next_block_with_aggregation(Block* block, bool* eof) override;
@@ -80,7 +79,7 @@ private:
// to minimize the comparison time in merge heap.
Status _unique_key_next_block(Block* block, bool* eof);
- Status _init_collect_iter(const ReaderParams& read_params, CompactionSampleInfo* sample_info);
+ Status _init_collect_iter(const ReaderParams& read_params);
Status _get_segment_iterators(const ReaderParams& read_params,
std::vector<RowwiseIteratorUPtr>* segment_iters,
diff --git a/be/src/vec/olap/vertical_merge_iterator.cpp b/be/src/vec/olap/vertical_merge_iterator.cpp
index 81cfc756d63..3323492ee90 100644
--- a/be/src/vec/olap/vertical_merge_iterator.cpp
+++ b/be/src/vec/olap/vertical_merge_iterator.cpp
@@ -21,7 +21,6 @@
#include <gen_cpp/olap_file.pb.h>
#include <stdlib.h>
-#include <cstddef>
#include <ostream>
#include "cloud/config.h"
@@ -30,7 +29,6 @@
#include "common/logging.h"
#include "io/cache/block_file_cache_factory.h"
#include "olap/field.h"
-#include "olap/iterators.h"
#include "olap/olap_common.h"
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
@@ -342,18 +340,13 @@ Status VerticalMergeIteratorContext::copy_rows(Block* block, bool advanced) {
return Status::OK();
}
-Status VerticalMergeIteratorContext::init(const StorageReadOptions& opts,
- CompactionSampleInfo* sample_info) {
+Status VerticalMergeIteratorContext::init(const StorageReadOptions& opts) {
if (LIKELY(_inited)) {
return Status::OK();
}
_block_row_max = opts.block_row_max;
_record_rowids = opts.record_rowids;
RETURN_IF_ERROR(_load_next_block());
- if (sample_info != nullptr) {
- sample_info->bytes += bytes();
- sample_info->rows += rows();
- }
if (valid()) {
RETURN_IF_ERROR(advance());
}
@@ -512,8 +505,7 @@ Status VerticalHeapMergeIterator::next_batch(Block* block) {
return Status::EndOfFile("no more data in segment");
}
-Status VerticalHeapMergeIterator::init(const StorageReadOptions& opts,
- CompactionSampleInfo* sample_info) {
+Status VerticalHeapMergeIterator::init(const StorageReadOptions& opts) {
DCHECK(_origin_iters.size() == _iterator_init_flags.size());
_record_rowids = opts.record_rowids;
if (_origin_iters.empty()) {
@@ -541,7 +533,7 @@ Status VerticalHeapMergeIterator::init(const StorageReadOptions& opts,
for (size_t i = 0; i < num_iters; ++i) {
if (_iterator_init_flags[i] || pre_iter_invalid) {
auto& ctx = _ori_iter_ctx[i];
- RETURN_IF_ERROR(ctx->init(opts, sample_info));
+ RETURN_IF_ERROR(ctx->init(opts));
if (!ctx->valid()) {
pre_iter_invalid = true;
continue;
@@ -614,8 +606,7 @@ Status VerticalFifoMergeIterator::next_batch(Block* block) {
return Status::EndOfFile("no more data in segment");
}
-Status VerticalFifoMergeIterator::init(const StorageReadOptions& opts,
- CompactionSampleInfo* sample_info) {
+Status VerticalFifoMergeIterator::init(const StorageReadOptions& opts) {
DCHECK(_origin_iters.size() == _iterator_init_flags.size());
DCHECK(_keys_type == KeysType::DUP_KEYS);
_record_rowids = opts.record_rowids;
@@ -635,7 +626,7 @@ Status VerticalFifoMergeIterator::init(const StorageReadOptions& opts,
std::unique_ptr<VerticalMergeIteratorContext> ctx(
new VerticalMergeIteratorContext(std::move(iter), _rowset_ids[seg_order],
_ori_return_cols, seg_order, _seq_col_idx));
- RETURN_IF_ERROR(ctx->init(opts, sample_info));
+ RETURN_IF_ERROR(ctx->init(opts));
if (!ctx->valid()) {
++seg_order;
continue;
@@ -676,7 +667,7 @@ Status VerticalMaskMergeIterator::next_row(vectorized::IteratorRowRef* ref) {
uint16_t order = row_source.get_source_num();
auto& ctx = _origin_iter_ctx[order];
// init ctx and this ctx must be valid
- RETURN_IF_ERROR(ctx->init(_opts, _sample_info));
+ RETURN_IF_ERROR(ctx->init(_opts));
DCHECK(ctx->valid());
if (UNLIKELY(ctx->is_first_row())) {
@@ -710,7 +701,7 @@ Status VerticalMaskMergeIterator::unique_key_next_row(vectorized::IteratorRowRef
auto row_source = _row_sources_buf->current();
uint16_t order = row_source.get_source_num();
auto& ctx = _origin_iter_ctx[order];
- RETURN_IF_ERROR(ctx->init(_opts, _sample_info));
+ RETURN_IF_ERROR(ctx->init(_opts));
DCHECK(ctx->valid());
if (!ctx->valid()) {
LOG(INFO) << "VerticalMergeIteratorContext not valid";
@@ -749,7 +740,7 @@ Status VerticalMaskMergeIterator::next_batch(Block* block) {
uint16_t order = _row_sources_buf->current().get_source_num();
DCHECK(order < _origin_iter_ctx.size());
auto& ctx = _origin_iter_ctx[order];
- RETURN_IF_ERROR(ctx->init(_opts, _sample_info));
+ RETURN_IF_ERROR(ctx->init(_opts));
DCHECK(ctx->valid());
if (!ctx->valid()) {
LOG(INFO) << "VerticalMergeIteratorContext not valid";
@@ -772,8 +763,7 @@ Status VerticalMaskMergeIterator::next_batch(Block* block) {
return st;
}
-Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts,
- CompactionSampleInfo* sample_info) {
+Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts) {
if (_origin_iters.empty()) {
return Status::OK();
}
@@ -788,7 +778,6 @@ Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts,
}
_origin_iters.clear();
- _sample_info = sample_info;
_block_row_max = opts.block_row_max;
return Status::OK();
}
diff --git a/be/src/vec/olap/vertical_merge_iterator.h b/be/src/vec/olap/vertical_merge_iterator.h
index 3751aa92c78..f46a0446cf2 100644
--- a/be/src/vec/olap/vertical_merge_iterator.h
+++ b/be/src/vec/olap/vertical_merge_iterator.h
@@ -164,7 +164,7 @@ public:
~VerticalMergeIteratorContext() = default;
Status block_reset(const std::shared_ptr<Block>& block);
- Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info = nullptr);
+ Status init(const StorageReadOptions& opts);
bool compare(const VerticalMergeIteratorContext& rhs) const;
Status copy_rows(Block* block, bool advanced = true);
Status copy_rows(Block* block, size_t count);
@@ -200,22 +200,6 @@ public:
return _block_row_locations[_index_in_block];
}
- size_t bytes() {
- if (_block) {
- return _block->bytes();
- } else {
- return 0;
- }
- }
-
- size_t rows() {
- if (_block) {
- return _block->rows();
- } else {
- return 0;
- }
- }
-
private:
// Load next block into _block
Status _load_next_block();
@@ -271,7 +255,7 @@ public:
VerticalHeapMergeIterator(const VerticalHeapMergeIterator&) = delete;
VerticalHeapMergeIterator& operator=(const VerticalHeapMergeIterator&) = delete;
- Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) override;
+ Status init(const StorageReadOptions& opts) override;
Status next_batch(Block* block) override;
const Schema& schema() const override { return *_schema; }
uint64_t merged_rows() const override { return _merged_rows; }
@@ -337,7 +321,7 @@ public:
VerticalFifoMergeIterator(const VerticalFifoMergeIterator&) = delete;
VerticalFifoMergeIterator& operator=(const VerticalFifoMergeIterator&) = delete;
- Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) override;
+ Status init(const StorageReadOptions& opts) override;
Status next_batch(Block* block) override;
const Schema& schema() const override { return *_schema; }
uint64_t merged_rows() const override { return _merged_rows; }
@@ -383,7 +367,7 @@ public:
VerticalMaskMergeIterator(const VerticalMaskMergeIterator&) = delete;
VerticalMaskMergeIterator& operator=(const VerticalMaskMergeIterator&) = delete;
- Status init(const StorageReadOptions& opts, CompactionSampleInfo* sample_info) override;
+ Status init(const StorageReadOptions& opts) override;
Status next_batch(Block* block) override;
@@ -412,7 +396,6 @@ private:
size_t _filtered_rows = 0;
RowSourcesBuffer* _row_sources_buf;
StorageReadOptions _opts;
- CompactionSampleInfo* _sample_info = nullptr;
};
// segment merge iterator
diff --git a/be/test/olap/base_compaction_test.cpp b/be/test/olap/base_compaction_test.cpp
deleted file mode 100644
index 7d9abe54ed2..00000000000
--- a/be/test/olap/base_compaction_test.cpp
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/base_compaction.h"
-
-#include <gen_cpp/AgentService_types.h>
-#include <gen_cpp/olap_file.pb.h>
-#include <gtest/gtest-message.h>
-#include <gtest/gtest-test-part.h>
-
-#include "gtest/gtest.h"
-#include "gtest/gtest_pred_impl.h"
-#include "olap/cumulative_compaction.h"
-#include "olap/cumulative_compaction_policy.h"
-#include "olap/olap_common.h"
-#include "olap/rowset/rowset_factory.h"
-#include "olap/rowset/rowset_meta.h"
-#include "olap/storage_engine.h"
-#include "olap/tablet.h"
-#include "olap/tablet_meta.h"
-#include "util/uid_util.h"
-
-namespace doris {
-
-class TestBaseCompaction : public testing::Test {};
-
-static RowsetSharedPtr create_rowset(Version version, int num_segments, bool overlapping,
- int data_size) {
- auto rs_meta = std::make_shared<RowsetMeta>();
- rs_meta->set_rowset_type(BETA_ROWSET); // important
- rs_meta->_rowset_meta_pb.set_start_version(version.first);
- rs_meta->_rowset_meta_pb.set_end_version(version.second);
- rs_meta->set_num_segments(num_segments);
- rs_meta->set_segments_overlap(overlapping ? OVERLAPPING : NONOVERLAPPING);
- rs_meta->set_total_disk_size(data_size);
- RowsetSharedPtr rowset;
- Status st = RowsetFactory::create_rowset(nullptr, "", std::move(rs_meta), &rowset);
- if (!st.ok()) {
- return nullptr;
- }
- return rowset;
-}
-
-TEST_F(TestBaseCompaction, filter_input_rowset) {
- StorageEngine engine({});
- TabletMetaSharedPtr tablet_meta;
- tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}},
- UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK,
- TCompressionType::LZ4F));
- TabletSharedPtr tablet(new Tablet(engine, tablet_meta, nullptr, CUMULATIVE_SIZE_BASED_POLICY));
- tablet->_cumulative_point = 25;
- BaseCompaction compaction(engine, tablet);
- //std::vector<RowsetSharedPtr> rowsets;
-
- RowsetSharedPtr init_rs = create_rowset({0, 1}, 1, false, 0);
- tablet->_rs_version_map.emplace(init_rs->version(), init_rs);
- for (int i = 2; i < 30; ++i) {
- RowsetSharedPtr rs = create_rowset({i, i}, 1, false, 1024);
- tablet->_rs_version_map.emplace(rs->version(), rs);
- }
- Status st = compaction.pick_rowsets_to_compact();
- EXPECT_TRUE(st.ok());
- EXPECT_EQ(compaction._input_rowsets.front()->start_version(), 0);
- EXPECT_EQ(compaction._input_rowsets.front()->end_version(), 1);
-
- EXPECT_EQ(compaction._input_rowsets.back()->start_version(), 21);
- EXPECT_EQ(compaction._input_rowsets.back()->end_version(), 21);
-}
-
-} // namespace doris
diff --git a/be/test/olap/rowid_conversion_test.cpp b/be/test/olap/rowid_conversion_test.cpp
index 5ae80398afb..7c56710f2e8 100644
--- a/be/test/olap/rowid_conversion_test.cpp
+++ b/be/test/olap/rowid_conversion_test.cpp
@@ -348,9 +348,9 @@ protected:
stats.rowid_conversion = &rowid_conversion;
Status s;
if (is_vertical_merger) {
- s = Merger::vertical_merge_rowsets(
- tablet, ReaderType::READER_BASE_COMPACTION, *tablet_schema, input_rs_readers,
- output_rs_writer.get(), 10000000, num_segments, &stats);
+ s = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION,
+ *tablet_schema, input_rs_readers,
+ output_rs_writer.get(), 10000000, &stats);
} else {
s = Merger::vmerge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION, *tablet_schema,
input_rs_readers, output_rs_writer.get(), &stats);
diff --git a/be/test/vec/olap/vertical_compaction_test.cpp b/be/test/vec/olap/vertical_compaction_test.cpp
index 4c4409a7506..3afd748e14d 100644
--- a/be/test/vec/olap/vertical_compaction_test.cpp
+++ b/be/test/vec/olap/vertical_compaction_test.cpp
@@ -490,7 +490,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMerge) {
stats.rowid_conversion = &rowid_conversion;
auto s = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION,
*tablet_schema, input_rs_readers,
- output_rs_writer.get(), 100, num_segments, &stats);
+ output_rs_writer.get(), 100, &stats);
ASSERT_TRUE(s.ok()) << s;
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
@@ -598,7 +598,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMerge) {
stats.rowid_conversion = &rowid_conversion;
auto s = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION,
*tablet_schema, input_rs_readers,
- output_rs_writer.get(), 100, num_segments, &stats);
+ output_rs_writer.get(), 100, &stats);
ASSERT_TRUE(s.ok()) << s;
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
@@ -706,7 +706,7 @@ TEST_F(VerticalCompactionTest, TestUniqueKeyVerticalMerge) {
stats.rowid_conversion = &rowid_conversion;
auto s = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION,
*tablet_schema, input_rs_readers,
- output_rs_writer.get(), 10000, num_segments, &stats);
+ output_rs_writer.get(), 10000, &stats);
EXPECT_TRUE(s.ok());
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
@@ -815,8 +815,7 @@ TEST_F(VerticalCompactionTest, TestDupKeyVerticalMergeWithDelete) {
RowIdConversion rowid_conversion;
stats.rowid_conversion = &rowid_conversion;
st = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION, *tablet_schema,
- input_rs_readers, output_rs_writer.get(), 100, num_segments,
- &stats);
+ input_rs_readers, output_rs_writer.get(), 100, &stats);
ASSERT_TRUE(st.ok()) << st;
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
@@ -918,8 +917,7 @@ TEST_F(VerticalCompactionTest, TestDupWithoutKeyVerticalMergeWithDelete) {
RowIdConversion rowid_conversion;
stats.rowid_conversion = &rowid_conversion;
st = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION, *tablet_schema,
- input_rs_readers, output_rs_writer.get(), 100, num_segments,
- &stats);
+ input_rs_readers, output_rs_writer.get(), 100, &stats);
ASSERT_TRUE(st.ok()) << st;
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
@@ -1012,7 +1010,7 @@ TEST_F(VerticalCompactionTest, TestAggKeyVerticalMerge) {
stats.rowid_conversion = &rowid_conversion;
auto s = Merger::vertical_merge_rowsets(tablet, ReaderType::READER_BASE_COMPACTION,
*tablet_schema, input_rs_readers,
- output_rs_writer.get(), 100, num_segments, &stats);
+ output_rs_writer.get(), 100, &stats);
EXPECT_TRUE(s.ok());
RowsetSharedPtr out_rowset;
EXPECT_EQ(Status::OK(), output_rs_writer->build(out_rowset));
diff --git a/regression-test/suites/compaction/compaction_width_array_column.groovy b/regression-test/suites/compaction/compaction_width_array_column.groovy
deleted file mode 100644
index 4e3fed354c7..00000000000
--- a/regression-test/suites/compaction/compaction_width_array_column.groovy
+++ /dev/null
@@ -1,137 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite('compaction_width_array_column', "p2") {
- String backend_id;
- def backendId_to_backendIP = [:]
- def backendId_to_backendHttpPort = [:]
- getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
- backend_id = backendId_to_backendIP.keySet()[0]
- def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
-
- logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- def configList = parseJson(out.trim())
- assert configList instanceof List
-
- def s3BucketName = getS3BucketName()
- def random = new Random();
-
- def s3WithProperties = """WITH S3 (
- |"AWS_ACCESS_KEY" = "${getS3AK()}",
- |"AWS_SECRET_KEY" = "${getS3SK()}",
- |"AWS_ENDPOINT" = "${getS3Endpoint()}",
- |"AWS_REGION" = "${getS3Region()}")
- |PROPERTIES(
- |"exec_mem_limit" = "8589934592",
- |"load_parallelism" = "3")""".stripMargin()
-
- // set fe configuration
- sql "ADMIN SET FRONTEND CONFIG ('max_bytes_per_broker_scanner' = '161061273600')"
-
- def tableName = "column_witdh_array"
-
- def table_create_task = { table_name ->
- // drop table if exists
- sql """drop table if exists ${table_name}"""
- // create table
- def create_table = new File("""${context.file.parent}/ddl/${table_name}.sql""").text
- create_table = create_table.replaceAll("\\\$\\{table\\_name\\}", table_name)
- sql create_table
- }
-
- def table_load_task = { table_name ->
- uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString()
- loadLabel = table_name + "_" + uniqueID
- //loadLabel = table_name + '_load_5'
- loadSql = new File("""${context.file.parent}/ddl/${table_name}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName)
- loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel)
- loadSql = loadSql.replaceAll("\\\$\\{table\\_name\\}", table_name)
- nowloadSql = loadSql + s3WithProperties
- try_sql nowloadSql
-
- while (true) {
- def stateResult = sql "show load where Label = '${loadLabel}'"
- logger.info("load result is ${stateResult}")
- def loadState = stateResult[stateResult.size() - 1][2].toString()
- if ("CANCELLED".equalsIgnoreCase(loadState)) {
- throw new IllegalStateException("load ${loadLabel} failed.")
- } else if ("FINISHED".equalsIgnoreCase(loadState)) {
- break
- }
- sleep(5000)
- }
- }
-
- table_create_task(tableName)
- table_load_task(tableName)
-
- def tablets = sql_return_maparray """ show tablets from ${tableName}; """
-
- boolean isOverLap = true
- int tryCnt = 0;
- while (isOverLap && tryCnt < 3) {
- isOverLap = false
-
- for (def tablet in tablets) {
- String tablet_id = tablet.TabletId
- backend_id = tablet.BackendId
- (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
- logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- def compactJson = parseJson(out.trim())
- assertEquals("success", compactJson.status.toLowerCase())
- }
-
- // wait for all compactions done
- for (def tablet in tablets) {
- boolean running = true
- do {
- Thread.sleep(1000)
- String tablet_id = tablet.TabletId
- backend_id = tablet.BackendId
- (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
- logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- def compactionStatus = parseJson(out.trim())
- assertEquals("success", compactionStatus.status.toLowerCase())
- running = compactionStatus.run_status
- } while (running)
- }
-
- for (def tablet in tablets) {
- String tablet_id = tablet.TabletId
- (code, out, err) = curl("GET", tablet.CompactionStatus)
- logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- def tabletJson = parseJson(out.trim())
- assert tabletJson.rowsets instanceof List
- for (String rowset in (List<String>) tabletJson.rowsets) {
- logger.info("rowset info" + rowset)
- String overLappingStr = rowset.split(" ")[3]
- if (overLappingStr == "OVERLAPPING") {
- isOverLap = true;
- }
- logger.info("is over lap " + isOverLap + " " + overLappingStr)
- }
- }
- tryCnt++;
- }
-
- assertFalse(isOverLap);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/21: [Fix](variant) ignore serialization of nothing type (#36997)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 15f162f69054ff62464f3cf1710ea6114fdbf438
Author: lihangyu <15...@163.com>
AuthorDate: Fri Jun 28 18:23:35 2024 +0800
[Fix](variant) ignore serialization of nothing type (#36997)
1. fix variant should not serialize nothing type
2. fix unstable cases
---
be/src/vec/data_types/data_type_object.cpp | 14 +++++++++++---
regression-test/data/variant_p0/load.out | 12 ++++++------
.../data/variant_p0/test_sub_path_pruning.out | 10 +---------
regression-test/suites/variant_p0/load.groovy | 8 ++++----
.../suites/variant_p0/test_sub_path_pruning.groovy | 20 +++++++++++---------
5 files changed, 33 insertions(+), 31 deletions(-)
diff --git a/be/src/vec/data_types/data_type_object.cpp b/be/src/vec/data_types/data_type_object.cpp
index 7adc4c17f56..7a75583cd7b 100644
--- a/be/src/vec/data_types/data_type_object.cpp
+++ b/be/src/vec/data_types/data_type_object.cpp
@@ -63,6 +63,9 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
size += sizeof(uint32_t);
for (const auto& entry : subcolumns) {
auto type = entry->data.get_least_common_type();
+ if (is_nothing(type)) {
+ continue;
+ }
PColumnMeta column_meta_pb;
column_meta_pb.set_name(entry->path.get_path());
@@ -91,15 +94,18 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve
const auto& subcolumns = column_object.get_subcolumns();
- // 1. serialize num of subcolumns
- *reinterpret_cast<uint32_t*>(buf) = subcolumns.size();
+ char* size_pos = buf;
buf += sizeof(uint32_t);
+ size_t num_of_columns = 0;
// 2. serialize each subcolumn in a loop
for (const auto& entry : subcolumns) {
// 2.1 serialize subcolumn column meta pb (path and type)
auto type = entry->data.get_least_common_type();
-
+ if (is_nothing(type)) {
+ continue;
+ }
+ ++num_of_columns;
PColumnMeta column_meta_pb;
column_meta_pb.set_name(entry->path.get_path());
type->to_pb_column_meta(&column_meta_pb);
@@ -113,6 +119,8 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve
// 2.2 serialize subcolumn
buf = type->serialize(entry->data.get_finalized_column(), buf, be_exec_version);
}
+ // serialize num of subcolumns
+ *reinterpret_cast<uint32_t*>(size_pos) = num_of_columns;
return buf;
}
diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out
index ab83a86cfb3..954faffccb0 100644
--- a/regression-test/data/variant_p0/load.out
+++ b/regression-test/data/variant_p0/load.out
@@ -79,11 +79,11 @@
-- !sql --
{"c":"123"}
{"c":123}
-{"cc":[123.0]}
+{"cc":[123.2]}
{"cc":[123.1]}
{"ccc":123}
{"ccc":123321}
-{"cccc":123.0}
+{"cccc":123.22}
{"cccc":123.11}
{"ccccc":[123]}
{"ccccc":[123456789]}
@@ -123,7 +123,7 @@
1.10111 1800
1.1111 17211
\N 123456
-123 191191
+123.22 191191
\N 123456789101112
-- !sql_7 --
@@ -155,7 +155,7 @@
123 {"A":123}
123456 {"A":123456}
123456789101112 {"A":123456789101112}
-191191 {"A":191191,"a":123.0,"c":123}
+191191 {"A":191191,"a":123.22,"c":123}
1800 {"A":1800,"a":1.10111,"c":[12345]}
17211 {"A":17211,"a":1.1111,"c":111111}
@@ -175,7 +175,7 @@
\N 123456789101112 {"A":123456789101112} \N
\N \N {"AA":[123456]} \N
\N \N {"AA":[123456789101112]} \N
-123 191191 {"A":191191,"a":123.0,"c":123} \N
+123.22 191191 {"A":191191,"a":123.22,"c":123} \N
123 \N {"a":"123","c":123456} \N
1.10111 1800 {"A":1800,"a":1.10111,"c":[12345]} \N
1.1111 17211 {"A":17211,"a":1.1111,"c":111111} \N
@@ -212,7 +212,7 @@
[123]
-- !sql_25 --
-50000 54999.99999999862 6150000
+50000 54999.999999999935 6150000
-- !sql_26 --
5000
diff --git a/regression-test/data/variant_p0/test_sub_path_pruning.out b/regression-test/data/variant_p0/test_sub_path_pruning.out
index a48bc550d00..16328739167 100644
--- a/regression-test/data/variant_p0/test_sub_path_pruning.out
+++ b/regression-test/data/variant_p0/test_sub_path_pruning.out
@@ -233,11 +233,7 @@
{"b":{"c":{"d":{"e":11}}},"c":{"d":{"e":12}},"d":{"e":13},"e":14}
-- !sql --
-
-{"c":{"d":{"e":11}}}
-
--- !sql --
-""
+1
{"c":{"d":{"e":11}}}
-- !sql --
@@ -252,10 +248,6 @@
\N
--- !sql --
-""
-{"e":11}
-
-- !sql --
1 1
2 1
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index 4103a2b34fa..cbd6bc1178c 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -93,7 +93,7 @@ suite("regression_test_variant", "nonConcurrent"){
sql """insert into ${table_name} values (8, '8.11111'),(1, '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
sql """insert into ${table_name} values (9, '"9999"'),(1, '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
sql """insert into ${table_name} values (10, '1000000'),(1, '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
- sql """insert into ${table_name} values (11, '[123.0]'),(1999, '{"a" : 1, "b" : {"c" : 1}}'),(19921, '{"a" : 1, "b" : 10}');"""
+ sql """insert into ${table_name} values (11, '[123.1]'),(1999, '{"a" : 1, "b" : {"c" : 1}}'),(19921, '{"a" : 1, "b" : 10}');"""
sql """insert into ${table_name} values (12, '[123.2]'),(1022, '{"a" : 1, "b" : 10}'),(1029, '{"a" : 1, "b" : {"c" : 1}}');"""
qt_sql1 "select k, cast(v['a'] as array<int>) from ${table_name} where size(cast(v['a'] as array<int>)) > 0 order by k, cast(v['a'] as string) asc"
qt_sql2 "select k, cast(v as int), cast(v['b'] as string) from ${table_name} where length(cast(v['b'] as string)) > 4 order by k, cast(v as string), cast(v['b'] as string) "
@@ -110,11 +110,11 @@ suite("regression_test_variant", "nonConcurrent"){
create_table table_name
sql """insert into ${table_name} values (1, '{"c" : "123"}');"""
sql """insert into ${table_name} values (2, '{"c" : 123}');"""
- sql """insert into ${table_name} values (3, '{"cc" : [123.0]}');"""
+ sql """insert into ${table_name} values (3, '{"cc" : [123.2]}');"""
sql """insert into ${table_name} values (4, '{"cc" : [123.1]}');"""
sql """insert into ${table_name} values (5, '{"ccc" : 123}');"""
sql """insert into ${table_name} values (6, '{"ccc" : 123321}');"""
- sql """insert into ${table_name} values (7, '{"cccc" : 123.0}');"""
+ sql """insert into ${table_name} values (7, '{"cccc" : 123.22}');"""
sql """insert into ${table_name} values (8, '{"cccc" : 123.11}');"""
sql """insert into ${table_name} values (9, '{"ccccc" : [123]}');"""
sql """insert into ${table_name} values (10, '{"ccccc" : [123456789]}');"""
@@ -139,7 +139,7 @@ suite("regression_test_variant", "nonConcurrent"){
qt_sql_4 "select cast(v['A'] as string), v['AA'], v from ${table_name} order by k"
qt_sql_5 "select v['A'], v['AA'], v, v from ${table_name} where cast(v['A'] as bigint) > 123 order by k"
- sql """insert into ${table_name} values (16, '{"a" : 123.0, "A" : 191191, "c": 123}');"""
+ sql """insert into ${table_name} values (16, '{"a" : 123.22, "A" : 191191, "c": 123}');"""
sql """insert into ${table_name} values (18, '{"a" : "123", "c" : 123456}');"""
sql """insert into ${table_name} values (20, '{"a" : 1.10111, "A" : 1800, "c" : [12345]}');"""
// sql """insert into ${table_name} values (12, '{"a" : [123]}, "c": "123456"');"""
diff --git a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
index a6bce99d5e4..bd819934910 100644
--- a/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
+++ b/regression-test/suites/variant_p0/test_sub_path_pruning.groovy
@@ -138,7 +138,9 @@ suite("variant_sub_path_pruning", "variant_type"){
// distinct could not push down, only push down union all
// two children
- order_qt_sql """select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;"""
+ order_qt_sql """
+ select /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=6,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_thresh [...]
+ """
order_qt_sql """select c1['a'] from (select dt as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test) v1;"""
order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
@@ -159,19 +161,19 @@ suite("variant_sub_path_pruning", "variant_type"){
order_qt_sql """select c1['c']['d'] from (select dt['a']['b'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
// one table + one const list
- order_qt_sql """select id, c1['a'] from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100;"""
+ order_qt_sql """select id, cast(c1['a'] as text) from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100;"""
order_qt_sql """select c1['a'] from (select id, c1 from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
- order_qt_sql """select c2['b'] from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
- order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
- order_qt_sql """select id, c1['c'] from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100;"""
+ order_qt_sql """select c2['b'] from (select id, cast(c1['a'] as text) as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+ // order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('1' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+ order_qt_sql """select id, cast(c1['c'] as text) from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by 1, 2 limit 100;"""
order_qt_sql """select c1['c'] from (select id, c1 from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
- order_qt_sql """select c2['d'] from (select id, c1['a'] as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
- order_qt_sql """select c2['c']['d'] from (select id, c1 as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+ order_qt_sql """select cast(c2['d'] as text) from (select id, c1['a'] as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+ // order_qt_sql """select c2['c']['d'] from (select id, c1 as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
// two const list
- order_qt_sql """select id, c1['a'] from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100;"""
+ order_qt_sql """select id, cast(c1['a'] as text) from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100;"""
order_qt_sql """select c1['a'] from (select id, c1 from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
- order_qt_sql """select c2['b'] from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
+ order_qt_sql """select cast(c2['b'] as text) from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 14/21: [Fix](autoinc) try fix concurrent load problem with auto inc column (#36421)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b2f90a1ce4f527490f603ff652a94e6a8a4c674d
Author: bobhan1 <bh...@outlook.com>
AuthorDate: Fri Jun 28 21:12:21 2024 +0800
[Fix](autoinc) try fix concurrent load problem with auto inc column (#36421)
1. increase the value of `AutoIncrementGenerator.BATCH_ID_INTERVAL` to
reduce the number of writes to BDBJE. (the default value of
`config::auto_inc_prefetch_size_ratio` is 10 and the default value of
`AutoIncIDBuffer::_batch_size` is 4064, so mostly the request length is
40960)
2. only allow master fe to offer `getAutoIncrementRange` service
3. write editlog before update `batchEndId` in memory in
`getAutoIncrementRange `
4. refactor `AutoIncIDBuffer`
---
be/src/vec/sink/autoinc_buffer.cpp | 150 +++++++++++++--------
be/src/vec/sink/autoinc_buffer.h | 35 +++--
.../doris/catalog/AutoIncrementGenerator.java | 5 +-
.../apache/doris/service/FrontendServiceImpl.java | 10 ++
gensrc/thrift/FrontendService.thrift | 1 +
.../unique/test_unique_auto_inc_concurrent.out | 10 ++
.../unique/test_unique_auto_inc_concurrent.groovy | 59 ++++++++
7 files changed, 203 insertions(+), 67 deletions(-)
diff --git a/be/src/vec/sink/autoinc_buffer.cpp b/be/src/vec/sink/autoinc_buffer.cpp
index c7c096ec6e8..f83dbcb55b8 100644
--- a/be/src/vec/sink/autoinc_buffer.cpp
+++ b/be/src/vec/sink/autoinc_buffer.cpp
@@ -19,14 +19,15 @@
#include <gen_cpp/HeartbeatService_types.h>
-#include <string>
+#include <chrono>
+#include <mutex>
+#include "common/logging.h"
#include "common/status.h"
#include "runtime/client_cache.h"
#include "runtime/exec_env.h"
#include "util/runtime_profile.h"
#include "util/thrift_rpc_helper.h"
-#include "vec/sink/vtablet_block_convertor.h"
namespace doris::vectorized {
@@ -42,54 +43,11 @@ void AutoIncIDBuffer::set_batch_size_at_least(size_t batch_size) {
}
}
-void AutoIncIDBuffer::_wait_for_prefetching() {
- if (_is_fetching) {
- _rpc_token->wait();
- }
-}
-
-Status AutoIncIDBuffer::sync_request_ids(size_t length,
- std::vector<std::pair<int64_t, size_t>>* result) {
- std::unique_lock<std::mutex> lock(_mutex);
- RETURN_IF_ERROR(_prefetch_ids(_prefetch_size()));
- if (_front_buffer.second > 0) {
- auto min_length = std::min(_front_buffer.second, length);
- length -= min_length;
- result->emplace_back(_front_buffer.first, min_length);
- _front_buffer.first += min_length;
- _front_buffer.second -= min_length;
- }
- if (length > 0) {
- _wait_for_prefetching();
- if (!_rpc_status.ok()) {
- return _rpc_status;
- }
-
- {
- std::lock_guard<std::mutex> lock(_backend_buffer_latch);
- std::swap(_front_buffer, _backend_buffer);
- }
-
- DCHECK_LE(length, _front_buffer.second);
- if (length > _front_buffer.second) {
- return Status::RpcError("auto inc sync result length > front buffer. " +
- std::to_string(length) + " vs " +
- std::to_string(_front_buffer.second));
- }
- result->emplace_back(_front_buffer.first, length);
- _front_buffer.first += length;
- _front_buffer.second -= length;
- }
- return Status::OK();
-}
-
-Status AutoIncIDBuffer::_prefetch_ids(size_t length) {
- if (_front_buffer.second > _low_water_level_mark() || _is_fetching) {
- return Status::OK();
- }
+Result<int64_t> AutoIncIDBuffer::_fetch_ids_from_fe(size_t length) {
+ constexpr uint32_t FETCH_AUTOINC_MAX_RETRY_TIMES = 3;
+ _rpc_status = Status::OK();
TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address;
- _is_fetching = true;
- RETURN_IF_ERROR(_rpc_token->submit_func([=, this]() {
+ for (uint32_t retry_times = 0; retry_times < FETCH_AUTOINC_MAX_RETRY_TIMES; retry_times++) {
TAutoIncrementRangeRequest request;
TAutoIncrementRangeResult result;
request.__set_db_id(_db_id);
@@ -97,7 +55,7 @@ Status AutoIncIDBuffer::_prefetch_ids(size_t length) {
request.__set_column_id(_column_id);
request.__set_length(length);
- int64_t get_auto_inc_range_rpc_ns;
+ int64_t get_auto_inc_range_rpc_ns = 0;
{
SCOPED_RAW_TIMER(&get_auto_inc_range_rpc_ns);
_rpc_status = ThriftRpcHelper::rpc<FrontendServiceClient>(
@@ -109,15 +67,95 @@ Status AutoIncIDBuffer::_prefetch_ids(size_t length) {
LOG(INFO) << "[auto-inc-range][start=" << result.start << ",length=" << result.length
<< "][elapsed=" << get_auto_inc_range_rpc_ns / 1000000 << " ms]";
- if (!_rpc_status.ok() || result.length <= 0) {
- LOG(WARNING) << "Failed to fetch auto-incremnt range, encounter rpc failure."
- << "errmsg=" << _rpc_status.to_string();
- return;
+ if (_rpc_status.is<ErrorCode::NOT_MASTER>()) {
+ LOG_WARNING(
+ "Failed to fetch auto-incremnt range, request to non-master FE, discard all "
+ "auto_increment ranges in _buffers. retry_time={}",
+ retry_times);
+ master_addr = result.master_address;
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ continue;
}
+ if (!_rpc_status.ok()) {
+ LOG(WARNING)
+ << "Failed to fetch auto-incremnt range, encounter rpc failure. retry_time="
+ << retry_times << ", errmsg=" << _rpc_status.to_string();
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ continue;
+ }
+ if (result.length != length) [[unlikely]] {
+ auto msg = fmt::format(
+ "Failed to fetch auto-incremnt range, request length={}, but get "
+ "result.length={}, retry_time={}",
+ length, result.length, retry_times);
+ LOG(WARNING) << msg;
+ _rpc_status = Status::RpcError<true>(msg);
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ continue;
+ }
+
+ return result.start;
+ }
+ CHECK(!_rpc_status.ok());
+ return _rpc_status;
+}
+
+void AutoIncIDBuffer::_get_autoinc_ranges_from_buffers(
+ size_t& request_length, std::vector<std::pair<int64_t, size_t>>* result) {
+ std::lock_guard<std::mutex> lock {_latch};
+ while (request_length > 0 && !_buffers.empty()) {
+ auto& autoinc_range = _buffers.front();
+ CHECK_GT(autoinc_range.length, 0);
+ auto min_length = std::min(request_length, autoinc_range.length);
+ result->emplace_back(autoinc_range.start, min_length);
+ autoinc_range.consume(min_length);
+ _current_volume -= min_length;
+ request_length -= min_length;
+ if (autoinc_range.empty()) {
+ _buffers.pop_front();
+ }
+ }
+}
+
+Status AutoIncIDBuffer::sync_request_ids(size_t request_length,
+ std::vector<std::pair<int64_t, size_t>>* result) {
+ std::lock_guard<std::mutex> lock(_mutex);
+ while (request_length > 0) {
+ _get_autoinc_ranges_from_buffers(request_length, result);
+ if (request_length == 0) {
+ break;
+ }
+ if (!_is_fetching) {
+ RETURN_IF_ERROR(
+ _launch_async_fetch_task(std::max<size_t>(request_length, _prefetch_size())));
+ }
+ _rpc_token->wait();
+ CHECK(!_is_fetching);
+ if (!_rpc_status.ok()) {
+ return _rpc_status;
+ }
+ }
+ CHECK_EQ(request_length, 0);
+ if (!_is_fetching && _current_volume < _low_water_level_mark()) {
+ RETURN_IF_ERROR(_launch_async_fetch_task(_prefetch_size()));
+ }
+ return Status::OK();
+}
+
+Status AutoIncIDBuffer::_launch_async_fetch_task(size_t length) {
+ _is_fetching = true;
+ RETURN_IF_ERROR(_rpc_token->submit_func([=, this]() {
+ auto&& res = _fetch_ids_from_fe(length);
+ if (!res.has_value()) [[unlikely]] {
+ _is_fetching = false;
+ return;
+ }
+ int64_t start = res.value();
{
- std::lock_guard<std::mutex> lock(_backend_buffer_latch);
- _backend_buffer = {result.start, result.length};
+ std::lock_guard<std::mutex> lock {_latch};
+ _buffers.emplace_back(start, length);
+ _current_volume += length;
}
_is_fetching = false;
}));
diff --git a/be/src/vec/sink/autoinc_buffer.h b/be/src/vec/sink/autoinc_buffer.h
index 3ec009b0960..032ac18981f 100644
--- a/be/src/vec/sink/autoinc_buffer.h
+++ b/be/src/vec/sink/autoinc_buffer.h
@@ -61,17 +61,35 @@ public:
// all public functions are thread safe
AutoIncIDBuffer(int64_t _db_id, int64_t _table_id, int64_t column_id);
void set_batch_size_at_least(size_t batch_size);
- Status sync_request_ids(size_t length, std::vector<std::pair<int64_t, size_t>>* result);
+ Status sync_request_ids(size_t request_length, std::vector<std::pair<int64_t, size_t>>* result);
+
+ struct AutoIncRange {
+ int64_t start;
+ size_t length;
+
+ bool empty() const { return length == 0; }
+
+ void consume(size_t l) {
+ start += l;
+ length -= l;
+ }
+ };
private:
- Status _prefetch_ids(size_t length);
[[nodiscard]] size_t _prefetch_size() const {
return _batch_size * config::auto_inc_prefetch_size_ratio;
}
+
[[nodiscard]] size_t _low_water_level_mark() const {
return _batch_size * config::auto_inc_low_water_level_mark_size_ratio;
};
- void _wait_for_prefetching();
+
+ void _get_autoinc_ranges_from_buffers(size_t& request_length,
+ std::vector<std::pair<int64_t, size_t>>* result);
+
+ Status _launch_async_fetch_task(size_t length);
+
+ Result<int64_t> _fetch_ids_from_fe(size_t length);
std::atomic<size_t> _batch_size {MIN_BATCH_SIZE};
@@ -81,12 +99,14 @@ private:
std::unique_ptr<ThreadPoolToken> _rpc_token;
Status _rpc_status {Status::OK()};
+
std::atomic<bool> _is_fetching {false};
- std::pair<int64_t, size_t> _front_buffer {0, 0};
- std::pair<int64_t, size_t> _backend_buffer {0, 0};
- std::mutex _backend_buffer_latch; // for _backend_buffer
std::mutex _mutex;
+
+ mutable std::mutex _latch;
+ std::list<AutoIncRange> _buffers;
+ size_t _current_volume {0};
};
class GlobalAutoIncBuffers {
@@ -115,8 +135,7 @@ public:
auto key = std::make_tuple(db_id, table_id, column_id);
auto it = _buffers.find(key);
if (it == _buffers.end()) {
- _buffers.emplace(std::make_pair(
- key, AutoIncIDBuffer::create_shared(db_id, table_id, column_id)));
+ _buffers.emplace(key, AutoIncIDBuffer::create_shared(db_id, table_id, column_id));
}
return _buffers[{db_id, table_id, column_id}];
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AutoIncrementGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AutoIncrementGenerator.java
index be110360850..e4c8cf5de01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AutoIncrementGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AutoIncrementGenerator.java
@@ -40,7 +40,7 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
public static final long NEXT_ID_INIT_VALUE = 1;
// _MIN_BATCH_SIZE = 4064 in load task
- private static final long BATCH_ID_INTERVAL = 50000;
+ private static final long BATCH_ID_INTERVAL = 500000;
@SerializedName(value = "dbId")
private Long dbId;
@@ -48,7 +48,6 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
private Long tableId;
@SerializedName(value = "columnId")
private Long columnId;
- @SerializedName(value = "nextId")
private long nextId;
@SerializedName(value = "batchEndId")
private long batchEndId;
@@ -86,10 +85,10 @@ public class AutoIncrementGenerator implements Writable, GsonPostProcessable {
long endId = startId + length;
nextId = startId + length;
if (endId > batchEndId) {
- batchEndId = (endId / BATCH_ID_INTERVAL + 1) * BATCH_ID_INTERVAL;
Preconditions.checkState(editLog != null);
AutoIncrementIdUpdateLog info = new AutoIncrementIdUpdateLog(dbId, tableId, columnId, batchEndId);
editLog.logUpdateAutoIncrementId(info);
+ batchEndId = (endId / BATCH_ID_INTERVAL + 1) * BATCH_ID_INTERVAL;
}
LOG.info("[getAutoIncrementRange result][{}, {}]", startId, length);
return Pair.of(startId, length);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 1b106576720..75a1987eb23 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -2626,6 +2626,16 @@ public class FrontendServiceImpl implements FrontendService.Iface {
TAutoIncrementRangeResult result = new TAutoIncrementRangeResult();
TStatus status = new TStatus(TStatusCode.OK);
result.setStatus(status);
+
+ if (!Env.getCurrentEnv().isMaster()) {
+ status.setStatusCode(TStatusCode.NOT_MASTER);
+ status.addToErrorMsgs(NOT_MASTER_ERR_MSG);
+ result.setMasterAddress(getMasterAddress());
+ LOG.error("failed to getAutoIncrementRange:{}, request:{}, backend:{}",
+ NOT_MASTER_ERR_MSG, request, getClientAddrAsString());
+ return result;
+ }
+
try {
Env env = Env.getCurrentEnv();
Database db = env.getInternalCatalog().getDbOrMetaException(request.getDbId());
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 9a57abc9b61..d25f5c0ac2b 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1323,6 +1323,7 @@ struct TAutoIncrementRangeResult {
1: optional Status.TStatus status
2: optional i64 start
3: optional i64 length
+ 4: optional Types.TNetworkAddress master_address
}
struct TCreatePartitionRequest {
diff --git a/regression-test/data/data_model_p0/unique/test_unique_auto_inc_concurrent.out b/regression-test/data/data_model_p0/unique/test_unique_auto_inc_concurrent.out
new file mode 100644
index 00000000000..03819c9a717
--- /dev/null
+++ b/regression-test/data/data_model_p0/unique/test_unique_auto_inc_concurrent.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1500000 1500000
+
+-- !sql --
+3000000 3000000
+
+-- !sql --
+4500000 4500000
+
diff --git a/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy b/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
new file mode 100644
index 00000000000..bf6d584b2af
--- /dev/null
+++ b/regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_unique_table_auto_inc_concurrent") {
+
+ def table1 = "test_unique_table_auto_inc_concurrent"
+ sql "drop table if exists ${table1}"
+ sql """
+ CREATE TABLE IF NOT EXISTS `${table1}` (
+ `id` BIGINT NOT NULL AUTO_INCREMENT,
+ `value` int(11) NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "enable_unique_key_merge_on_write" = "true"
+ )
+ """
+
+ def run_test = {thread_num, rows, iters ->
+ def threads = []
+ (1..thread_num).each { id1 ->
+ threads.add(Thread.start {
+ (1..iters).each { id2 ->
+ sql """insert into ${table1}(value) select number from numbers("number" = "${rows}");"""
+ }
+ })
+ }
+
+ threads.each { thread -> thread.join() }
+
+ qt_sql "select count(id), count(distinct id) from ${table1};"
+ }
+
+ run_test(15, 10000, 10)
+ run_test(15, 100000, 1)
+ run_test(5, 30000, 10)
+
+ sql "drop table if exists ${table1};"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 10/21: [fix](regression test) fix unstable single compaction test p2 (#36881)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 039d4f722e3a77c2970e3811e84ba3e9ea0f28f3
Author: Sun Chenyang <cs...@gmail.com>
AuthorDate: Fri Jun 28 19:22:05 2024 +0800
[fix](regression test) fix unstable single compaction test p2 (#36881)
---
.../test_single_compaction_fault_injection.groovy | 68 ++++++++++++--------
.../test_single_replica_compaction.groovy | 72 ++++++++++++----------
2 files changed, 83 insertions(+), 57 deletions(-)
diff --git a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy
index 7acafae51ab..ebc74257032 100644
--- a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy
+++ b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy
@@ -32,22 +32,20 @@ suite("test_single_compaction_fault_injection", "p2") {
}
def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id ->
- StringBuilder sb = new StringBuilder();
- sb.append("curl -X POST http://${be_host}:${be_http_port}")
- sb.append("/api/compaction/run?tablet_id=")
- sb.append(tablet_id)
- sb.append("&compact_type=${compact_type}")
-
- String command = sb.toString()
- logger.info(command)
- process = command.execute()
- code = process.waitFor()
- err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
- out = process.getText()
- logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- return out
- }
+ if (compact_type == "cumulative") {
+ def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id)
+ logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1)
+ assertEquals(code_1, 0)
+ return out_1
+ } else if (compact_type == "full") {
+ def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id)
+ logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2)
+ assertEquals(code_2, 0)
+ return out_2
+ } else {
+ assertFalse(True)
+ }
+ }
def triggerSingleCompaction = { be_host, be_http_port, tablet_id ->
StringBuilder sb = new StringBuilder();
@@ -56,15 +54,33 @@ suite("test_single_compaction_fault_injection", "p2") {
sb.append(tablet_id)
sb.append("&compact_type=cumulative&remote=true")
- String command = sb.toString()
- logger.info(command)
- process = command.execute()
- code = process.waitFor()
- err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
- out = process.getText()
- logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
- assertEquals(code, 0)
- return out
+ Integer maxRetries = 10; // Maximum number of retries
+ Integer retryCount = 0; // Current retry count
+ Integer sleepTime = 5000; // Sleep time in milliseconds
+ String cmd = sb.toString()
+ def process
+ int code_3
+ String err_3
+ String out_3
+
+ while (retryCount < maxRetries) {
+ process = cmd.execute()
+ code_3 = process.waitFor()
+ err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())))
+ out_3 = process.getText()
+
+ // If the command was successful, break the loop
+ if (code_3 == 0) {
+ break
+ }
+
+ // If the command was not successful, increment the retry count, sleep for a while and try again
+ retryCount++
+ sleep(sleepTime)
+ }
+ assertEquals(code_3, 0)
+ logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3)
+ return out_3
}
def waitForCompaction = { be_host, be_http_port, tablet_id ->
boolean running = true
@@ -337,7 +353,7 @@ suite("test_single_compaction_fault_injection", "p2") {
// trigger master be to do base compaction
assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id],
- "base", tablet_id).contains("Success"));
+ "full", tablet_id).contains("Success"));
waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id)
// trigger follower be to fetch compaction result
diff --git a/regression-test/suites/compaction/test_single_replica_compaction.groovy b/regression-test/suites/compaction/test_single_replica_compaction.groovy
index 455aff9cf86..2387cc8f323 100644
--- a/regression-test/suites/compaction/test_single_replica_compaction.groovy
+++ b/regression-test/suites/compaction/test_single_replica_compaction.groovy
@@ -64,25 +64,20 @@ suite("test_single_replica_compaction", "p2") {
has_update_be_config = true
def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id ->
- StringBuilder sb = new StringBuilder();
- sb.append("curl -X POST http://${be_host}:${be_http_port}")
- sb.append("/api/compaction/run?tablet_id=")
- sb.append(tablet_id)
- sb.append("&compact_type=${compact_type}")
-
- String command = sb.toString()
- logger.info(command)
- process = command.execute()
- code = process.waitFor()
- err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
- out = process.getText()
- logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err)
- if (!disableAutoCompaction) {
- return "Success, " + out
+ if (compact_type == "cumulative") {
+ def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id)
+ logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1)
+ assertEquals(code_1, 0)
+ return out_1
+ } else if (compact_type == "full") {
+ def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id)
+ logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2)
+ assertEquals(code_2, 0)
+ return out_2
+ } else {
+ assertFalse(True)
}
- assertEquals(code, 0)
- return out
- }
+ }
def triggerSingleCompaction = { be_host, be_http_port, tablet_id ->
StringBuilder sb = new StringBuilder();
@@ -91,18 +86,33 @@ suite("test_single_replica_compaction", "p2") {
sb.append(tablet_id)
sb.append("&compact_type=cumulative&remote=true")
- String command = sb.toString()
- logger.info(command)
- process = command.execute()
- code = process.waitFor()
- err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
- out = process.getText()
- logger.info("Run compaction: code=" + code + ", out=" + out + ", disableAutoCompaction " + disableAutoCompaction + ", err=" + err)
- if (!disableAutoCompaction) {
- return "Success, " + out
+ Integer maxRetries = 10; // Maximum number of retries
+ Integer retryCount = 0; // Current retry count
+ Integer sleepTime = 5000; // Sleep time in milliseconds
+ String cmd = sb.toString()
+ def process
+ int code_3
+ String err_3
+ String out_3
+
+ while (retryCount < maxRetries) {
+ process = cmd.execute()
+ code_3 = process.waitFor()
+ err_3 = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())))
+ out_3 = process.getText()
+
+ // If the command was successful, break the loop
+ if (code_3 == 0) {
+ break
+ }
+
+ // If the command was not successful, increment the retry count, sleep for a while and try again
+ retryCount++
+ sleep(sleepTime)
}
- assertEquals(code, 0)
- return out
+ assertEquals(code_3, 0)
+ logger.info("Get compaction status: code=" + code_3 + ", out=" + out_3)
+ return out_3
}
def waitForCompaction = { be_host, be_http_port, tablet_id ->
boolean running = true
@@ -261,9 +271,9 @@ suite("test_single_replica_compaction", "p2") {
// check rowsets
checkCompactionResult.call()
- // trigger master be to do base compaction
+ // trigger master be to do full compaction
assertTrue(triggerCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id],
- "base", tablet_id).contains("Success"));
+ "full", tablet_id).contains("Success"));
waitForCompaction(backendId_to_backendIP[master_backend_id], backendId_to_backendHttpPort[master_backend_id], tablet_id)
// trigger follower be to fetch compaction result
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/21: [refactor](nereids) New distribute planner (#36531)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5b68de56cd1a48517c823320579786e3454b1a3d
Author: 924060929 <92...@qq.com>
AuthorDate: Fri Jun 28 17:11:24 2024 +0800
[refactor](nereids) New distribute planner (#36531)
## Proposed changes
The legacy coordinator act not only scheduler but also distribute planner. The code is so complex to understand, and hard to extend, and exist many limitations.
This pr extract and refine the computation of degree of parallel(dop) to a new DistributePlanner and resolve the limitations.
## How to use this function
This function only use for nereids + pipelinex, and current only support query statement, and non cloud mode.
Open this session variables to use this function:
```sql
set enable_nereids_distribute_planner=true; -- default is false
set enable_nereids_planner=true; -- default is true
```
## Core process and concepts
```
┌──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ │
│ ┌──────────────┐ ┌───────────────┐ ┌───────────────────┐ ┌─────────────────────────┐ │
│ Translate │ │ Typed │ │ Assign │ │ Wrap │ │ │
│ ──────────► │ PlanFragment │ ──────► │ UnassignedJob │ ───────► │ StaticAssignedJob │ ─────► │ PipelineDistributedPlan │ │
│ │ │ │ │ │ │ │ │ │
│ └──────────────┘ └───────────────┘ └───────────────────┘ └─────────────────────────┘ │
│ │
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
│ │
│ │
└──────────────────────────────────────────────────┐ ┌─────────────────────────────────────────────────────┘
│ │
│ │
┌──────────────┐ ┌─────────────────┐ ┌───────────────────┐
│ │ Distribute │ │ AdHoc │ │
│ PhysicalPlan │ ───────────► │ DistributedPlan │ ──────► │ PipelineScheduler │
│ │ │ │ │ │
└──────────────┘ └─────────────────┘ └───────────────────┘
```
DistributePlanner is a new planner to compute dop and generate instances, it consume PlanFragment and do this tasks
1. Use PlanFragment to generate `UnassignedJob`, it's a **Typed Fragment**, decided how to calculate dop and how to select the datasource, but this fragment not yet assigned some backends and datasources. These are some unassignedJobs: UnassignedScanSingleOlapTableJob, UnassignedScanBucketOlapTableJob, UnassignedShuffleJob, UnassignedQueryConstantJob. Keep UnassignedJob different can decoupling unrelated logic, and easy to extend: just and a new type of UnassignedJob.
2. Use UnassignedJob to select datasource, compute dop, and generate `AssignedJob`, means a instance, which already assigned datasource and backend. There are StaticAssignedJob and LocalShuffleAssignedJob, we will add DynamicAssignedJob when support StageScheduler and adaptive query execution
3. Wrap PlanFragment, UnassignedJob and AssignedJob to `PipelineDistributedPlan`, the coordinator will consume the DistributedPlan and translate to TPlan and schedule instances
## Resolve limitations
**1. left table shuffle to right table**
if right table has distribution which distribute by `storage hash`, and left table has distribution which distribute by `compute hash`, we can shuffle left to right by `storage hash` to do shuffle bucket join, and keep right side not move.
```sql
select *
from
(
select id2
from test_shuffle_left
group by id2
) a
inner join [shuffle]
test_shuffle_left b
on a.id2=b.id;
| PhysicalResultSink[288] ( outputExprs=[id2#1, id#2, id2#3] ) ...
| +--PhysicalHashJoin[285]@4 ( type=INNER_JOIN, stats=3, hashCondition=[(id2#1 = id#2)], otherCondition=[], markCondition=[], hint=[shuffle] ) ...
| |--PhysicalDistribute[281]@2 ( stats=1.5, distributionSpec=DistributionSpecHash ( orderedShuffledColumns=[1], shuffleType=STORAGE_BUCKETED, tableId=-1, selectedIndexId=-1, partitionIds=...
| | +--PhysicalHashAggregate[278]@2 ( aggPhase=GLOBAL, aggMode=BUFFER_TO_RESULT, maybeUseStreaming=false, groupByExpr=[id2#1], outputExpr=[id2#1], partitionExpr=Optional[[id2#1]], requir...
| | +--PhysicalDistribute[275]@7 ( stats=1.5, distributionSpec=DistributionSpecHash ( orderedShuffledColumns=[1], shuffleType=EXECUTION_BUCKETED, tableId=-1, selectedIndexId=-1, parti...
| | +--PhysicalHashAggregate[272]@7 ( aggPhase=LOCAL, aggMode=INPUT_TO_BUFFER, maybeUseStreaming=true, groupByExpr=[id2#1], outputExpr=[id2#1], partitionExpr=Optional[[id2#1]], req...
| | +--PhysicalProject[269]@1 ( stats=3, projects=[id2#1] ) ...
| | +--PhysicalOlapScan[test_shuffle_left]@0 ( stats=3 ) ...
| +--PhysicalOlapScan[test_shuffle_left]@3 ( stats=3 )
```
**2. support colocate union numbers function**
support use one instance to union/join numbers, note this plan no any PhysicalDistribute plan:
```sql
explain physical plan
select * from numbers('number'='3')a
union all
select * from numbers('number'='4')b
PhysicalResultSink[98] ( outputExprs=[number#2] )
+--PhysicalUnion@ ( qualifier=ALL, outputs=[number#2], regularChildrenOutputs=[[number#0], [number#1]], constantExprsList=[], stats=7 )
|--PhysicalTVFRelation ( qualified=NumbersTableValuedFunction, output=[number#0], function=numbers('number' = '3') )
+--PhysicalTVFRelation ( qualified=NumbersTableValuedFunction, output=[number#1], function=numbers('number' = '4') )
```
**3. support bucket prune with right outer bucket shuffle join**
left table prune some buckets, say [bucket 1, bucket 3]
we should process the right bucket shuffle join like this
```
[
(left bucket 1) right outer join (exchange right table which should process by bucket 1),
(empty bucket) right outer join (exchange right table which should process by bucket 2),
(left bucket 3) right outer join (exchange right table which should process by bucket 3)
]
```
the left bucket 2 is pruned, so right table can not shuffle to left, because the left instance not exists, so bucket 2 will return empty rows and wrong.
new DistributePlanner can fill up this instance.
the case:
```sql
explain physical plan
SELECT * FROM
(select * from test_outer_join1 where c0 =1)a
RIGHT OUTER JOIN
(select * from test_outer_join2)b
ON a.c0 = b.c0
```
### New feature
add an explain statement to show distributed plans
```sql
explain distributed plan select ...
```
for example, you can use this function to check how many instances generated, how many bytes the instance will scan, which backend will process the instance:
```sql
MySQL root@127.0.0.1:test> explain distributed plan select * from test_shuffle_left2 a join [shuffle] test_shuffle_left2 b on a.id2=b.id;
Explain String(Nereids Planner)
-------------------------------------------------------------------------------------------------------
PipelineDistributedPlan(
id: 0,
parallel: 2,
fragmentJob: UnassignedScanSingleOlapTableJob,
fragment: {
OUTPUT EXPRS:
id[#8]
id2[#9]
id[#10]
id2[#11]
PARTITION: HASH_PARTITIONED: id2[#3]
HAS_COLO_PLAN_NODE: false
VRESULT SINK
MYSQL_PROTOCAL
3:VHASH JOIN(152)
| join op: INNER JOIN(PARTITIONED)[]
| equal join conjunct: (id2[#3] = id[#0])
| cardinality=3
| vec output tuple id: 3
| output tuple id: 3
| vIntermediate tuple ids: 2
| hash output slot ids: 0 1 2 3
| isMarkJoin: false
| final projections: id[#4], id2[#5], id[#6], id2[#7]
| final project output tuple id: 3
| distribute expr lists: id2[#3]
| distribute expr lists: id[#0]
| tuple ids: 1 0
|
|----0:VOlapScanNode(149)
| TABLE: test.test_shuffle_left2(test_shuffle_left2), PREAGGREGATION: ON
| partitions=1/1 (test_shuffle_left2)
| tablets=10/10, tabletList=22038,22040,22042 ...
| cardinality=3, avgRowSize=0.0, numNodes=1
| pushAggOp=NONE
| tuple ids: 0
|
2:VEXCHANGE
offset: 0
distribute expr lists: id[#2]
tuple ids: 1
},
instanceJobs: [
LocalShuffleAssignedJob(
index: 0,
worker: BackendWorker(id: 10095, address: 192.168.126.1:9050),
shareScanIndex: 0,
scanSource: [
{
scanNode: OlapScanNode{id=0, tid=0, tblName=test_shuffle_left2, keyRanges=, preds= limit=-1},
scanRanges: ScanRanges(bytes: 400, ranges: [
tablet 22038, bytes: 0,
tablet 22042, bytes: 0,
tablet 22046, bytes: 0,
tablet 22050, bytes: 400,
tablet 22054, bytes: 0
])
}
]
),
LocalShuffleAssignedJob(
index: 1,
worker: BackendWorker(id: 10096, address: 192.168.126.2:9051),
shareScanIndex: 1,
scanSource: [
{
scanNode: OlapScanNode{id=0, tid=0, tblName=test_shuffle_left2, keyRanges=, preds= limit=-1},
scanRanges: ScanRanges(bytes: 796, ranges: [
tablet 22040, bytes: 397,
tablet 22044, bytes: 0,
tablet 22048, bytes: 399,
tablet 22052, bytes: 0,
tablet 22056, bytes: 0
])
}
]
)
]
)
PipelineDistributedPlan(
id: 1,
parallel: 2,
fragmentJob: UnassignedScanSingleOlapTableJob,
fragment: {
PARTITION: HASH_PARTITIONED: id[#2]
HAS_COLO_PLAN_NODE: false
STREAM DATA SINK
EXCHANGE ID: 02
HASH_PARTITIONED: id2[#3]
1:VOlapScanNode(145)
TABLE: test.test_shuffle_left2(test_shuffle_left2), PREAGGREGATION: ON
partitions=1/1 (test_shuffle_left2)
tablets=10/10, tabletList=22038,22040,22042 ...
cardinality=3, avgRowSize=0.0, numNodes=1
pushAggOp=NONE
tuple ids: 1
},
instanceJobs: [
LocalShuffleAssignedJob(
index: 0,
worker: BackendWorker(id: 10095, address: 192.168.126.1:9050),
shareScanIndex: 0,
scanSource: [
{
scanNode: OlapScanNode{id=1, tid=1, tblName=test_shuffle_left2, keyRanges=, preds= limit=-1},
scanRanges: ScanRanges(bytes: 400, ranges: [
tablet 22038, bytes: 0,
tablet 22042, bytes: 0,
tablet 22046, bytes: 0,
tablet 22050, bytes: 400,
tablet 22054, bytes: 0
])
}
]
),
LocalShuffleAssignedJob(
index: 1,
worker: BackendWorker(id: 10096, address: 192.168.126.2:9051),
shareScanIndex: 1,
scanSource: [
{
scanNode: OlapScanNode{id=1, tid=1, tblName=test_shuffle_left2, keyRanges=, preds= limit=-1},
scanRanges: ScanRanges(bytes: 796, ranges: [
tablet 22040, bytes: 397,
tablet 22044, bytes: 0,
tablet 22048, bytes: 399,
tablet 22052, bytes: 0,
tablet 22056, bytes: 0
])
}
]
)
]
)
Hint log:
Used: [shuffle]_2
UnUsed:
SyntaxError:
```
## TODO
1. extract PipelineScheduler from Coordinator
2. move this framework into cascades and compute cost by dop
3. support StageScheduler, adaptive query execution and DynamicAssignedJob
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../src/main/java/org/apache/doris/common/Id.java | 7 +-
.../java/org/apache/doris/common/TreeNode.java | 25 ++
.../org/apache/doris/common/profile/Profile.java | 10 +
.../doris/common/profile/SummaryProfile.java | 13 +
.../org/apache/doris/nereids/NereidsPlanner.java | 60 +++-
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +
.../properties/ChildrenPropertiesRegulator.java | 52 ++--
.../functions/table/TableValuedFunction.java | 4 +
.../doris/nereids/trees/plans/PlaceholderId.java | 7 +-
.../trees/plans/commands/ExplainCommand.java | 1 +
.../plans/distribute/BucketSpecifyInstances.java | 30 ++
.../plans/distribute/DefaultSpecifyInstances.java | 30 ++
.../trees/plans/distribute/DistributePlanner.java | 66 ++++
.../trees/plans/distribute/DistributedPlan.java | 57 ++++
.../trees/plans/distribute/FragmentIdMapping.java | 71 +++++
.../plans/distribute/NereidsSpecifyInstances.java | 59 ++++
.../plans/distribute/PipelineDistributedPlan.java | 72 +++++
.../BackendDistributedPlanWorkerManager.java | 63 ++++
.../plans/distribute/worker/BackendWorker.java | 74 +++++
.../distribute/worker/DistributedPlanWorker.java | 40 +++
.../worker/DistributedPlanWorkerManager.java | 25 ++
.../worker/LoadBalanceScanWorkerSelector.java | 336 +++++++++++++++++++++
.../distribute/worker/ScanWorkerSelector.java | 62 ++++
.../plans/distribute/worker/WorkerScanRanges.java | 33 ++
.../trees/plans/distribute/worker/Workload.java | 22 ++
.../worker/job/AbstractUnassignedJob.java | 74 +++++
.../worker/job/AbstractUnassignedScanJob.java | 202 +++++++++++++
.../plans/distribute/worker/job/AssignedJob.java | 39 +++
.../distribute/worker/job/AssignedJobBuilder.java | 64 ++++
.../distribute/worker/job/BucketScanSource.java | 148 +++++++++
.../distribute/worker/job/CustomAssignmentJob.java | 29 ++
.../distribute/worker/job/DefaultScanSource.java | 111 +++++++
.../worker/job/LocalShuffleAssignedJob.java | 43 +++
.../plans/distribute/worker/job/ScanRange.java | 23 ++
.../plans/distribute/worker/job/ScanRanges.java | 109 +++++++
.../plans/distribute/worker/job/ScanSource.java | 41 +++
.../plans/distribute/worker/job/Splittable.java | 69 +++++
.../distribute/worker/job/StaticAssignedJob.java | 106 +++++++
.../UnassignedGatherScanMultiRemoteTablesJob.java | 87 ++++++
.../plans/distribute/worker/job/UnassignedJob.java | 52 ++++
.../worker/job/UnassignedJobBuilder.java | 266 ++++++++++++++++
.../worker/job/UnassignedQueryConstantJob.java | 50 +++
.../job/UnassignedScanBucketOlapTableJob.java | 310 +++++++++++++++++++
.../job/UnassignedScanSingleOlapTableJob.java | 82 +++++
.../job/UnassignedScanSingleRemoteTableJob.java | 53 ++++
.../worker/job/UnassignedShuffleJob.java | 124 ++++++++
.../worker/job/UnassignedSpecifyInstancesJob.java | 49 +++
.../worker/job/UninstancedScanSource.java | 36 +++
.../distribute/worker/job/WorkerScanSource.java | 31 ++
.../org/apache/doris/nereids/util/JoinUtils.java | 22 +-
.../java/org/apache/doris/nereids/util/Utils.java | 14 +
.../org/apache/doris/planner/DataGenScanNode.java | 4 +
.../org/apache/doris/planner/OlapScanNode.java | 37 ++-
.../org/apache/doris/planner/PlanFragment.java | 25 ++
.../java/org/apache/doris/planner/PlanNode.java | 25 ++
.../java/org/apache/doris/planner/ScanNode.java | 9 +-
.../java/org/apache/doris/qe/ConnectContext.java | 6 +
.../main/java/org/apache/doris/qe/Coordinator.java | 51 ++--
.../org/apache/doris/qe/NereidsCoordinator.java | 194 ++++++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 54 +++-
.../java/org/apache/doris/qe/StmtExecutor.java | 12 +-
.../org/apache/doris/qe/HmsQueryCacheTest.java | 14 +-
.../distribute/colocate_union_numbers.out | 10 +
.../nereids_syntax_p0/distribute/local_shuffle.out | 36 +++
.../prune_bucket_with_bucket_shuffle_join.out | 5 +
.../distribute/query_constant.out | 12 +
.../data/nereids_syntax_p0/distribute/shuffle.out | 8 +
.../distribute/shuffle_left_join.out | 9 +
.../test_forbid_unknown_col_stats.groovy | 2 +-
.../distribute/colocate_union_numbers.groovy | 70 +++++
.../distribute/local_shuffle.groovy | 187 ++++++++++++
.../prune_bucket_with_bucket_shuffle_join.groovy | 98 ++++++
.../distribute/query_constant.groovy | 39 +++
.../nereids_syntax_p0/distribute/shuffle.groovy | 34 +++
.../distribute/shuffle_left_join.groovy | 117 +++++++
76 files changed, 4438 insertions(+), 77 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index ee9b91fdf6f..05a675997fc 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -262,6 +262,7 @@ planType
| OPTIMIZED | PHYSICAL // same type
| SHAPE
| MEMO
+ | DISTRIBUTED
| ALL // default type
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Id.java b/fe/fe-core/src/main/java/org/apache/doris/common/Id.java
index 9d6dad50a46..a6bd3896708 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Id.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Id.java
@@ -25,7 +25,7 @@ import java.util.ArrayList;
/**
* Integer ids that cannot accidentally be compared with ints.
*/
-public class Id<IdType extends Id<IdType>> {
+public class Id<IdType extends Id<IdType>> implements Comparable<Id<IdType>> {
protected final int id;
public Id(int id) {
@@ -62,4 +62,9 @@ public class Id<IdType extends Id<IdType>> {
public String toString() {
return Integer.toString(id);
}
+
+ @Override
+ public int compareTo(Id<IdType> idTypeId) {
+ return id - idTypeId.id;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
index 8070505798b..b254dd9a1d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java
@@ -249,4 +249,29 @@ public class TreeNode<NodeType extends TreeNode<NodeType>> {
child.foreach(func);
}
}
+
+ /** anyMatch */
+ public boolean anyMatch(Predicate<TreeNode<? extends NodeType>> func) {
+ if (func.apply(this)) {
+ return true;
+ }
+
+ for (NodeType child : children) {
+ if (child.anyMatch(func)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** foreachDown */
+ public void foreachDown(Predicate<TreeNode<NodeType>> visitor) {
+ if (!visitor.test(this)) {
+ return;
+ }
+
+ for (TreeNode<NodeType> child : getChildren()) {
+ child.foreachDown(visitor);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
index 96cecd19ec6..b45527f60bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java
@@ -20,6 +20,8 @@ package org.apache.doris.common.profile;
import org.apache.doris.common.util.ProfileManager;
import org.apache.doris.common.util.RuntimeProfile;
import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.planner.Planner;
@@ -108,6 +110,14 @@ public class Profile {
}
summaryInfo.put(SummaryProfile.PHYSICAL_PLAN,
builder.toString().replace("\n", "\n "));
+
+ FragmentIdMapping<DistributedPlan> distributedPlans = nereidsPlanner.getDistributedPlans();
+ if (distributedPlans != null) {
+ summaryInfo.put(SummaryProfile.DISTRIBUTED_PLAN,
+ DistributedPlan.toString(Lists.newArrayList(distributedPlans.values()))
+ .replace("\n", "\n ")
+ );
+ }
}
summaryProfile.update(summaryInfo);
for (ExecutionProfile executionProfile : executionProfiles) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
index 8a7119cb2ce..2e686171238 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java
@@ -54,6 +54,7 @@ public class SummaryProfile {
public static final String TRACE_ID = "Trace ID";
public static final String WORKLOAD_GROUP = "Workload Group";
public static final String PHYSICAL_PLAN = "Physical Plan";
+ public static final String DISTRIBUTED_PLAN = "Distributed Plan";
// Execution Summary
public static final String EXECUTION_SUMMARY_PROFILE_NAME = "Execution Summary";
public static final String ANALYSIS_TIME = "Analysis Time";
@@ -86,6 +87,7 @@ public class SummaryProfile {
public static final String NEREIDS_REWRITE_TIME = "Nereids Rewrite Time";
public static final String NEREIDS_OPTIMIZE_TIME = "Nereids Optimize Time";
public static final String NEREIDS_TRANSLATE_TIME = "Nereids Translate Time";
+ public static final String NEREIDS_DISTRIBUTE_TIME = "Nereids Distribute Time";
public static final String FRAGMENT_COMPRESSED_SIZE = "Fragment Compressed Size";
public static final String FRAGMENT_RPC_COUNT = "Fragment RPC Count";
@@ -109,6 +111,7 @@ public class SummaryProfile {
public static final ImmutableList<String> SUMMARY_KEYS = new ImmutableList.Builder<String>()
.addAll(SUMMARY_CAPTIONS)
.add(PHYSICAL_PLAN)
+ .add(DISTRIBUTED_PLAN)
.build();
// The display order of execution summary items.
@@ -199,6 +202,7 @@ public class SummaryProfile {
private long nereidsRewriteFinishTime = -1;
private long nereidsOptimizeFinishTime = -1;
private long nereidsTranslateFinishTime = -1;
+ private long nereidsDistributeFinishTime = -1;
// timestamp of query begin
private long queryBeginTime = -1;
// Analysis end time
@@ -315,6 +319,7 @@ public class SummaryProfile {
executionSummaryProfile.addInfoString(NEREIDS_REWRITE_TIME, getPrettyNereidsRewriteTime());
executionSummaryProfile.addInfoString(NEREIDS_OPTIMIZE_TIME, getPrettyNereidsOptimizeTime());
executionSummaryProfile.addInfoString(NEREIDS_TRANSLATE_TIME, getPrettyNereidsTranslateTime());
+ executionSummaryProfile.addInfoString(NEREIDS_DISTRIBUTE_TIME, getPrettyNereidsDistributeTime());
executionSummaryProfile.addInfoString(ANALYSIS_TIME,
getPrettyTime(queryAnalysisFinishTime, queryBeginTime, TUnit.TIME_MS));
executionSummaryProfile.addInfoString(PLAN_TIME,
@@ -419,6 +424,10 @@ public class SummaryProfile {
this.nereidsTranslateFinishTime = TimeUtils.getStartTimeMs();
}
+ public void setNereidsDistributeTime() {
+ this.nereidsDistributeFinishTime = TimeUtils.getStartTimeMs();
+ }
+
public void setQueryBeginTime() {
this.queryBeginTime = TimeUtils.getStartTimeMs();
}
@@ -654,6 +663,10 @@ public class SummaryProfile {
return getPrettyTime(nereidsTranslateFinishTime, nereidsOptimizeFinishTime, TUnit.TIME_MS);
}
+ public String getPrettyNereidsDistributeTime() {
+ return getPrettyTime(nereidsDistributeFinishTime, nereidsTranslateFinishTime, TUnit.TIME_MS);
+ }
+
private String getPrettyGetPartitionVersionTime() {
if (getPartitionVersionTime == 0) {
return "N/A";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 829cf6512d6..e0d6f2f7589 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -52,6 +52,9 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
+import org.apache.doris.nereids.trees.plans.distribute.DistributePlanner;
+import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
@@ -70,6 +73,7 @@ import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.qe.ResultSetMetaData;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.cache.CacheAnalyzer;
import com.google.common.annotations.VisibleForTesting;
@@ -102,6 +106,7 @@ public class NereidsPlanner extends Planner {
private Plan rewrittenPlan;
private Plan optimizedPlan;
private PhysicalPlan physicalPlan;
+ private FragmentIdMapping<DistributedPlan> distributedPlans;
// The cost of optimized plan
private double cost = 0;
private LogicalPlanAdapter logicalPlanAdapter;
@@ -130,17 +135,18 @@ public class NereidsPlanner extends Planner {
LogicalPlan parsedPlan = logicalPlanAdapter.getLogicalPlan();
NereidsTracer.logImportantTime("EndParsePlan");
setParsedPlan(parsedPlan);
+
PhysicalProperties requireProperties = buildInitRequireProperties();
statementContext.getStopwatch().start();
boolean showPlanProcess = showPlanProcess(queryStmt.getExplainOptions());
Plan resultPlan = plan(parsedPlan, requireProperties, explainLevel, showPlanProcess);
statementContext.getStopwatch().stop();
setOptimizedPlan(resultPlan);
- if (explainLevel.isPlanLevel) {
- return;
+
+ if (resultPlan instanceof PhysicalPlan) {
+ physicalPlan = (PhysicalPlan) resultPlan;
+ distribute(physicalPlan, explainLevel);
}
- physicalPlan = (PhysicalPlan) resultPlan;
- translate(physicalPlan);
}
@VisibleForTesting
@@ -315,7 +321,7 @@ public class NereidsPlanner extends Planner {
}
}
- private void translate(PhysicalPlan resultPlan) throws UserException {
+ private void splitFragments(PhysicalPlan resultPlan) throws UserException {
if (resultPlan instanceof PhysicalSqlCache) {
return;
}
@@ -360,6 +366,27 @@ public class NereidsPlanner extends Planner {
ScanNode.setVisibleVersionForOlapScanNodes(getScanNodes());
}
+ private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) throws UserException {
+ boolean canUseNereidsDistributePlanner = SessionVariable.canUseNereidsDistributePlanner();
+ if ((!canUseNereidsDistributePlanner && explainLevel.isPlanLevel)) {
+ return;
+ } else if ((canUseNereidsDistributePlanner && explainLevel.isPlanLevel
+ && (explainLevel != ExplainLevel.ALL_PLAN && explainLevel != ExplainLevel.DISTRIBUTED_PLAN))) {
+ return;
+ }
+
+ splitFragments(physicalPlan);
+
+ if (!canUseNereidsDistributePlanner) {
+ return;
+ }
+
+ distributedPlans = new DistributePlanner(fragments).plan();
+ if (statementContext.getConnectContext().getExecutor() != null) {
+ statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsDistributeTime();
+ }
+ }
+
private PhysicalPlan postProcess(PhysicalPlan physicalPlan) {
return new PlanPostProcessors(cascadesContext).process(physicalPlan);
}
@@ -498,6 +525,17 @@ public class NereidsPlanner extends Planner {
+ "\n\n========== MATERIALIZATIONS ==========\n"
+ materializationStringBuilder;
break;
+ case DISTRIBUTED_PLAN:
+ StringBuilder distributedPlanStringBuilder = new StringBuilder();
+
+ distributedPlanStringBuilder.append("========== DISTRIBUTED PLAN ==========\n");
+ if (distributedPlans == null || distributedPlans.isEmpty()) {
+ plan = "Distributed plan not generated, please set enable_nereids_distribute_planner "
+ + "and enable_pipeline_x_engine to true";
+ } else {
+ plan += DistributedPlan.toString(Lists.newArrayList(distributedPlans.values())) + "\n\n";
+ }
+ break;
case ALL_PLAN:
plan = "========== PARSED PLAN "
+ getTimeMetricString(SummaryProfile::getPrettyParseSqlTime) + " ==========\n"
@@ -510,7 +548,13 @@ public class NereidsPlanner extends Planner {
+ rewrittenPlan.treeString() + "\n\n"
+ "========== OPTIMIZED PLAN "
+ getTimeMetricString(SummaryProfile::getPrettyNereidsOptimizeTime) + " ==========\n"
- + optimizedPlan.treeString();
+ + optimizedPlan.treeString() + "\n\n";
+
+ if (distributedPlans != null && !distributedPlans.isEmpty()) {
+ plan += "========== DISTRIBUTED PLAN "
+ + getTimeMetricString(SummaryProfile::getPrettyNereidsDistributeTime) + " ==========\n";
+ plan += DistributedPlan.toString(Lists.newArrayList(distributedPlans.values())) + "\n\n";
+ }
break;
default:
plan = super.getExplainString(explainOptions)
@@ -681,6 +725,10 @@ public class NereidsPlanner extends Planner {
return physicalPlan;
}
+ public FragmentIdMapping<DistributedPlan> getDistributedPlans() {
+ return distributedPlans;
+ }
+
public LogicalPlanAdapter getLogicalPlanAdapter() {
return logicalPlanAdapter;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index ac4f25a7ab7..b6f679da5e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -3451,6 +3451,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
if (planTypeContext.MEMO() != null) {
return ExplainLevel.MEMO_PLAN;
}
+ if (planTypeContext.DISTRIBUTED() != null) {
+ return ExplainLevel.DISTRIBUTED_PLAN;
+ }
return ExplainLevel.ALL_PLAN;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
index 3beed014aac..f4029b3e7d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
@@ -46,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.JoinUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -213,12 +214,12 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
|| joinType == JoinType.FULL_OUTER_JOIN);
boolean isSpecInScope = (leftHashSpec.getShuffleType() == ShuffleType.NATURAL
|| rightHashSpec.getShuffleType() == ShuffleType.NATURAL);
- return isJoinTypeInScope && isSpecInScope;
+ return isJoinTypeInScope && isSpecInScope && !SessionVariable.canUseNereidsDistributePlanner();
}
@Override
- public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends Plan> hashJoin,
- Void context) {
+ public Boolean visitPhysicalHashJoin(
+ PhysicalHashJoin<? extends Plan, ? extends Plan> hashJoin, Void context) {
Preconditions.checkArgument(children.size() == 2, "children.size() != 2");
Preconditions.checkArgument(childrenProperties.size() == 2);
Preconditions.checkArgument(requiredProperties.size() == 2);
@@ -303,13 +304,24 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
(DistributionSpecHash) requiredProperties.get(1).getDistributionSpec()));
} else if (leftHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED
&& rightHashSpec.getShuffleType() == ShuffleType.NATURAL) {
- // TODO: we must do shuffle on right because coordinator could not do right be selection in this case,
- // since it always to check the left most node whether olap scan node.
- // after we fix coordinator problem, we could do right to left bucket shuffle
- updatedForRight = Optional.of(calAnotherSideRequired(
- ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec,
- (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(),
- (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec()));
+ if (SessionVariable.canUseNereidsDistributePlanner()) {
+ // nereids coordinator can exchange left side to right side to do bucket shuffle join
+ // TODO: maybe we should check if left child is PhysicalDistribute.
+ // If so add storage bucketed shuffle on left side. Other wise,
+ // add execution bucketed shuffle on right side.
+ updatedForLeft = Optional.of(calAnotherSideRequired(
+ ShuffleType.STORAGE_BUCKETED, rightHashSpec, leftHashSpec,
+ (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec(),
+ (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec()));
+ } else {
+ // legacy coordinator could not do right be selection in this case,
+ // since it always to check the left most node whether olap scan node.
+ // so we can only shuffle right to left side to do normal shuffle join
+ updatedForRight = Optional.of(calAnotherSideRequired(
+ ShuffleType.EXECUTION_BUCKETED, leftHashSpec, rightHashSpec,
+ (DistributionSpecHash) requiredProperties.get(0).getDistributionSpec(),
+ (DistributionSpecHash) requiredProperties.get(1).getDistributionSpec()));
+ }
} else if (leftHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED
&& rightHashSpec.getShuffleType() == ShuffleType.EXECUTION_BUCKETED) {
if (bothSideShuffleKeysAreSameOrder(rightHashSpec, leftHashSpec,
@@ -537,20 +549,20 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
* calAnotherSideRequiredShuffleIds's comment.
*
* @param shuffleType real output shuffle type
- * @param notShuffleSideOutput not shuffle side real output used hash spec
- * @param shuffleSideOutput shuffle side real output used hash spec
- * @param notShuffleSideRequired not shuffle side required used hash spec
- * @param shuffleSideRequired shuffle side required hash spec
+ * @param notNeedShuffleSideOutput not shuffle side real output used hash spec
+ * @param needShuffleSideOutput shuffle side real output used hash spec
+ * @param notNeedShuffleSideRequired not shuffle side required used hash spec
+ * @param needShuffleSideRequired shuffle side required hash spec
* @return shuffle side new required hash spec
*/
private PhysicalProperties calAnotherSideRequired(ShuffleType shuffleType,
- DistributionSpecHash notShuffleSideOutput, DistributionSpecHash shuffleSideOutput,
- DistributionSpecHash notShuffleSideRequired, DistributionSpecHash shuffleSideRequired) {
- List<ExprId> shuffleSideIds = calAnotherSideRequiredShuffleIds(notShuffleSideOutput,
- notShuffleSideRequired, shuffleSideRequired);
+ DistributionSpecHash notNeedShuffleSideOutput, DistributionSpecHash needShuffleSideOutput,
+ DistributionSpecHash notNeedShuffleSideRequired, DistributionSpecHash needShuffleSideRequired) {
+ List<ExprId> shuffleSideIds = calAnotherSideRequiredShuffleIds(notNeedShuffleSideOutput,
+ notNeedShuffleSideRequired, needShuffleSideRequired);
return new PhysicalProperties(new DistributionSpecHash(shuffleSideIds, shuffleType,
- shuffleSideOutput.getTableId(), shuffleSideOutput.getSelectedIndexId(),
- shuffleSideOutput.getPartitionIds()));
+ needShuffleSideOutput.getTableId(), needShuffleSideOutput.getSelectedIndexId(),
+ needShuffleSideOutput.getPartitionIds()));
}
private void updateChildEnforceAndCost(int index, PhysicalProperties targetProperties) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
index 26602435651..fda53020598 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
@@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -113,6 +114,9 @@ public abstract class TableValuedFunction extends BoundFunction
}
public PhysicalProperties getPhysicalProperties() {
+ if (SessionVariable.canUseNereidsDistributePlanner()) {
+ return PhysicalProperties.ANY;
+ }
return PhysicalProperties.STORAGE_ANY;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java
index be3cb645fe6..f1d410100e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java
@@ -23,7 +23,7 @@ import org.apache.doris.common.IdGenerator;
/**
* placeholder id for prepared statement parameters
*/
-public class PlaceholderId extends Id<PlaceholderId> implements Comparable<PlaceholderId> {
+public class PlaceholderId extends Id<PlaceholderId> {
public PlaceholderId(int id) {
super(id);
@@ -55,9 +55,4 @@ public class PlaceholderId extends Id<PlaceholderId> implements Comparable<Place
public int hashCode() {
return super.hashCode();
}
-
- @Override
- public int compareTo(PlaceholderId o) {
- return this.id - o.id;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index 500f1d4a09a..03278723d34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -49,6 +49,7 @@ public class ExplainCommand extends Command implements NoForward {
OPTIMIZED_PLAN(true),
SHAPE_PLAN(true),
MEMO_PLAN(true),
+ DISTRIBUTED_PLAN(true),
ALL_PLAN(true)
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/BucketSpecifyInstances.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/BucketSpecifyInstances.java
new file mode 100644
index 00000000000..24c0ddf5509
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/BucketSpecifyInstances.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.WorkerScanSource;
+
+import java.util.List;
+
+/** DefaultNereidsSpecifyInstances */
+public class BucketSpecifyInstances extends NereidsSpecifyInstances<BucketScanSource> {
+ public BucketSpecifyInstances(List<WorkerScanSource<BucketScanSource>> workerScanSources) {
+ super(workerScanSources);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DefaultSpecifyInstances.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DefaultSpecifyInstances.java
new file mode 100644
index 00000000000..75d4b24c2d4
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DefaultSpecifyInstances.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.WorkerScanSource;
+
+import java.util.List;
+
+/** DefaultSpecifyInstances */
+public class DefaultSpecifyInstances extends NereidsSpecifyInstances<DefaultScanSource> {
+ public DefaultSpecifyInstances(List<WorkerScanSource<DefaultScanSource>> workerToScanSources) {
+ super(workerToScanSources);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
new file mode 100644
index 00000000000..ceef281c0fc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJobBuilder;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJobBuilder;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanFragmentId;
+
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+/** DistributePlanner */
+public class DistributePlanner {
+ private final List<PlanFragment> fragments;
+ private final FragmentIdMapping<PlanFragment> idToFragments;
+
+ public DistributePlanner(List<PlanFragment> fragments) {
+ this.fragments = Objects.requireNonNull(fragments, "fragments can not be null");
+ this.idToFragments = FragmentIdMapping.buildFragmentMapping(fragments);
+ }
+
+ public FragmentIdMapping<DistributedPlan> plan() {
+ FragmentIdMapping<UnassignedJob> fragmentJobs = UnassignedJobBuilder.buildJobs(idToFragments);
+ ListMultimap<PlanFragmentId, AssignedJob> instanceJobs = AssignedJobBuilder.buildJobs(fragmentJobs);
+ return buildDistributePlans(fragmentJobs, instanceJobs);
+ }
+
+ private FragmentIdMapping<DistributedPlan> buildDistributePlans(
+ Map<PlanFragmentId, UnassignedJob> idToUnassignedJobs,
+ ListMultimap<PlanFragmentId, AssignedJob> idToAssignedJobs) {
+ FragmentIdMapping<PipelineDistributedPlan> idToDistributedPlans = new FragmentIdMapping<>();
+ for (Entry<PlanFragmentId, PlanFragment> kv : idToFragments.entrySet()) {
+ PlanFragmentId fragmentId = kv.getKey();
+ PlanFragment fragment = kv.getValue();
+
+ UnassignedJob fragmentJob = idToUnassignedJobs.get(fragmentId);
+ List<AssignedJob> instanceJobs = idToAssignedJobs.get(fragmentId);
+
+ List<PipelineDistributedPlan> childrenPlans = idToDistributedPlans.getByChildrenFragments(fragment);
+ idToDistributedPlans.put(fragmentId, new PipelineDistributedPlan(fragmentJob, instanceJobs, childrenPlans));
+ }
+ return (FragmentIdMapping) idToDistributedPlans;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java
new file mode 100644
index 00000000000..8f176e3cacc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.AbstractTreeNode;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.nereids.util.Utils;
+
+import java.util.List;
+import java.util.Objects;
+
+/** DistributedPlan */
+@lombok.Getter
+public abstract class DistributedPlan extends AbstractTreeNode<DistributedPlan> {
+ protected final UnassignedJob fragmentJob;
+ protected final List<DistributedPlan> inputs;
+
+ public DistributedPlan(UnassignedJob fragmentJob, List<? extends DistributedPlan> inputs) {
+ this.fragmentJob = Objects.requireNonNull(fragmentJob, "fragmentJob can not be null");
+ this.inputs = Utils.fastToImmutableList(Objects.requireNonNull(inputs, "inputs can not be null"));
+ }
+
+ @Override
+ public DistributedPlan withChildren(List<DistributedPlan> children) {
+ throw new UnsupportedOperationException();
+ }
+
+ public abstract String toString(int displayFragmentId);
+
+ /** toString */
+ public static String toString(List<DistributedPlan> distributedPlansBottomToTop) {
+ StringBuilder distributedPlanStringBuilder = new StringBuilder();
+ int fragmentDisplayId = 0;
+ for (int i = distributedPlansBottomToTop.size() - 1; i >= 0; i--) {
+ DistributedPlan distributedPlan = distributedPlansBottomToTop.get(i);
+ distributedPlanStringBuilder
+ .append(distributedPlan.toString(fragmentDisplayId++))
+ .append("\n");
+ }
+ return distributedPlanStringBuilder.toString();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java
new file mode 100644
index 00000000000..95bf36051d2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanFragmentId;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+/**
+ * FragmentIdMapping:
+ * key: PlanFragmentId
+ * value: T
+ *
+ * NOTE: this map should order by PlanFragmentId asc
+ */
+public class FragmentIdMapping<T> extends TreeMap<PlanFragmentId, T> {
+ public FragmentIdMapping() {
+ }
+
+ public FragmentIdMapping(Comparator<? super PlanFragmentId> comparator) {
+ super(comparator);
+ }
+
+ public FragmentIdMapping(Map<? extends PlanFragmentId, ? extends T> m) {
+ super(m);
+ }
+
+ public FragmentIdMapping(SortedMap<PlanFragmentId, ? extends T> m) {
+ super(m);
+ }
+
+ /** getByChildrenFragments */
+ public List<T> getByChildrenFragments(PlanFragment fragment) {
+ List<PlanFragment> children = fragment.getChildren();
+ ImmutableList.Builder<T> values = ImmutableList.builderWithExpectedSize(children.size());
+ for (PlanFragment child : children) {
+ values.add(get(child.getFragmentId()));
+ }
+ return values.build();
+ }
+
+ public static FragmentIdMapping<PlanFragment> buildFragmentMapping(List<PlanFragment> fragments) {
+ FragmentIdMapping<PlanFragment> idToFragments = new FragmentIdMapping<>();
+ for (PlanFragment fragment : fragments) {
+ idToFragments.put(fragment.getFragmentId(), fragment);
+ }
+ return idToFragments;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/NereidsSpecifyInstances.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/NereidsSpecifyInstances.java
new file mode 100644
index 00000000000..58a564d3f87
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/NereidsSpecifyInstances.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.StaticAssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.WorkerScanSource;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+import java.util.Objects;
+
+/** NereidsSpecifyInstances */
+public abstract class NereidsSpecifyInstances<S extends ScanSource> {
+ public final List<WorkerScanSource<S>> workerScanSources;
+
+ public NereidsSpecifyInstances(List<WorkerScanSource<S>> workerScanSources) {
+ this.workerScanSources = Objects.requireNonNull(workerScanSources,
+ "workerScanSources can not be null");
+ }
+
+ /** buildAssignedJobs */
+ public List<AssignedJob> buildAssignedJobs(UnassignedJob unassignedJob) {
+ List<AssignedJob> instances = Lists.newArrayListWithCapacity(workerScanSources.size());
+ int instanceNum = 0;
+ ConnectContext context = ConnectContext.get();
+ for (WorkerScanSource<S> workerToScanSource : workerScanSources) {
+ TUniqueId instanceId = context.nextInstanceId();
+ DistributedPlanWorker worker = workerToScanSource.worker;
+ ScanSource scanSource = workerToScanSource.scanSource;
+ StaticAssignedJob assignedJob = new StaticAssignedJob(
+ instanceNum++, instanceId, unassignedJob, worker, scanSource
+ );
+ instances.add(assignedJob);
+ }
+ return instances;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java
new file mode 100644
index 00000000000..13f903d8a2c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TExplainLevel;
+
+import java.util.List;
+import java.util.Objects;
+
+/** PipelineDistributedPlan */
+public class PipelineDistributedPlan extends DistributedPlan {
+ protected final List<AssignedJob> instanceJobs;
+
+ public PipelineDistributedPlan(
+ UnassignedJob fragmentJob,
+ List<AssignedJob> instanceJobs,
+ List<PipelineDistributedPlan> inputs) {
+ super(fragmentJob, inputs);
+ this.instanceJobs = Utils.fastToImmutableList(
+ Objects.requireNonNull(instanceJobs, "instanceJobs can not be null")
+ );
+ }
+
+ public List<AssignedJob> getInstanceJobs() {
+ return instanceJobs;
+ }
+
+ @Override
+ public String toString(int displayFragmentId) {
+ StringBuilder instancesStr = new StringBuilder();
+ for (int i = 0; i < instanceJobs.size(); i++) {
+ instancesStr.append(instanceJobs.get(i).toString(false));
+ if (i + 1 < instanceJobs.size()) {
+ instancesStr.append(",\n");
+ }
+ }
+ String instancesStrWithIndent = Utils.addLinePrefix(instancesStr.toString(), " ");
+
+ String explainString = Utils.addLinePrefix(
+ fragmentJob.getFragment().getExplainString(TExplainLevel.VERBOSE).trim(), " "
+ );
+
+ return "PipelineDistributedPlan(\n"
+ + " id: " + displayFragmentId + ",\n"
+ + " parallel: " + instanceJobs.size() + ",\n"
+ + " fragmentJob: " + fragmentJob + ",\n"
+ + " fragment: {\n"
+ + " " + explainString + "\n"
+ + " },\n"
+ + " instanceJobs: [\n" + instancesStrWithIndent + "\n"
+ + " ]\n"
+ + ")";
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java
new file mode 100644
index 00000000000..190d6d898a9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.NereidsException;
+import org.apache.doris.common.Reference;
+import org.apache.doris.qe.SimpleScheduler;
+import org.apache.doris.system.Backend;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.function.Supplier;
+
+/** BackendWorkerManager */
+public class BackendDistributedPlanWorkerManager implements DistributedPlanWorkerManager {
+ private final Supplier<ImmutableMap<Long, Backend>> backends = Suppliers.memoize(() -> {
+ try {
+ return Env.getCurrentSystemInfo().getBackendsWithIdByCurrentCluster();
+ } catch (Exception t) {
+ throw new NereidsException("Can not get backends: " + t, t);
+ }
+ });
+
+ @Override
+ public DistributedPlanWorker getWorker(long backendId) {
+ ImmutableMap<Long, Backend> backends = this.backends.get();
+ Backend backend = backends.get(backendId);
+ if (backend == null) {
+ throw new IllegalStateException("Backend " + backendId + " is not exist");
+ }
+ return new BackendWorker(backend);
+ }
+
+ @Override
+ public DistributedPlanWorker randomAvailableWorker() {
+ try {
+ Reference<Long> selectedBackendId = new Reference<>();
+ ImmutableMap<Long, Backend> backends = this.backends.get();
+ SimpleScheduler.getHost(backends, selectedBackendId);
+ Backend selctedBackend = backends.get(selectedBackendId.getRef());
+ return new BackendWorker(selctedBackend);
+ } catch (Exception t) {
+ throw new NereidsException("Can not get backends: " + t, t);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java
new file mode 100644
index 00000000000..702a00dd358
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+import org.apache.doris.system.Backend;
+
+import java.util.Objects;
+
+/** BackendWorker */
+public class BackendWorker implements DistributedPlanWorker {
+ private final Backend backend;
+
+ public BackendWorker(Backend backend) {
+ this.backend = backend;
+ }
+
+ @Override
+ public long id() {
+ return backend.getId();
+ }
+
+ @Override
+ public String address() {
+ return backend.getAddress();
+ }
+
+ @Override
+ public String host() {
+ return backend.getHost();
+ }
+
+ @Override
+ public int port() {
+ return backend.getBePort();
+ }
+
+ @Override
+ public boolean available() {
+ return backend.isQueryAvailable();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(backend.getId());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof BackendWorker)) {
+ return false;
+ }
+ return backend.getId() == ((BackendWorker) obj).backend.getId();
+ }
+
+ @Override
+ public String toString() {
+ return "BackendWorker(id: " + backend.getId() + ", address: " + backend.getAddress() + ")";
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java
new file mode 100644
index 00000000000..c86675a6dab
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorker.java
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+/**
+ * DistributedPlanWorker: a worker who can execute the assigned job(instance) of the DistributedPlan
+ */
+public interface DistributedPlanWorker extends Comparable<DistributedPlanWorker> {
+ long id();
+
+ // ipv4/ipv6 address
+ String address();
+
+ String host();
+
+ int port();
+
+ // whether is this worker alive?
+ boolean available();
+
+ @Override
+ default int compareTo(DistributedPlanWorker worker) {
+ return address().compareTo(worker.address());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
new file mode 100644
index 00000000000..f0604d13e34
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+/** DistributedPlanWorkerManager */
+public interface DistributedPlanWorkerManager {
+ DistributedPlanWorker getWorker(long backendId);
+
+ DistributedPlanWorker randomAvailableWorker();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java
new file mode 100644
index 00000000000..60021281181
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanRanges;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedScanBucketOlapTableJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UninstancedScanSource;
+import org.apache.doris.planner.DataPartition;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.thrift.TExternalScanRange;
+import org.apache.doris.thrift.TFileRangeDesc;
+import org.apache.doris.thrift.TFileScanRange;
+import org.apache.doris.thrift.TPaloScanRange;
+import org.apache.doris.thrift.TScanRange;
+import org.apache.doris.thrift.TScanRangeLocation;
+import org.apache.doris.thrift.TScanRangeLocations;
+import org.apache.doris.thrift.TScanRangeParams;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.BiFunction;
+
+/** LoadBalanceScanWorkerSelector */
+public class LoadBalanceScanWorkerSelector implements ScanWorkerSelector {
+ private final BackendDistributedPlanWorkerManager workerManager = new BackendDistributedPlanWorkerManager();
+ private final Map<DistributedPlanWorker, WorkerWorkload> workloads = Maps.newLinkedHashMap();
+
+ @Override
+ public DistributedPlanWorkerManager getWorkerManager() {
+ return workerManager;
+ }
+
+ @Override
+ public DistributedPlanWorker selectMinWorkloadWorker(List<DistributedPlanWorker> workers) {
+ DistributedPlanWorker minWorkloadWorker = null;
+ WorkerWorkload minWorkload = new WorkerWorkload(Integer.MAX_VALUE, Long.MAX_VALUE);
+ for (DistributedPlanWorker worker : workers) {
+ WorkerWorkload workload = getWorkload(worker);
+ if (minWorkload.compareTo(workload) > 0) {
+ minWorkloadWorker = worker;
+ minWorkload = workload;
+ }
+ }
+ minWorkload.recordOneScanTask(1);
+ return minWorkloadWorker;
+ }
+
+ @Override
+ public Map<DistributedPlanWorker, UninstancedScanSource> selectReplicaAndWorkerWithoutBucket(ScanNode scanNode) {
+ Map<DistributedPlanWorker, UninstancedScanSource> workerScanRanges = Maps.newLinkedHashMap();
+ // allScanRangesLocations is all scan ranges in all partition which need to scan
+ List<TScanRangeLocations> allScanRangesLocations = scanNode.getScanRangeLocations(0);
+ for (TScanRangeLocations onePartitionOneScanRangeLocation : allScanRangesLocations) {
+ // usually, the onePartitionOneScanRangeLocation is a tablet in one partition
+ long bytes = getScanRangeSize(scanNode, onePartitionOneScanRangeLocation);
+
+ WorkerScanRanges assigned = selectScanReplicaAndMinWorkloadWorker(
+ onePartitionOneScanRangeLocation, bytes);
+ UninstancedScanSource scanRanges = workerScanRanges.computeIfAbsent(
+ assigned.worker,
+ w -> new UninstancedScanSource(
+ new DefaultScanSource(ImmutableMap.of(scanNode, new ScanRanges()))
+ )
+ );
+ DefaultScanSource scanSource = (DefaultScanSource) scanRanges.scanSource;
+ scanSource.scanNodeToScanRanges.get(scanNode).addScanRanges(assigned.scanRanges);
+ }
+ return workerScanRanges;
+ }
+
+ @Override
+ public Map<DistributedPlanWorker, UninstancedScanSource> selectReplicaAndWorkerWithBucket(
+ UnassignedScanBucketOlapTableJob unassignedJob) {
+ PlanFragment fragment = unassignedJob.getFragment();
+ List<ScanNode> scanNodes = unassignedJob.getScanNodes();
+ List<OlapScanNode> olapScanNodes = unassignedJob.getOlapScanNodes();
+
+ BiFunction<ScanNode, Integer, List<TScanRangeLocations>> bucketScanRangeSupplier = bucketScanRangeSupplier();
+ Function<ScanNode, Map<Integer, Long>> bucketBytesSupplier = bucketBytesSupplier();
+ // all are olap scan nodes
+ if (!scanNodes.isEmpty() && scanNodes.size() == olapScanNodes.size()) {
+ if (olapScanNodes.size() == 1 && fragment.hasBucketShuffleJoin()) {
+ return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier);
+ } else if (fragment.hasColocatePlanNode()) {
+ return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier);
+ }
+ } else if (olapScanNodes.isEmpty() && fragment.getDataPartition() == DataPartition.UNPARTITIONED) {
+ return selectForBucket(unassignedJob, scanNodes, bucketScanRangeSupplier, bucketBytesSupplier);
+ }
+ throw new IllegalStateException(
+ "Illegal bucket shuffle join or colocate join in fragment: " + fragment.getFragmentId()
+ );
+ }
+
+ private BiFunction<ScanNode, Integer, List<TScanRangeLocations>> bucketScanRangeSupplier() {
+ return (scanNode, bucketIndex) -> {
+ if (scanNode instanceof OlapScanNode) {
+ return (List) ((OlapScanNode) scanNode).bucketSeq2locations.get(bucketIndex);
+ } else {
+ // the backend is selected by XxxScanNode.createScanRangeLocations()
+ return scanNode.getScanRangeLocations(0);
+ }
+ };
+ }
+
+ private Function<ScanNode, Map<Integer, Long>> bucketBytesSupplier() {
+ return scanNode -> {
+ if (scanNode instanceof OlapScanNode) {
+ return ((OlapScanNode) scanNode).bucketSeq2Bytes;
+ } else {
+ // not supported yet
+ return ImmutableMap.of(0, 0L);
+ }
+ };
+ }
+
+ private Map<DistributedPlanWorker, UninstancedScanSource> selectForBucket(
+ UnassignedJob unassignedJob, List<ScanNode> scanNodes,
+ BiFunction<ScanNode, Integer, List<TScanRangeLocations>> bucketScanRangeSupplier,
+ Function<ScanNode, Map<Integer, Long>> bucketBytesSupplier) {
+ Map<DistributedPlanWorker, UninstancedScanSource> assignment = Maps.newLinkedHashMap();
+
+ Map<Integer, Long> bucketIndexToBytes = computeEachBucketScanBytes(scanNodes, bucketBytesSupplier);
+
+ for (Entry<Integer, Long> kv : bucketIndexToBytes.entrySet()) {
+ Integer bucketIndex = kv.getKey();
+ long allScanNodeScanBytesInOneBucket = kv.getValue();
+
+ DistributedPlanWorker selectedWorker = null;
+ for (ScanNode scanNode : scanNodes) {
+ List<TScanRangeLocations> allPartitionTabletsInOneBucketInOneTable
+ = bucketScanRangeSupplier.apply(scanNode, bucketIndex);
+ if (!allPartitionTabletsInOneBucketInOneTable.isEmpty()) {
+ WorkerScanRanges replicaAndWorker = selectScanReplicaAndMinWorkloadWorker(
+ allPartitionTabletsInOneBucketInOneTable.get(0), allScanNodeScanBytesInOneBucket);
+ selectedWorker = replicaAndWorker.worker;
+ break;
+ }
+ // else: the bucket is pruned, we should use another ScanNode to select worker for this bucket
+ }
+ if (selectedWorker == null) {
+ throw new IllegalStateException("Can not assign worker for bucket: " + bucketIndex);
+ }
+
+ long workerId = selectedWorker.id();
+ for (ScanNode scanNode : scanNodes) {
+ List<TScanRangeLocations> allPartitionTabletsInOneBucket
+ = bucketScanRangeSupplier.apply(scanNode, bucketIndex);
+ List<Pair<TScanRangeParams, Long>> selectedReplicasInOneBucket = filterReplicaByWorkerInBucket(
+ scanNode, workerId, bucketIndex, allPartitionTabletsInOneBucket
+ );
+ UninstancedScanSource bucketIndexToScanNodeToTablets
+ = assignment.computeIfAbsent(
+ selectedWorker,
+ worker -> new UninstancedScanSource(new BucketScanSource(Maps.newLinkedHashMap()))
+ );
+ BucketScanSource scanSource = (BucketScanSource) bucketIndexToScanNodeToTablets.scanSource;
+ Map<ScanNode, ScanRanges> scanNodeToScanRanges = scanSource.bucketIndexToScanNodeToTablets
+ .computeIfAbsent(bucketIndex, bucket -> Maps.newLinkedHashMap());
+ ScanRanges scanRanges = scanNodeToScanRanges.computeIfAbsent(scanNode, node -> new ScanRanges());
+ for (Pair<TScanRangeParams, Long> replica : selectedReplicasInOneBucket) {
+ TScanRangeParams replicaParam = replica.first;
+ Long scanBytes = replica.second;
+ scanRanges.addScanRange(replicaParam, scanBytes);
+ }
+ }
+ }
+ return assignment;
+ }
+
+ private WorkerScanRanges selectScanReplicaAndMinWorkloadWorker(
+ TScanRangeLocations tabletLocation, long tabletBytes) {
+ List<TScanRangeLocation> replicaLocations = tabletLocation.getLocations();
+ int replicaNum = replicaLocations.size();
+ WorkerWorkload minWorkload = new WorkerWorkload(Integer.MAX_VALUE, Long.MAX_VALUE);
+ DistributedPlanWorker minWorkLoadWorker = null;
+ TScanRangeLocation selectedReplicaLocation = null;
+
+ for (int i = 0; i < replicaNum; i++) {
+ TScanRangeLocation replicaLocation = replicaLocations.get(i);
+ DistributedPlanWorker worker = workerManager.getWorker(replicaLocation.getBackendId());
+ if (!worker.available()) {
+ continue;
+ }
+
+ WorkerWorkload workload = getWorkload(worker);
+ if (workload.compareTo(minWorkload) < 0) {
+ minWorkLoadWorker = worker;
+ minWorkload = workload;
+ selectedReplicaLocation = replicaLocation;
+ }
+ }
+ if (minWorkLoadWorker == null) {
+ throw new AnalysisException("No available workers");
+ } else {
+ minWorkload.recordOneScanTask(tabletBytes);
+ ScanRanges scanRanges = new ScanRanges();
+ TScanRangeParams scanReplicaParams =
+ ScanWorkerSelector.buildScanReplicaParams(tabletLocation, selectedReplicaLocation);
+ scanRanges.addScanRange(scanReplicaParams, tabletBytes);
+ return new WorkerScanRanges(minWorkLoadWorker, scanRanges);
+ }
+ }
+
+ private List<Pair<TScanRangeParams, Long>> filterReplicaByWorkerInBucket(
+ ScanNode scanNode, long filterWorkerId, int bucketIndex,
+ List<TScanRangeLocations> allPartitionTabletsInOneBucket) {
+ List<Pair<TScanRangeParams, Long>> selectedReplicasInOneBucket = Lists.newArrayList();
+ for (TScanRangeLocations onePartitionOneTabletLocation : allPartitionTabletsInOneBucket) {
+ TScanRange scanRange = onePartitionOneTabletLocation.getScanRange();
+ if (scanRange.getPaloScanRange() != null) {
+ long tabletId = scanRange.getPaloScanRange().getTabletId();
+ boolean foundTabletInThisWorker = false;
+ for (TScanRangeLocation replicaLocation : onePartitionOneTabletLocation.getLocations()) {
+ if (replicaLocation.getBackendId() == filterWorkerId) {
+ TScanRangeParams scanReplicaParams = ScanWorkerSelector.buildScanReplicaParams(
+ onePartitionOneTabletLocation, replicaLocation);
+ Long replicaSize = ((OlapScanNode) scanNode).getTabletSingleReplicaSize(tabletId);
+ selectedReplicasInOneBucket.add(Pair.of(scanReplicaParams, replicaSize));
+ foundTabletInThisWorker = true;
+ break;
+ }
+ }
+ if (!foundTabletInThisWorker) {
+ throw new IllegalStateException(
+ "Can not find tablet " + tabletId + " in the bucket: " + bucketIndex);
+ }
+ } else if (onePartitionOneTabletLocation.getLocations().size() == 1) {
+ TScanRangeLocation replicaLocation = onePartitionOneTabletLocation.getLocations().get(0);
+ TScanRangeParams scanReplicaParams = ScanWorkerSelector.buildScanReplicaParams(
+ onePartitionOneTabletLocation, replicaLocation);
+ Long replicaSize = 0L;
+ selectedReplicasInOneBucket.add(Pair.of(scanReplicaParams, replicaSize));
+ } else {
+ throw new IllegalStateException("Unsupported");
+ }
+ }
+ return selectedReplicasInOneBucket;
+ }
+
+ private Map<Integer, Long> computeEachBucketScanBytes(
+ List<ScanNode> scanNodes, Function<ScanNode, Map<Integer, Long>> bucketBytesSupplier) {
+ Map<Integer, Long> bucketIndexToBytes = Maps.newLinkedHashMap();
+ for (ScanNode scanNode : scanNodes) {
+ Map<Integer, Long> bucketSeq2Bytes = bucketBytesSupplier.apply(scanNode);
+ for (Entry<Integer, Long> bucketSeq2Byte : bucketSeq2Bytes.entrySet()) {
+ Integer bucketIndex = bucketSeq2Byte.getKey();
+ Long scanBytes = bucketSeq2Byte.getValue();
+ bucketIndexToBytes.merge(bucketIndex, scanBytes, Long::sum);
+ }
+ }
+ return bucketIndexToBytes;
+ }
+
+ private WorkerWorkload getWorkload(DistributedPlanWorker worker) {
+ return workloads.computeIfAbsent(worker, w -> new WorkerWorkload());
+ }
+
+ private long getScanRangeSize(ScanNode scanNode, TScanRangeLocations scanRangeLocations) {
+ TScanRange scanRange = scanRangeLocations.getScanRange();
+ TPaloScanRange paloScanRange = scanRange.getPaloScanRange();
+ if (paloScanRange != null) {
+ long tabletId = paloScanRange.getTabletId();
+ Long tabletBytes = ((OlapScanNode) scanNode).getTabletSingleReplicaSize(tabletId);
+ return tabletBytes == null ? 0L : tabletBytes;
+ }
+
+ TExternalScanRange extScanRange = scanRange.getExtScanRange();
+ if (extScanRange != null) {
+ TFileScanRange fileScanRange = extScanRange.getFileScanRange();
+ long size = 0;
+ for (TFileRangeDesc range : fileScanRange.getRanges()) {
+ size += range.getSize();
+ }
+ return size;
+ }
+
+ return 0L;
+ }
+
+ private static class WorkerWorkload implements Comparable<WorkerWorkload> {
+ private int taskNum;
+ private long scanBytes;
+
+ public WorkerWorkload() {
+ this(0, 0);
+ }
+
+ public WorkerWorkload(int taskNum, long scanBytes) {
+ this.taskNum = taskNum;
+ this.scanBytes = scanBytes;
+ }
+
+ public void recordOneScanTask(long scanBytes) {
+ this.scanBytes += scanBytes;
+ }
+
+ // order by scanBytes asc, taskNum asc
+ @Override
+ public int compareTo(WorkerWorkload workerWorkload) {
+ int compareScanBytes = Long.compare(this.scanBytes, workerWorkload.scanBytes);
+ if (compareScanBytes != 0) {
+ return compareScanBytes;
+ }
+ return taskNum - workerWorkload.taskNum;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java
new file mode 100644
index 00000000000..40876a09e44
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedScanBucketOlapTableJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UninstancedScanSource;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.thrift.TScanRangeLocation;
+import org.apache.doris.thrift.TScanRangeLocations;
+import org.apache.doris.thrift.TScanRangeParams;
+
+import java.util.List;
+import java.util.Map;
+
+/** ScanWorkerSelector */
+public interface ScanWorkerSelector {
+ DistributedPlanWorkerManager getWorkerManager();
+
+ DistributedPlanWorker selectMinWorkloadWorker(List<DistributedPlanWorker> workers);
+
+ // for a scan node, select replica for each scan range(denote tablet if the ScanNode is OlapScanNode),
+ // use the replica location to build a worker execute the instance
+ Map<DistributedPlanWorker, UninstancedScanSource> selectReplicaAndWorkerWithoutBucket(ScanNode scanNode);
+
+ // return
+ // key: Worker, the backend which will process this fragment
+ // value.key: Integer, the bucket index, from 0 to (bucket_num - 1)
+ // for example, create table statement contains: distributed by hash(id) buckets 10,
+ // the bucket index will from 0 to 9
+ // value.value.key: ScanNode, which ScanNode the worker will process scan task
+ // value.value.value: ScanRanges, the tablets in current bucket,
+ // for example, colocate table `tbl` has 2 range partitions:
+ // p1 values[(1), (10)) and p2 values[(10), 11) with integer partition column part,
+ // and distributed by hash(id) buckets 10. And, so, there has 10 buckets from bucket 0 to
+ // bucket 9, and every bucket contains two tablets, because there are two partitions.
+ Map<DistributedPlanWorker, UninstancedScanSource> selectReplicaAndWorkerWithBucket(
+ UnassignedScanBucketOlapTableJob unassignedJob);
+
+ static TScanRangeParams buildScanReplicaParams(
+ TScanRangeLocations tabletLocation, TScanRangeLocation replicaLocation) {
+ TScanRangeParams replicaParam = new TScanRangeParams();
+ replicaParam.scan_range = tabletLocation.scan_range;
+ // Volume is optional, so we need to set the value and the is-set bit
+ replicaParam.setVolumeId(replicaLocation.volume_id);
+ return replicaParam;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/WorkerScanRanges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/WorkerScanRanges.java
new file mode 100644
index 00000000000..25daa29bd87
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/WorkerScanRanges.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanRanges;
+
+import java.util.Objects;
+
+/** WorkerScanRange */
+public class WorkerScanRanges {
+ public final DistributedPlanWorker worker;
+ public final ScanRanges scanRanges;
+
+ public WorkerScanRanges(DistributedPlanWorker worker, ScanRanges scanRanges) {
+ this.worker = Objects.requireNonNull(worker, "scanRangeParams can not be null");
+ this.scanRanges = Objects.requireNonNull(scanRanges, "scanRanges can not be null");
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/Workload.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/Workload.java
new file mode 100644
index 00000000000..faf6ec1c5e8
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/Workload.java
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker;
+
+/** Workload */
+public interface Workload extends Comparable<Workload> {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java
new file mode 100644
index 00000000000..f53ee614523
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.AbstractTreeNode;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+import java.util.Objects;
+
+/** AbstractUnassignedJob */
+public abstract class AbstractUnassignedJob
+ extends AbstractTreeNode<UnassignedJob> implements UnassignedJob {
+ protected final PlanFragment fragment;
+ protected final List<ScanNode> scanNodes;
+ protected final ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob;
+
+ /** AbstractUnassignedJob */
+ public AbstractUnassignedJob(PlanFragment fragment, List<ScanNode> scanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob) {
+ super(Utils.fastToImmutableList(exchangeToChildJob.values()));
+ this.fragment = Objects.requireNonNull(fragment, "fragment can not be null");
+ this.scanNodes = Utils.fastToImmutableList(
+ Objects.requireNonNull(scanNodes, "scanNodes can not be null")
+ );
+ this.exchangeToChildJob
+ = Objects.requireNonNull(exchangeToChildJob, "exchangeToChildJob can not be null");
+ }
+
+ @Override
+ public PlanFragment getFragment() {
+ return fragment;
+ }
+
+ @Override
+ public List<ScanNode> getScanNodes() {
+ return scanNodes;
+ }
+
+ @Override
+ public ListMultimap<ExchangeNode, UnassignedJob> getExchangeToChildJob() {
+ return exchangeToChildJob;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+ @Override
+ public UnassignedJob withChildren(List<UnassignedJob> children) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
new file mode 100644
index 00000000000..92bb059122f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** AbstractUnassignedScanJob */
+public abstract class AbstractUnassignedScanJob extends AbstractUnassignedJob {
+ protected final AtomicInteger shareScanIdGenerator = new AtomicInteger();
+
+ public AbstractUnassignedScanJob(PlanFragment fragment,
+ List<ScanNode> scanNodes, ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob) {
+ super(fragment, scanNodes, exchangeToChildJob);
+ }
+
+ @Override
+ public List<AssignedJob> computeAssignedJobs(DistributedPlanWorkerManager workerManager,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanSource = multipleMachinesParallelization(
+ workerManager, inputJobs);
+
+ return insideMachineParallelization(workerToScanSource, inputJobs);
+ }
+
+ protected abstract Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs);
+
+ protected List<AssignedJob> insideMachineParallelization(
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+
+ ConnectContext context = ConnectContext.get();
+ boolean useLocalShuffleToAddParallel = useLocalShuffleToAddParallel(workerToScanRanges);
+ int instanceIndexInFragment = 0;
+ List<AssignedJob> instances = Lists.newArrayList();
+ for (Entry<DistributedPlanWorker, UninstancedScanSource> entry : workerToScanRanges.entrySet()) {
+ DistributedPlanWorker worker = entry.getKey();
+
+ // the scanRanges which this worker should scan,
+ // for example:
+ // {
+ // scan tbl1: [tablet_10001, tablet_10002, tablet_10003, tablet_10004] // no instances
+ // }
+ ScanSource scanSource = entry.getValue().scanSource;
+
+ // usually, its tablets num, or buckets num
+ int scanSourceMaxParallel = scanSource.maxParallel(scanNodes);
+
+ // now we should compute how many instances to process the data,
+ // for example: two instances
+ int instanceNum = degreeOfParallelism(scanSourceMaxParallel);
+
+ List<ScanSource> instanceToScanRanges;
+ if (useLocalShuffleToAddParallel) {
+ // only generate one instance to scan all data, in this step
+ instanceToScanRanges = scanSource.parallelize(
+ scanNodes, 1
+ );
+
+ // Some tablets too big, we need add parallel to process these tablets after scan,
+ // for example, use one OlapScanNode to scan data, and use some local instances
+ // to process Aggregation parallel. We call it `share scan`. Backend will know this
+ // instances share the same ScanSource, and will not scan same data multiple times.
+ //
+ // +-------------------------------- same fragment in one host -------------------------------------+
+ // | instance1 instance2 instance3 instance4 |
+ // | \ \ / / |
+ // | |
+ // | OlapScanNode |
+ // |(share scan node, and local shuffle data to other local instances to parallel compute this data)|
+ // +------------------------------------------------------------------------------------------------+
+ ScanSource shareScanSource = instanceToScanRanges.get(0);
+
+ // one scan range generate multiple instances,
+ // different instances reference the same scan source
+ int shareScanId = shareScanIdGenerator.getAndIncrement();
+ for (int i = 0; i < instanceNum; i++) {
+ LocalShuffleAssignedJob instance = new LocalShuffleAssignedJob(
+ instanceIndexInFragment++, shareScanId, context.nextInstanceId(),
+ this, worker, shareScanSource);
+ instances.add(instance);
+ }
+ } else {
+ // split the scanRanges to some partitions, one partition for one instance
+ // for example:
+ // [
+ // scan tbl1: [tablet_10001, tablet_10003], // instance 1
+ // scan tbl1: [tablet_10002, tablet_10004] // instance 2
+ // ]
+ instanceToScanRanges = scanSource.parallelize(
+ scanNodes, instanceNum
+ );
+
+ for (ScanSource instanceToScanRange : instanceToScanRanges) {
+ instances.add(
+ assignWorkerAndDataSources(
+ instanceIndexInFragment++, context.nextInstanceId(), worker, instanceToScanRange
+ )
+ );
+ }
+ }
+ }
+
+ return instances;
+ }
+
+ protected boolean useLocalShuffleToAddParallel(
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges) {
+ if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isForceToLocalShuffle()) {
+ return true;
+ }
+ return parallelTooLittle(workerToScanRanges);
+ }
+
+ protected boolean parallelTooLittle(Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges) {
+ if (this instanceof UnassignedScanBucketOlapTableJob) {
+ return scanRangesToLittle(workerToScanRanges) && bucketsTooLittle(workerToScanRanges);
+ } else if (this instanceof UnassignedScanSingleOlapTableJob
+ || this instanceof UnassignedScanSingleRemoteTableJob) {
+ return scanRangesToLittle(workerToScanRanges);
+ } else {
+ return false;
+ }
+ }
+
+ protected boolean scanRangesToLittle(
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges) {
+ ConnectContext context = ConnectContext.get();
+ int backendNum = workerToScanRanges.size();
+ for (ScanNode scanNode : scanNodes) {
+ if (!scanNode.ignoreStorageDataDistribution(context, backendNum)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ protected int degreeOfParallelism(int maxParallel) {
+ Preconditions.checkArgument(maxParallel > 0, "maxParallel must be positive");
+ if (!fragment.getDataPartition().isPartitioned()) {
+ return 1;
+ }
+ if (scanNodes.size() == 1 && scanNodes.get(0) instanceof OlapScanNode) {
+ OlapScanNode olapScanNode = (OlapScanNode) scanNodes.get(0);
+ // if the scan node have limit and no conjuncts, only need 1 instance to save cpu and mem resource,
+ // e.g. select * from tbl limit 10
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext != null && olapScanNode.shouldUseOneInstance(connectContext)) {
+ return 1;
+ }
+ }
+
+ // the scan instance num should not larger than the tablets num
+ return Math.min(maxParallel, Math.max(fragment.getParallelExecNum(), 1));
+ }
+
+ protected boolean bucketsTooLittle(Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges) {
+ int parallelExecNum = fragment.getParallelExecNum();
+ for (UninstancedScanSource uninstancedScanSource : workerToScanRanges.values()) {
+ ScanSource scanSource = uninstancedScanSource.scanSource;
+ if (scanSource instanceof BucketScanSource) {
+ BucketScanSource bucketScanSource = (BucketScanSource) scanSource;
+ int bucketNum = bucketScanSource.bucketIndexToScanNodeToTablets.size();
+ if (bucketNum >= parallelExecNum) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java
new file mode 100644
index 00000000000..f9f6b9dea14
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJob.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.thrift.TUniqueId;
+
+/**
+ * AssignedJob.
+ * for example: an instance job in a fragment job, which already assign to a worker and some data sources
+ */
+public interface AssignedJob {
+ int indexInUnassignedJob();
+
+ TUniqueId instanceId();
+
+ UnassignedJob unassignedJob();
+
+ DistributedPlanWorker getAssignedWorker();
+
+ ScanSource getScanSource();
+
+ String toString(boolean showUnassignedJob);
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
new file mode 100644
index 00000000000..aab6316a3ba
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragmentId;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** AssignedJobBuilder */
+public class AssignedJobBuilder {
+ /** buildJobs */
+ public static ListMultimap<PlanFragmentId, AssignedJob> buildJobs(
+ Map<PlanFragmentId, UnassignedJob> unassignedJobs) {
+ BackendDistributedPlanWorkerManager workerManager = new BackendDistributedPlanWorkerManager();
+ ListMultimap<PlanFragmentId, AssignedJob> allAssignedJobs = ArrayListMultimap.create();
+ for (Entry<PlanFragmentId, UnassignedJob> kv : unassignedJobs.entrySet()) {
+ PlanFragmentId fragmentId = kv.getKey();
+ UnassignedJob unassignedJob = kv.getValue();
+ ListMultimap<ExchangeNode, AssignedJob> inputAssignedJobs
+ = getInputAssignedJobs(unassignedJob, allAssignedJobs);
+ List<AssignedJob> fragmentAssignedJobs =
+ unassignedJob.computeAssignedJobs(workerManager, inputAssignedJobs);
+ allAssignedJobs.putAll(fragmentId, fragmentAssignedJobs);
+ }
+ return allAssignedJobs;
+ }
+
+ private static ListMultimap<ExchangeNode, AssignedJob> getInputAssignedJobs(
+ UnassignedJob unassignedJob, ListMultimap<PlanFragmentId, AssignedJob> assignedJobs) {
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs = ArrayListMultimap.create();
+ for (Entry<ExchangeNode, Collection<UnassignedJob>> exchangeNodeToChildJobs
+ : unassignedJob.getExchangeToChildJob().asMap().entrySet()) {
+ ExchangeNode exchangeNode = exchangeNodeToChildJobs.getKey();
+ Collection<UnassignedJob> childJobs = exchangeNodeToChildJobs.getValue();
+ for (UnassignedJob childJob : childJobs) {
+ inputJobs.putAll(exchangeNode, assignedJobs.get(childJob.getFragment().getFragmentId()));
+ }
+ }
+ return inputJobs;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java
new file mode 100644
index 00000000000..33d066a02b9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.common.util.ListUtil;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** BucketScanSource */
+public class BucketScanSource extends ScanSource {
+ // for example:
+ // 1. bucket 0 use OlapScanNode(tableName=`tbl`) to scan with tablet: [tablet 10001, tablet 10003]
+ // 2. bucket 1 use OlapScanNode(tableName=`tbl`) to scan with tablet: [tablet 10002, tablet 10004]
+ public final Map<Integer, Map<ScanNode, ScanRanges>> bucketIndexToScanNodeToTablets;
+
+ public BucketScanSource(Map<Integer, Map<ScanNode, ScanRanges>> bucketIndexToScanNodeToTablets) {
+ this.bucketIndexToScanNodeToTablets = bucketIndexToScanNodeToTablets;
+ }
+
+ @Override
+ public int maxParallel(List<ScanNode> scanNodes) {
+ // maxParallel is buckets num
+ return bucketIndexToScanNodeToTablets.size();
+ }
+
+ @Override
+ public List<ScanSource> parallelize(List<ScanNode> scanNodes, int instanceNum) {
+ // current state, no any instance, we only known how many buckets
+ // this worker should process, and the data that this buckets should process:
+ //
+ // [
+ // bucket 0: {
+ // scanNode1: ScanRanges([tablet_10001, tablet_10004, tablet_10007]),
+ // scanNode2: ScanRanges([tablet_10010, tablet_10013, tablet_10016])
+ // },
+ // bucket 1: {
+ // scanNode1: ScanRanges([tablet_10002, tablet_10005, tablet_10008]),
+ // scanNode2: ScanRanges([tablet_10011, tablet_10014, tablet_10017])
+ // },
+ // bucket 3: {
+ // scanNode1: ScanRanges([tablet_10003, tablet_10006, tablet_10009]),
+ // scanNode2: ScanRanges([tablet_10012, tablet_10015, tablet_10018])
+ // }
+ // ]
+ List<Entry<Integer, Map<ScanNode, ScanRanges>>> bucketIndexToScanRanges
+ = Lists.newArrayList(bucketIndexToScanNodeToTablets.entrySet());
+
+ // separate buckets to instanceNum groups.
+ // for example:
+ // [
+ // // instance 1 process two buckets
+ // [
+ // bucket 0: {
+ // scanNode1: ScanRanges([tablet_10001, tablet_10004, tablet_10007]),
+ // scanNode2: ScanRanges([tablet_10010, tablet_10013, tablet_10016])
+ // },
+ // bucket 3: {
+ // scanNode1: ScanRanges([tablet_10003, tablet_10006, tablet_10009]),
+ // scanNode2: ScanRanges([tablet_10012, tablet_10015, tablet_10018])
+ // }
+ // ],
+ // // instance 2 process one bucket
+ // [
+ // bucket 1: {
+ // scanNode1: ScanRanges([tablet_10002, tablet_10005, tablet_10008]),
+ // scanNode2: ScanRanges([tablet_10011, tablet_10014, tablet_10017])
+ // }
+ // ]
+ // ]
+ List<List<Entry<Integer, Map<ScanNode, ScanRanges>>>> scanBucketsPerInstance
+ = ListUtil.splitBySize(bucketIndexToScanRanges, instanceNum);
+
+ // rebuild BucketScanSource for each instance
+ ImmutableList.Builder<ScanSource> instancesScanSource = ImmutableList.builder();
+ for (List<Entry<Integer, Map<ScanNode, ScanRanges>>> oneInstanceScanBuckets : scanBucketsPerInstance) {
+ Map<Integer, Map<ScanNode, ScanRanges>> bucketsScanSources = Maps.newLinkedHashMap();
+ for (Entry<Integer, Map<ScanNode, ScanRanges>> bucketIndexToScanNodeToScanRange : oneInstanceScanBuckets) {
+ Integer bucketIndex = bucketIndexToScanNodeToScanRange.getKey();
+ Map<ScanNode, ScanRanges> scanNodeToScanRanges = bucketIndexToScanNodeToScanRange.getValue();
+ bucketsScanSources.put(bucketIndex, scanNodeToScanRanges);
+ }
+
+ instancesScanSource.add(new BucketScanSource(
+ bucketsScanSources
+ ));
+ }
+ return instancesScanSource.build();
+ }
+
+ /** getBucketIndexToScanRanges */
+ public Map<Integer, ScanRanges> getBucketIndexToScanRanges(ScanNode scanNode) {
+ Map<Integer, ScanRanges> bucketIndexToScanRanges = Maps.newLinkedHashMap();
+ for (Entry<Integer, Map<ScanNode, ScanRanges>> entry : bucketIndexToScanNodeToTablets.entrySet()) {
+ Integer bucketIndex = entry.getKey();
+ Map<ScanNode, ScanRanges> scanNodeToScanRanges = entry.getValue();
+ ScanRanges scanRanges = scanNodeToScanRanges.get(scanNode);
+ if (scanRanges != null) {
+ bucketIndexToScanRanges.put(bucketIndex, scanRanges);
+ }
+ }
+
+ return bucketIndexToScanRanges;
+ }
+
+ /** toString */
+ public void toString(StringBuilder str, String prefix) {
+ int i = 0;
+ String nextIndent = prefix + " ";
+ str.append("[\n");
+ for (Entry<Integer, Map<ScanNode, ScanRanges>> entry : bucketIndexToScanNodeToTablets.entrySet()) {
+ Integer bucketId = entry.getKey();
+ Map<ScanNode, ScanRanges> scanNodeToScanRanges = entry.getValue();
+ str.append(prefix).append(" bucket ").append(bucketId).append(": ");
+ DefaultScanSource.toString(scanNodeToScanRanges, str, nextIndent);
+ if (++i < bucketIndexToScanNodeToTablets.size()) {
+ str.append(",\n");
+ }
+ }
+ str.append("\n").append(prefix).append("]");
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return bucketIndexToScanNodeToTablets.isEmpty();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/CustomAssignmentJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/CustomAssignmentJob.java
new file mode 100644
index 00000000000..dc009dba09a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/CustomAssignmentJob.java
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.planner.ExchangeNode;
+
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+
+/** CustomAssignmentJob */
+public interface CustomAssignmentJob {
+ List<AssignedJob> customAssignment(ListMultimap<ExchangeNode, AssignedJob> inputJobs);
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java
new file mode 100644
index 00000000000..89e5270801d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** DefaultScanSource */
+public class DefaultScanSource extends ScanSource {
+ // for example:
+ // 1. use OlapScanNode(tableName=`tbl1`) to scan with tablet: [tablet 10001, tablet 10002]
+ // 2. use OlapScanNode(tableName=`tbl2`) to scan with tablet: [tablet 10003, tablet 10004]
+ public final Map<ScanNode, ScanRanges> scanNodeToScanRanges;
+
+ public DefaultScanSource(Map<ScanNode, ScanRanges> scanNodeToScanRanges) {
+ this.scanNodeToScanRanges = scanNodeToScanRanges;
+ }
+
+ @Override
+ public int maxParallel(List<ScanNode> scanNodes) {
+ if (scanNodes.size() == 1) {
+ ScanRanges scanRanges = scanNodeToScanRanges.get(scanNodes.get(0));
+ if (scanRanges != null) {
+ // max parallel is the scan ranges(tablets) num
+ return Math.max(scanRanges.params.size(), 1);
+ }
+ }
+ return 1;
+ }
+
+ @Override
+ public List<ScanSource> parallelize(List<ScanNode> scanNodes, int instanceNum) {
+ Preconditions.checkArgument(scanNodes.size() == 1,
+ "Only support parallelize one ScanNode, but meet " + scanNodes.size() + " ScanNodes");
+
+ ScanNode scanNode = scanNodes.get(0);
+ ScanRanges scanRanges = scanNodeToScanRanges.get(scanNode);
+ if (scanRanges == null) {
+ return ImmutableList.of();
+ }
+
+ List<ScanRanges> scanRangesPerInstance = scanRanges.split(instanceNum);
+
+ ImmutableList.Builder<ScanSource> instancesSource
+ = ImmutableList.builderWithExpectedSize(scanRangesPerInstance.size());
+ for (ScanRanges oneInstanceScanRanges : scanRangesPerInstance) {
+ DefaultScanSource oneInstanceScanSource
+ = new DefaultScanSource(ImmutableMap.of(scanNode, oneInstanceScanRanges));
+ instancesSource.add(oneInstanceScanSource);
+ }
+ return instancesSource.build();
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return scanNodeToScanRanges.isEmpty();
+ }
+
+ @Override
+ public void toString(StringBuilder str, String prefix) {
+ toString(scanNodeToScanRanges, str, prefix);
+ }
+
+ /** toString */
+ public static void toString(Map<ScanNode, ScanRanges> scanNodeToScanRanges, StringBuilder str, String prefix) {
+ if (scanNodeToScanRanges.isEmpty()) {
+ str.append("[]");
+ return;
+ }
+ int i = 0;
+ String nextIndent = prefix + " ";
+ str.append("[\n");
+ for (Entry<ScanNode, ScanRanges> entry : scanNodeToScanRanges.entrySet()) {
+ ScanNode scanNode = entry.getKey();
+ ScanRanges scanRanges = entry.getValue();
+ str.append(prefix).append(" {\n")
+ .append(prefix).append(" scanNode: ").append(scanNode).append(",\n")
+ .append(prefix).append(" scanRanges: ");
+
+ scanRanges.toString(str, nextIndent);
+ str.append("\n").append(prefix).append(" }");
+
+ if (++i < scanNodeToScanRanges.size()) {
+ str.append(",\n");
+ }
+ }
+ str.append("\n").append(prefix).append("]");
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
new file mode 100644
index 00000000000..50e43fc0282
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/** LocalShuffleAssignedJob */
+public class LocalShuffleAssignedJob extends StaticAssignedJob {
+ public final int shareScanId;
+
+ public LocalShuffleAssignedJob(
+ int indexInUnassignedJob, int shareScanId, TUniqueId instanceId,
+ UnassignedJob unassignedJob,
+ DistributedPlanWorker worker, ScanSource scanSource) {
+ super(indexInUnassignedJob, instanceId, unassignedJob, worker, scanSource);
+ this.shareScanId = shareScanId;
+ }
+
+ @Override
+ protected Map<String, String> extraInfo() {
+ return ImmutableMap.of("shareScanIndex", String.valueOf(shareScanId));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRange.java
new file mode 100644
index 00000000000..24bf7a6d910
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRange.java
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+/** ScanRange */
+public class ScanRange {
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRanges.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRanges.java
new file mode 100644
index 00000000000..2539b8416de
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRanges.java
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.thrift.TPaloScanRange;
+import org.apache.doris.thrift.TScanRangeParams;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**ScanRanges */
+public class ScanRanges implements Splittable<ScanRanges> {
+ // usually, it's tablets
+ public final List<TScanRangeParams> params;
+ // size corresponding to tablets one by one
+ public final List<Long> bytes;
+ public long totalBytes;
+
+ public ScanRanges() {
+ this(Lists.newArrayList(), Lists.newArrayList());
+ }
+
+ /** ScanRanges */
+ public ScanRanges(List<TScanRangeParams> params, List<Long> bytes) {
+ this.params = params;
+ this.bytes = bytes;
+ long totalBytes = 0;
+ for (Long size : bytes) {
+ totalBytes += size;
+ }
+ this.totalBytes = totalBytes;
+ }
+
+ public void addScanRanges(ScanRanges scanRanges) {
+ this.params.addAll(scanRanges.params);
+ this.bytes.addAll(scanRanges.bytes);
+ this.totalBytes += scanRanges.totalBytes;
+ }
+
+ public void addScanRange(TScanRangeParams params, long bytes) {
+ this.params.add(params);
+ this.bytes.add(bytes);
+ this.totalBytes += bytes;
+ }
+
+ @Override
+ public int itemSize() {
+ return params.size();
+ }
+
+ @Override
+ public void addItem(ScanRanges other, int index) {
+ addScanRange(other.params.get(index), other.bytes.get(index));
+ }
+
+ @Override
+ public ScanRanges newSplittable() {
+ return new ScanRanges();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder str = new StringBuilder();
+ toString(str, "");
+ return str.toString();
+ }
+
+ /** toString */
+ public void toString(StringBuilder str, String prefix) {
+ str.append("ScanRanges(bytes: " + totalBytes + ", ranges: [");
+ if (params.isEmpty()) {
+ str.append("])");
+ return;
+ }
+ str.append("\n");
+ for (int i = 0; i < params.size(); i++) {
+ str.append(prefix).append(" " + toString(params.get(i)) + ", bytes: " + bytes.get(i));
+ if (i + 1 < params.size()) {
+ str.append(",\n");
+ }
+ }
+ str.append("\n").append(prefix).append("])");
+ }
+
+ private String toString(TScanRangeParams scanRange) {
+ TPaloScanRange paloScanRange = scanRange.getScanRange().getPaloScanRange();
+ if (paloScanRange != null) {
+ return "tablet " + paloScanRange.getTabletId();
+ } else {
+ return scanRange.toString();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java
new file mode 100644
index 00000000000..b124e14bd73
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanSource.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.planner.ScanNode;
+
+import java.util.List;
+
+/** ScanSource */
+public abstract class ScanSource {
+
+ public abstract int maxParallel(List<ScanNode> scanNodes);
+
+ public abstract List<ScanSource> parallelize(List<ScanNode> scanNodes, int instanceNum);
+
+ public abstract boolean isEmpty();
+
+ public abstract void toString(StringBuilder str, String prefix);
+
+ @Override
+ public String toString() {
+ StringBuilder str = new StringBuilder();
+ toString(str, "");
+ return str.toString();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/Splittable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/Splittable.java
new file mode 100644
index 00000000000..269dcedfb86
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/Splittable.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import com.google.common.base.Preconditions;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Splittable */
+public interface Splittable<S extends Splittable<S>> {
+
+ int itemSize();
+
+ void addItem(S other, int index);
+
+ S newSplittable();
+
+ default List<S> split(int splitSize) {
+ return Splittable.split(this, splitSize);
+ }
+
+ /**
+ * split a list to multi expected number sublist
+ * for example:
+ *
+ * list is : [1, 2, 3, 4, 5, 6, 7]
+ * expectedSize is : 3
+ *
+ * return :
+ * [1, 4, 7]
+ * [2, 5]
+ * [3, 6]
+ */
+ static <S extends Splittable<S>> List<S> split(Splittable<S> splittable, int splitSize) {
+ Preconditions.checkNotNull(splittable, "splittable must not be null");
+ Preconditions.checkArgument(splitSize > 0, "splitSize must larger than 0");
+
+ int itemSize = splittable.itemSize();
+ splitSize = Math.min(splitSize, itemSize);
+
+ List<S> result = new ArrayList<>(splitSize);
+ for (int i = 0; i < splitSize; i++) {
+ result.add(splittable.newSplittable());
+ }
+
+ int index = 0;
+ for (int i = 0; i < itemSize; i++) {
+ result.get(index).addItem((S) splittable, i);
+ index = (index + 1) % splitSize;
+ }
+ return result;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java
new file mode 100644
index 00000000000..1a92cf71019
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+/** StaticAssignedJob */
+public class StaticAssignedJob implements AssignedJob {
+ private final int indexInUnassignedJob;
+ private final UnassignedJob unassignedJob;
+ private final TUniqueId instanceId;
+ private final DistributedPlanWorker worker;
+ private final ScanSource scanSource;
+
+ public StaticAssignedJob(
+ int indexInUnassignedJob, TUniqueId instanceId, UnassignedJob unassignedJob, DistributedPlanWorker worker,
+ ScanSource scanSource) {
+ this.indexInUnassignedJob = indexInUnassignedJob;
+ this.instanceId = Objects.requireNonNull(instanceId, "instanceId can not be null");
+ this.unassignedJob = Objects.requireNonNull(unassignedJob, "unassignedJob can not be null");
+ this.worker = worker;
+ this.scanSource = Objects.requireNonNull(scanSource, "scanSource can not be null");
+ }
+
+ @Override
+ public int indexInUnassignedJob() {
+ return indexInUnassignedJob;
+ }
+
+ @Override
+ public TUniqueId instanceId() {
+ return instanceId;
+ }
+
+ @Override
+ public UnassignedJob unassignedJob() {
+ return unassignedJob;
+ }
+
+ @Override
+ public DistributedPlanWorker getAssignedWorker() {
+ return worker;
+ }
+
+ @Override
+ public ScanSource getScanSource() {
+ return scanSource;
+ }
+
+ @Override
+ public String toString() {
+ return toString(true);
+ }
+
+ @Override
+ public String toString(boolean showUnassignedJob) {
+ StringBuilder scanSourceString = new StringBuilder();
+ if (!scanSource.isEmpty()) {
+ scanSource.toString(scanSourceString, " ");
+ } else {
+ scanSourceString = new StringBuilder("[]");
+ }
+ StringBuilder str = new StringBuilder(getClass().getSimpleName()).append("(");
+ if (showUnassignedJob) {
+ str.append("\n unassignedJob: ").append(unassignedJob).append(",");
+ }
+ str.append("\n index: " + indexInUnassignedJob)
+ .append(",\n instanceId: " + DebugUtil.printId(instanceId))
+ .append(",\n worker: " + worker);
+ for (Entry<String, String> kv : extraInfo().entrySet()) {
+ str.append(",\n ").append(kv.getKey()).append(": ").append(kv.getValue());
+ }
+
+ return str
+ .append(",\n scanSource: " + scanSourceString)
+ .append("\n)")
+ .toString();
+ }
+
+ protected Map<String, String> extraInfo() {
+ return ImmutableMap.of();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java
new file mode 100644
index 00000000000..558af884497
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.DataGenScanNode;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TScanRangeLocations;
+import org.apache.doris.thrift.TScanRangeParams;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+
+/** UnassignedGatherScanMultiRemoteTablesJob */
+public class UnassignedGatherScanMultiRemoteTablesJob extends AbstractUnassignedJob {
+
+ public UnassignedGatherScanMultiRemoteTablesJob(PlanFragment fragment,
+ List<ScanNode> scanNodes, ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob) {
+ super(fragment, scanNodes, exchangeToChildJob);
+ }
+
+ /** canApply */
+ public static boolean canApply(List<ScanNode> scanNodes) {
+ if (scanNodes.size() <= 1) {
+ return false;
+ }
+ for (ScanNode scanNode : scanNodes) {
+ if (!(scanNode instanceof DataGenScanNode)) {
+ return false;
+ }
+ DataGenScanNode dataGenScanNode = (DataGenScanNode) scanNode;
+ if (dataGenScanNode.getScanRangeLocations(0).size() != 1) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public List<AssignedJob> computeAssignedJobs(DistributedPlanWorkerManager workerManager,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ ConnectContext context = ConnectContext.get();
+ Map<ScanNode, ScanRanges> scanNodeToScanRanges = Maps.newLinkedHashMap();
+ for (ScanNode scanNode : scanNodes) {
+ List<TScanRangeLocations> scanRangeLocations = scanNode.getScanRangeLocations(0);
+ ScanRanges scanRanges = new ScanRanges();
+ for (TScanRangeLocations scanRangeLocation : scanRangeLocations) {
+ TScanRangeParams replica = ScanWorkerSelector.buildScanReplicaParams(
+ scanRangeLocation, scanRangeLocation.locations.get(0));
+ scanRanges.addScanRange(replica, 0);
+ }
+
+ scanNodeToScanRanges.put(scanNode, scanRanges);
+ }
+
+ DistributedPlanWorker randomWorker = workerManager.randomAvailableWorker();
+ return ImmutableList.of(
+ assignWorkerAndDataSources(0, context.nextInstanceId(),
+ randomWorker, new DefaultScanSource(scanNodeToScanRanges)
+ )
+ );
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java
new file mode 100644
index 00000000000..d142460ea2d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.TreeNode;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+
+/**
+ * WorkerJob.
+ * for example: a fragment job, which doesn't parallelization to some instance jobs and also no worker to invoke it
+ */
+public interface UnassignedJob extends TreeNode<UnassignedJob> {
+ PlanFragment getFragment();
+
+ List<ScanNode> getScanNodes();
+
+ ListMultimap<ExchangeNode, UnassignedJob> getExchangeToChildJob();
+
+ List<AssignedJob> computeAssignedJobs(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs);
+
+ // generate an instance job
+ // e.g. build an instance job by a backends and the replica ids it contains
+ default AssignedJob assignWorkerAndDataSources(
+ int instanceIndexInFragment, TUniqueId instanceId, DistributedPlanWorker worker, ScanSource scanSource) {
+ return new StaticAssignedJob(instanceIndexInFragment, instanceId, this, worker, scanSource);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
new file mode 100644
index 00000000000..c5fadc8fb8d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
@@ -0,0 +1,266 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
+import org.apache.doris.nereids.trees.plans.distribute.worker.LoadBalanceScanWorkerSelector;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.DataSink;
+import org.apache.doris.planner.DataStreamSink;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.MultiCastDataSink;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.PlanFragmentId;
+import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * UnassignedJobBuilder.
+ * build UnassignedJob by fragment
+ */
+public class UnassignedJobBuilder {
+ private final ScanWorkerSelector scanWorkerSelector = new LoadBalanceScanWorkerSelector();
+
+ /**
+ * build job from fragment.
+ */
+ public static FragmentIdMapping<UnassignedJob> buildJobs(FragmentIdMapping<PlanFragment> fragments) {
+ UnassignedJobBuilder builder = new UnassignedJobBuilder();
+
+ FragmentLineage fragmentLineage = buildFragmentLineage(fragments);
+ FragmentIdMapping<UnassignedJob> unassignedJobs = new FragmentIdMapping<>();
+
+ // build from leaf to parent
+ for (Entry<PlanFragmentId, PlanFragment> kv : fragments.entrySet()) {
+ PlanFragmentId fragmentId = kv.getKey();
+ PlanFragment fragment = kv.getValue();
+
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs = findInputJobs(
+ fragmentLineage, fragmentId, unassignedJobs);
+ UnassignedJob unassignedJob = builder.buildJob(fragment, inputJobs);
+ unassignedJobs.put(fragmentId, unassignedJob);
+ }
+ return unassignedJobs;
+ }
+
+ private UnassignedJob buildJob(
+ PlanFragment planFragment, ListMultimap<ExchangeNode, UnassignedJob> inputJobs) {
+ List<ScanNode> scanNodes = collectScanNodesInThisFragment(planFragment);
+ if (planFragment.specifyInstances.isPresent()) {
+ return buildSpecifyInstancesJob(planFragment, scanNodes, inputJobs);
+ } else if (!scanNodes.isEmpty() || isLeafFragment(planFragment)) {
+ return buildLeafOrScanJob(planFragment, scanNodes, inputJobs);
+ } else {
+ return buildShuffleJob(planFragment, inputJobs);
+ }
+ }
+
+ private UnassignedJob buildLeafOrScanJob(
+ PlanFragment planFragment, List<ScanNode> scanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs) {
+ int olapScanNodeNum = olapScanNodeNum(scanNodes);
+
+ UnassignedJob unassignedJob = null;
+ if (!scanNodes.isEmpty() && olapScanNodeNum == scanNodes.size()) {
+ // we need assign a backend which contains the data,
+ // so that the OlapScanNode can find the data in the backend
+ // e.g. select * from olap_table
+ unassignedJob = buildScanOlapTableJob(planFragment, (List) scanNodes, inputJobs, scanWorkerSelector);
+ } else if (scanNodes.isEmpty()) {
+ // select constant without table,
+ // e.g. select 100 union select 200
+ unassignedJob = buildQueryConstantJob(planFragment);
+ } else if (olapScanNodeNum == 0) {
+ // only scan external tables or cloud tables or table valued functions
+ // e,g. select * from numbers('number'='100')
+ unassignedJob = buildScanRemoteTableJob(planFragment, scanNodes, inputJobs, scanWorkerSelector);
+ }
+
+ if (unassignedJob != null) {
+ return unassignedJob;
+ }
+
+ throw new IllegalStateException(
+ "Unsupported fragment which contains multiple scan nodes and some of them are not OlapScanNode"
+ );
+ }
+
+ private UnassignedJob buildSpecifyInstancesJob(
+ PlanFragment planFragment, List<ScanNode> scanNodes, ListMultimap<ExchangeNode, UnassignedJob> inputJobs) {
+ return new UnassignedSpecifyInstancesJob(planFragment, scanNodes, inputJobs);
+ }
+
+ private UnassignedJob buildScanOlapTableJob(
+ PlanFragment planFragment, List<OlapScanNode> olapScanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs,
+ ScanWorkerSelector scanWorkerSelector) {
+ if (shouldAssignByBucket(planFragment)) {
+ return new UnassignedScanBucketOlapTableJob(
+ planFragment, olapScanNodes, inputJobs, scanWorkerSelector);
+ } else if (olapScanNodes.size() == 1) {
+ return new UnassignedScanSingleOlapTableJob(
+ planFragment, olapScanNodes.get(0), inputJobs, scanWorkerSelector);
+ } else {
+ throw new IllegalStateException("Not supported multiple scan multiple "
+ + "OlapTable but not contains colocate join or bucket shuffle join");
+ }
+ }
+
+ private List<ScanNode> collectScanNodesInThisFragment(PlanFragment planFragment) {
+ return planFragment.getPlanRoot().collectInCurrentFragment(ScanNode.class::isInstance);
+ }
+
+ private int olapScanNodeNum(List<ScanNode> scanNodes) {
+ int olapScanNodeNum = 0;
+ for (ScanNode scanNode : scanNodes) {
+ if (scanNode instanceof OlapScanNode) {
+ olapScanNodeNum++;
+ }
+ }
+ return olapScanNodeNum;
+ }
+
+ private boolean isLeafFragment(PlanFragment planFragment) {
+ return planFragment.getChildren().isEmpty();
+ }
+
+ private UnassignedQueryConstantJob buildQueryConstantJob(PlanFragment planFragment) {
+ return new UnassignedQueryConstantJob(planFragment);
+ }
+
+ private UnassignedJob buildScanRemoteTableJob(
+ PlanFragment planFragment, List<ScanNode> scanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs,
+ ScanWorkerSelector scanWorkerSelector) {
+ if (scanNodes.size() == 1) {
+ return new UnassignedScanSingleRemoteTableJob(
+ planFragment, scanNodes.get(0), inputJobs, scanWorkerSelector);
+ } else if (UnassignedGatherScanMultiRemoteTablesJob.canApply(scanNodes)) {
+ // select * from numbers("number" = "10") a union all select * from numbers("number" = "20") b;
+ // use an instance to scan table a and table b
+ return new UnassignedGatherScanMultiRemoteTablesJob(planFragment, scanNodes, inputJobs);
+ } else {
+ return null;
+ }
+ }
+
+ private UnassignedShuffleJob buildShuffleJob(
+ PlanFragment planFragment, ListMultimap<ExchangeNode, UnassignedJob> inputJobs) {
+ return new UnassignedShuffleJob(planFragment, inputJobs);
+ }
+
+ private static ListMultimap<ExchangeNode, UnassignedJob> findInputJobs(
+ FragmentLineage lineage, PlanFragmentId fragmentId, FragmentIdMapping<UnassignedJob> unassignedJobs) {
+ ListMultimap<ExchangeNode, UnassignedJob> inputJobs = ArrayListMultimap.create();
+ Map<PlanNodeId, ExchangeNode> exchangeNodes = lineage.parentFragmentToExchangeNode.get(fragmentId);
+ if (exchangeNodes != null) {
+ for (Entry<PlanNodeId, ExchangeNode> idToExchange : exchangeNodes.entrySet()) {
+ PlanNodeId exchangeId = idToExchange.getKey();
+ ExchangeNode exchangeNode = idToExchange.getValue();
+ List<PlanFragmentId> childFragmentIds = lineage.exchangeToChildFragment.get(exchangeId);
+ for (PlanFragmentId childFragmentId : childFragmentIds) {
+ inputJobs.put(exchangeNode, unassignedJobs.get(childFragmentId));
+ }
+ }
+ }
+ return inputJobs;
+ }
+
+ private static List<ExchangeNode> collectExchangeNodesInThisFragment(PlanFragment planFragment) {
+ return planFragment
+ .getPlanRoot()
+ .collectInCurrentFragment(ExchangeNode.class::isInstance);
+ }
+
+ private static FragmentLineage buildFragmentLineage(
+ FragmentIdMapping<PlanFragment> fragments) {
+ ListMultimap<PlanNodeId, PlanFragmentId> exchangeToChildFragment = ArrayListMultimap.create();
+ FragmentIdMapping<Map<PlanNodeId, ExchangeNode>> parentFragmentToExchangeNode = new FragmentIdMapping<>();
+
+ for (PlanFragment fragment : fragments.values()) {
+ PlanFragmentId fragmentId = fragment.getFragmentId();
+
+ // 1. link child fragment to exchange node
+ DataSink sink = fragment.getSink();
+ if (sink instanceof DataStreamSink) {
+ PlanNodeId exchangeNodeId = sink.getExchNodeId();
+ exchangeToChildFragment.put(exchangeNodeId, fragmentId);
+ } else if (sink instanceof MultiCastDataSink) {
+ MultiCastDataSink multiCastDataSink = (MultiCastDataSink) sink;
+ for (DataStreamSink dataStreamSink : multiCastDataSink.getDataStreamSinks()) {
+ PlanNodeId exchangeNodeId = dataStreamSink.getExchNodeId();
+ exchangeToChildFragment.put(exchangeNodeId, fragmentId);
+ }
+ }
+
+ // 2. link parent fragment to exchange node
+ List<ExchangeNode> exchangeNodes = collectExchangeNodesInThisFragment(fragment);
+ Map<PlanNodeId, ExchangeNode> exchangeNodesInFragment = Maps.newLinkedHashMap();
+ for (ExchangeNode exchangeNode : exchangeNodes) {
+ exchangeNodesInFragment.put(exchangeNode.getId(), exchangeNode);
+ }
+ parentFragmentToExchangeNode.put(fragmentId, exchangeNodesInFragment);
+ }
+
+ return new FragmentLineage(parentFragmentToExchangeNode, exchangeToChildFragment);
+ }
+
+ private static boolean shouldAssignByBucket(PlanFragment fragment) {
+ if (fragment.hasColocatePlanNode()) {
+ return true;
+ }
+ if (enableBucketShuffleJoin() && fragment.hasBucketShuffleJoin()) {
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean enableBucketShuffleJoin() {
+ if (ConnectContext.get() != null) {
+ SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
+ if (!sessionVariable.isEnableBucketShuffleJoin() && !sessionVariable.isEnableNereidsPlanner()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // the class support find exchange nodes in the fragment, and find child fragment by exchange node id
+ private static class FragmentLineage {
+ private final FragmentIdMapping<Map<PlanNodeId, ExchangeNode>> parentFragmentToExchangeNode;
+ private final ListMultimap<PlanNodeId, PlanFragmentId> exchangeToChildFragment;
+
+ public FragmentLineage(
+ FragmentIdMapping<Map<PlanNodeId, ExchangeNode>> parentFragmentToExchangeNode,
+ ListMultimap<PlanNodeId, PlanFragmentId> exchangeToChildFragment) {
+ this.parentFragmentToExchangeNode = parentFragmentToExchangeNode;
+ this.exchangeToChildFragment = exchangeToChildFragment;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java
new file mode 100644
index 00000000000..bfbafd07390
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+
+/** UnassignedQueryConstantJob */
+public class UnassignedQueryConstantJob extends AbstractUnassignedJob {
+ public UnassignedQueryConstantJob(PlanFragment fragment) {
+ super(fragment, ImmutableList.of(), ArrayListMultimap.create());
+ }
+
+ @Override
+ public List<AssignedJob> computeAssignedJobs(DistributedPlanWorkerManager workerManager,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ DistributedPlanWorker randomWorker = workerManager.randomAvailableWorker();
+ ConnectContext context = ConnectContext.get();
+ return ImmutableList.of(
+ new StaticAssignedJob(0, context.nextInstanceId(), this,
+ randomWorker, new DefaultScanSource(ImmutableMap.of())
+ )
+ );
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
new file mode 100644
index 00000000000..7a14d5a2f1c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
@@ -0,0 +1,310 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.analysis.JoinOperator;
+import org.apache.doris.catalog.MaterializedIndex;
+import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Replica;
+import org.apache.doris.catalog.Tablet;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.HashJoinNode;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * UnassignedScanBucketOlapTableJob.
+ * bucket shuffle join olap table, or colocate join olap table
+ */
+public class UnassignedScanBucketOlapTableJob extends AbstractUnassignedScanJob {
+ private final ScanWorkerSelector scanWorkerSelector;
+ private final List<OlapScanNode> olapScanNodes;
+
+ /** UnassignedScanNativeTableJob */
+ public UnassignedScanBucketOlapTableJob(
+ PlanFragment fragment, List<OlapScanNode> olapScanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob,
+ ScanWorkerSelector scanWorkerSelector) {
+ super(fragment, (List) olapScanNodes, exchangeToChildJob);
+ this.scanWorkerSelector = Objects.requireNonNull(
+ scanWorkerSelector, "scanWorkerSelector cat not be null");
+
+ Preconditions.checkArgument(!olapScanNodes.isEmpty(), "OlapScanNode is empty");
+ this.olapScanNodes = olapScanNodes;
+ }
+
+ public List<OlapScanNode> getOlapScanNodes() {
+ return olapScanNodes;
+ }
+
+ @Override
+ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ // for every bucket tablet, select its replica and worker.
+ // for example, colocate join:
+ // {
+ // BackendWorker("172.0.0.1"): {
+ // bucket 0: {
+ // olapScanNode1: ScanRanges([tablet_10001, tablet_10002, tablet_10003, tablet_10004]),
+ // olapScanNode2: ScanRanges([tablet_10009, tablet_10010, tablet_10011, tablet_10012])
+ // },
+ // bucket 1: {
+ // olapScanNode1: ScanRanges([tablet_10005, tablet_10006, tablet_10007, tablet_10008])
+ // olapScanNode2: ScanRanges([tablet_10013, tablet_10014, tablet_10015, tablet_10016])
+ // },
+ // ...
+ // },
+ // BackendWorker("172.0.0.2"): {
+ // ...
+ // }
+ // }
+ return scanWorkerSelector.selectReplicaAndWorkerWithBucket(this);
+ }
+
+ @Override
+ protected List<AssignedJob> insideMachineParallelization(
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ // separate buckets to instanceNum groups, let one instance process some buckets.
+ // for example, colocate join:
+ // {
+ // // 172.0.0.1 has two instances
+ // BackendWorker("172.0.0.1"): [
+ // // instance 1 process two buckets
+ // {
+ // bucket 0: {
+ // olapScanNode1: ScanRanges([tablet_10001, tablet_10002, tablet_10003, tablet_10004]),
+ // olapScanNode2: ScanRanges([tablet_10009, tablet_10010, tablet_10011, tablet_10012])
+ // },
+ // bucket 1: {
+ // olapScanNode1: ScanRanges([tablet_10005, tablet_10006, tablet_10007, tablet_10008])
+ // olapScanNode2: ScanRanges([tablet_10013, tablet_10014, tablet_10015, tablet_10016])
+ // }
+ // },
+ // // instance 1 process one bucket
+ // {
+ // bucket 3: ...
+ // }
+ // ]
+ // // instance 4... in "172.0.0.1"
+ // BackendWorker("172.0.0.2"): [
+ // ...
+ // ],
+ // ...
+ // }
+ List<AssignedJob> assignedJobs = super.insideMachineParallelization(workerToScanRanges, inputJobs);
+
+ // the case:
+ // ```sql
+ // SELECT * FROM
+ // (select * from tbl1 where c0 =1)a
+ // RIGHT OUTER JOIN
+ // (select * from tbl2)b
+ // ON a.id = b.id;
+ // ```
+ // contains right outer join and missing instance in left side because of tablet pruner, for example
+ // left: [bucket 1]
+ // right: [bucket 1, bucket 2]
+ //
+ // we should join buckets corresponding:
+ // [
+ // (left bucket 1) right outer join (right bucket 1)
+ // (no any machine) right outer join (right bucket 2)
+ // ]
+ // if missing the left bucket 2, it will compute an empty result
+ // because right bucket 2 doesn't exist destination instance,
+ // so we should fill up this instance
+ List<HashJoinNode> hashJoinNodes = fragment.getPlanRoot()
+ .collectInCurrentFragment(HashJoinNode.class::isInstance);
+ if (shouldFillUpInstances(hashJoinNodes)) {
+ return fillUpInstances(assignedJobs);
+ }
+
+ return assignedJobs;
+ }
+
+ private boolean shouldFillUpInstances(List<HashJoinNode> hashJoinNodes) {
+ for (HashJoinNode hashJoinNode : hashJoinNodes) {
+ if (!hashJoinNode.isBucketShuffle()) {
+ continue;
+ }
+ JoinOperator joinOp = hashJoinNode.getJoinOp();
+ switch (joinOp) {
+ case INNER_JOIN:
+ case CROSS_JOIN:
+ break;
+ default:
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private List<AssignedJob> fillUpInstances(List<AssignedJob> instances) {
+ Set<Integer> missingBucketIndexes = missingBuckets(instances);
+ if (missingBucketIndexes.isEmpty()) {
+ return instances;
+ }
+
+ ConnectContext context = ConnectContext.get();
+
+ OlapScanNode olapScanNode = (OlapScanNode) scanNodes.get(0);
+ MaterializedIndex randomPartition = randomPartition(olapScanNode);
+ ListMultimap<DistributedPlanWorker, Integer> missingBuckets = selectWorkerForMissingBuckets(
+ olapScanNode, randomPartition, missingBucketIndexes);
+
+ boolean useLocalShuffle = instances.stream().anyMatch(LocalShuffleAssignedJob.class::isInstance);
+ List<AssignedJob> newInstances = new ArrayList<>(instances);
+ for (Entry<DistributedPlanWorker, Collection<Integer>> workerToBuckets : missingBuckets.asMap().entrySet()) {
+ Map<Integer, Map<ScanNode, ScanRanges>> scanEmptyBuckets = Maps.newLinkedHashMap();
+ for (Integer bucketIndex : workerToBuckets.getValue()) {
+ Map<ScanNode, ScanRanges> scanTableWithEmptyData = Maps.newLinkedHashMap();
+ for (ScanNode scanNode : scanNodes) {
+ scanTableWithEmptyData.put(scanNode, new ScanRanges());
+ }
+ scanEmptyBuckets.put(bucketIndex, scanTableWithEmptyData);
+ }
+
+ AssignedJob fillUpInstance = null;
+ DistributedPlanWorker worker = workerToBuckets.getKey();
+ BucketScanSource scanSource = new BucketScanSource(scanEmptyBuckets);
+ if (useLocalShuffle) {
+ // when use local shuffle, we should ensure every backend only process one instance!
+ // so here we should try to merge the missing buckets into exist instances
+ boolean mergedBucketsInSameWorkerInstance = false;
+ for (AssignedJob newInstance : newInstances) {
+ if (newInstance.getAssignedWorker().equals(worker)) {
+ BucketScanSource bucketScanSource = (BucketScanSource) newInstance.getScanSource();
+ bucketScanSource.bucketIndexToScanNodeToTablets.putAll(scanEmptyBuckets);
+ mergedBucketsInSameWorkerInstance = true;
+ }
+ }
+ if (!mergedBucketsInSameWorkerInstance) {
+ fillUpInstance = new LocalShuffleAssignedJob(
+ newInstances.size(), shareScanIdGenerator.getAndIncrement(),
+ context.nextInstanceId(), this, worker, scanSource
+ );
+ }
+ } else {
+ fillUpInstance = assignWorkerAndDataSources(
+ newInstances.size(), context.nextInstanceId(), worker, scanSource
+ );
+ }
+ if (fillUpInstance != null) {
+ newInstances.add(fillUpInstance);
+ }
+ }
+ return newInstances;
+ }
+
+ private int fullBucketNum() {
+ for (ScanNode scanNode : scanNodes) {
+ if (scanNode instanceof OlapScanNode) {
+ return ((OlapScanNode) scanNode).getBucketNum();
+ }
+ }
+ throw new IllegalStateException("Not support bucket shuffle join with non OlapTable");
+ }
+
+ private Set<Integer> missingBuckets(List<AssignedJob> instances) {
+ Set<Integer> usedBuckets = usedBuckets(instances);
+ int bucketNum = fullBucketNum();
+ Set<Integer> missingBuckets = new TreeSet<>();
+ for (int i = 0; i < bucketNum; i++) {
+ if (!usedBuckets.contains(i)) {
+ missingBuckets.add(i);
+ }
+ }
+ return missingBuckets;
+ }
+
+ private Set<Integer> usedBuckets(List<AssignedJob> instances) {
+ Set<Integer> usedBuckets = new TreeSet<>();
+ for (AssignedJob instance : instances) {
+ ScanSource scanSource = instance.getScanSource();
+ if (scanSource instanceof BucketScanSource) {
+ BucketScanSource bucketScanSource = (BucketScanSource) scanSource;
+ usedBuckets.addAll(bucketScanSource.bucketIndexToScanNodeToTablets.keySet());
+ }
+ }
+ return usedBuckets;
+ }
+
+ private MaterializedIndex randomPartition(OlapScanNode olapScanNode) {
+ List<Long> selectedPartitionIds = ImmutableList.copyOf(olapScanNode.getSelectedPartitionIds());
+ if (selectedPartitionIds.isEmpty()) {
+ throw new IllegalStateException("Missing selected partitions in " + olapScanNode);
+ }
+
+ Long randomSelectPartitionId = selectedPartitionIds.get((int) (Math.random() * selectedPartitionIds.size()));
+ Partition partition = olapScanNode.getOlapTable().getPartition(randomSelectPartitionId);
+ return partition.getBaseIndex();
+ }
+
+ private ListMultimap<DistributedPlanWorker, Integer> selectWorkerForMissingBuckets(
+ OlapScanNode olapScanNode, MaterializedIndex partition, Set<Integer> selectBucketIndexes) {
+ List<Long> tabletIdsInOrder = partition.getTabletIdsInOrder();
+ ListMultimap<DistributedPlanWorker, Integer> fillUpWorkerToBuckets = ArrayListMultimap.create();
+ for (Integer bucketIndex : selectBucketIndexes) {
+ Long tabletIdInBucket = tabletIdsInOrder.get(bucketIndex);
+ Tablet tabletInBucket = partition.getTablet(tabletIdInBucket);
+ List<DistributedPlanWorker> workers = getWorkersByReplicas(tabletInBucket);
+ if (workers.isEmpty()) {
+ throw new IllegalStateException("Can not found available replica for bucket " + bucketIndex
+ + ", table: " + olapScanNode);
+ }
+ DistributedPlanWorker worker = scanWorkerSelector.selectMinWorkloadWorker(workers);
+ fillUpWorkerToBuckets.put(worker, bucketIndex);
+ }
+ return fillUpWorkerToBuckets;
+ }
+
+ private List<DistributedPlanWorker> getWorkersByReplicas(Tablet tablet) {
+ DistributedPlanWorkerManager workerManager = scanWorkerSelector.getWorkerManager();
+ List<Replica> replicas = tablet.getReplicas();
+ List<DistributedPlanWorker> workers = Lists.newArrayListWithCapacity(replicas.size());
+ for (Replica replica : replicas) {
+ DistributedPlanWorker worker = workerManager.getWorker(replica.getBackendId());
+ if (worker.available()) {
+ workers.add(worker);
+ }
+ }
+ return workers;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java
new file mode 100644
index 00000000000..94b545f9f9e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/** UnassignedScanSingleOlapTableJob */
+public class UnassignedScanSingleOlapTableJob extends AbstractUnassignedScanJob {
+ private OlapScanNode olapScanNode;
+ private final ScanWorkerSelector scanWorkerSelector;
+
+ public UnassignedScanSingleOlapTableJob(
+ PlanFragment fragment, OlapScanNode olapScanNode,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob,
+ ScanWorkerSelector scanWorkerSelector) {
+ super(fragment, ImmutableList.of(olapScanNode), exchangeToChildJob);
+ this.scanWorkerSelector = Objects.requireNonNull(
+ scanWorkerSelector, "scanWorkerSelector cat not be null");
+ this.olapScanNode = olapScanNode;
+ }
+
+ @Override
+ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ // for every tablet, select its replica and worker.
+ // for example:
+ // {
+ // BackendWorker("172.0.0.1"):
+ // olapScanNode1: ScanRanges([tablet_10001, tablet_10002, tablet_10003, tablet_10004]),
+ // BackendWorker("172.0.0.2"):
+ // olapScanNode1: ScanRanges([tablet_10005, tablet_10006, tablet_10007, tablet_10008, tablet_10009])
+ // }
+ return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(olapScanNode);
+ }
+
+ @Override
+ protected List<AssignedJob> insideMachineParallelization(
+ Map<DistributedPlanWorker, UninstancedScanSource> workerToScanRanges,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ // for each worker, compute how many instances should be generated, and which data should be scanned.
+ // for example:
+ // {
+ // BackendWorker("172.0.0.1"): [
+ // instance 1: olapScanNode1: ScanRanges([tablet_10001, tablet_10003])
+ // instance 2: olapScanNode1: ScanRanges([tablet_10002, tablet_10004])
+ // ],
+ // BackendWorker("172.0.0.2"): [
+ // instance 3: olapScanNode1: ScanRanges([tablet_10005, tablet_10008])
+ // instance 4: olapScanNode1: ScanRanges([tablet_10006, tablet_10009])
+ // instance 5: olapScanNode1: ScanRanges([tablet_10007])
+ // ],
+ // }
+ return super.insideMachineParallelization(workerToScanRanges, inputJobs);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java
new file mode 100644
index 00000000000..8205f0d67d2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.nereids.trees.plans.distribute.worker.ScanWorkerSelector;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ListMultimap;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * UnassignedScanSingleRemoteTableJob
+ * it should be a leaf job which not contains scan native olap table node,
+ * for example, select literal without table, or scan an external table
+ */
+public class UnassignedScanSingleRemoteTableJob extends AbstractUnassignedScanJob {
+ private final ScanWorkerSelector scanWorkerSelector;
+
+ public UnassignedScanSingleRemoteTableJob(
+ PlanFragment fragment, ScanNode scanNode, ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob,
+ ScanWorkerSelector scanWorkerSelector) {
+ super(fragment, ImmutableList.of(scanNode), exchangeToChildJob);
+ this.scanWorkerSelector = Objects.requireNonNull(scanWorkerSelector, "scanWorkerSelector is not null");
+ }
+
+ @Override
+ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesParallelization(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ return scanWorkerSelector.selectReplicaAndWorkerWithoutBucket(scanNodes.get(0));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
new file mode 100644
index 00000000000..5b1cb1f9878
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.function.Function;
+
+/** UnassignedShuffleJob */
+public class UnassignedShuffleJob extends AbstractUnassignedJob {
+ public UnassignedShuffleJob(PlanFragment fragment, ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob) {
+ super(fragment, ImmutableList.of(), exchangeToChildJob);
+ }
+
+ @Override
+ public List<AssignedJob> computeAssignedJobs(
+ DistributedPlanWorkerManager workerManager, ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ int expectInstanceNum = degreeOfParallelism();
+ List<AssignedJob> biggestParallelChildFragment = getInstancesOfBiggestParallelChildFragment(inputJobs);
+
+ if (expectInstanceNum > 0 && expectInstanceNum < biggestParallelChildFragment.size()) {
+ // When group by cardinality is smaller than number of backend, only some backends always
+ // process while other has no data to process.
+ // So we shuffle instances to make different backends handle different queries.
+ List<DistributedPlanWorker> shuffleWorkersInBiggestParallelChildFragment
+ = distinctShuffleWorkers(biggestParallelChildFragment);
+ Function<Integer, DistributedPlanWorker> workerSelector = instanceIndex -> {
+ int selectIndex = instanceIndex % shuffleWorkersInBiggestParallelChildFragment.size();
+ return shuffleWorkersInBiggestParallelChildFragment.get(selectIndex);
+ };
+ return buildInstances(expectInstanceNum, workerSelector);
+ } else {
+ // keep same instance num like child fragment
+ Function<Integer, DistributedPlanWorker> workerSelector = instanceIndex -> {
+ int selectIndex = instanceIndex % biggestParallelChildFragment.size();
+ return biggestParallelChildFragment.get(selectIndex).getAssignedWorker();
+ };
+ return buildInstances(biggestParallelChildFragment.size(), workerSelector);
+ }
+ }
+
+ protected int degreeOfParallelism() {
+ if (!fragment.getDataPartition().isPartitioned()) {
+ return 1;
+ }
+
+ // TODO: check we use nested loop join do right outer / semi / anti join,
+ // we should add an exchange node with gather distribute under the nested loop join
+
+ int expectInstanceNum = -1;
+ if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
+ expectInstanceNum = ConnectContext.get().getSessionVariable().getExchangeInstanceParallel();
+ }
+ return expectInstanceNum;
+ }
+
+ private List<AssignedJob> getInstancesOfBiggestParallelChildFragment(
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ int maxInstanceNum = -1;
+ List<AssignedJob> biggestParallelChildFragment = ImmutableList.of();
+ // skip broadcast exchange
+ for (Entry<ExchangeNode, Collection<AssignedJob>> exchangeToChildInstances : inputJobs.asMap().entrySet()) {
+ List<AssignedJob> instances = (List) exchangeToChildInstances.getValue();
+ if (instances.size() > maxInstanceNum) {
+ biggestParallelChildFragment = instances;
+ maxInstanceNum = instances.size();
+ }
+ }
+ return biggestParallelChildFragment;
+ }
+
+ private List<AssignedJob> buildInstances(int instanceNum, Function<Integer, DistributedPlanWorker> workerSelector) {
+ ImmutableList.Builder<AssignedJob> instances = ImmutableList.builderWithExpectedSize(instanceNum);
+ ConnectContext context = ConnectContext.get();
+ for (int i = 0; i < instanceNum; i++) {
+ DistributedPlanWorker selectedWorker = workerSelector.apply(i);
+ AssignedJob assignedJob = assignWorkerAndDataSources(
+ i, context.nextInstanceId(), selectedWorker, new DefaultScanSource(ImmutableMap.of())
+ );
+ instances.add(assignedJob);
+ }
+ return instances.build();
+ }
+
+ private List<DistributedPlanWorker> distinctShuffleWorkers(List<AssignedJob> instances) {
+ Set<DistributedPlanWorker> candidateWorkerSet = Sets.newLinkedHashSet();
+ for (AssignedJob instance : instances) {
+ candidateWorkerSet.add(instance.getAssignedWorker());
+ }
+ List<DistributedPlanWorker> candidateWorkers = Lists.newArrayList(candidateWorkerSet);
+ Collections.shuffle(candidateWorkers);
+ return candidateWorkers;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java
new file mode 100644
index 00000000000..c450161ae41
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.NereidsSpecifyInstances;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorkerManager;
+import org.apache.doris.planner.ExchangeNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.ScanNode;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ListMultimap;
+
+import java.util.List;
+
+/** UnassignedSpecifyInstancesJob */
+public class UnassignedSpecifyInstancesJob extends AbstractUnassignedJob {
+ private final NereidsSpecifyInstances<ScanSource> specifyInstances;
+
+ public UnassignedSpecifyInstancesJob(
+ PlanFragment fragment, List<ScanNode> scanNodes,
+ ListMultimap<ExchangeNode, UnassignedJob> exchangeToChildJob) {
+ super(fragment, scanNodes, exchangeToChildJob);
+ Preconditions.checkArgument(fragment.specifyInstances.isPresent(),
+ "Missing fragment specifyInstances");
+ this.specifyInstances = fragment.specifyInstances.get();
+ }
+
+ @Override
+ public List<AssignedJob> computeAssignedJobs(DistributedPlanWorkerManager workerManager,
+ ListMultimap<ExchangeNode, AssignedJob> inputJobs) {
+ return specifyInstances.buildAssignedJobs(this);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UninstancedScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UninstancedScanSource.java
new file mode 100644
index 00000000000..cce4c2f25f9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UninstancedScanSource.java
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * UninstancedScanSource:
+ * a ScanSource which doesn't parallelize/split to instances
+ */
+public class UninstancedScanSource {
+ public final ScanSource scanSource;
+
+ public UninstancedScanSource(ScanSource scanSource) {
+ this.scanSource = scanSource;
+ }
+
+ public static UninstancedScanSource emptyDefaultScanSource() {
+ return new UninstancedScanSource(new DefaultScanSource(ImmutableMap.of()));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/WorkerScanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/WorkerScanSource.java
new file mode 100644
index 00000000000..f9c5993a5b5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/WorkerScanSource.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.distribute.worker.job;
+
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+
+/** WorkerScanSource */
+public class WorkerScanSource<S extends ScanSource> {
+ public final DistributedPlanWorker worker;
+ public final S scanSource;
+
+ public WorkerScanSource(DistributedPlanWorker worker, S scanSource) {
+ this.worker = worker;
+ this.scanSource = scanSource;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
index c9067e7cc46..986aef4fc17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
@@ -25,6 +25,7 @@ import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.DistributionSpecHash;
import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
import org.apache.doris.nereids.properties.DistributionSpecReplicated;
+import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.rewrite.ForeignKeyContext;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -224,12 +225,25 @@ public class JoinUtils {
* return true if we should do bucket shuffle join when translate plan.
*/
public static boolean shouldBucketShuffleJoin(AbstractPhysicalJoin<PhysicalPlan, PhysicalPlan> join) {
- DistributionSpec rightDistributionSpec = join.right().getPhysicalProperties().getDistributionSpec();
- if (!(rightDistributionSpec instanceof DistributionSpecHash)) {
+ if (isStorageBucketed(join.right().getPhysicalProperties())) {
+ return true;
+ } else if (SessionVariable.canUseNereidsDistributePlanner()
+ && isStorageBucketed(join.left().getPhysicalProperties())) {
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean isStorageBucketed(PhysicalProperties physicalProperties) {
+ DistributionSpec distributionSpec = physicalProperties.getDistributionSpec();
+ if (!(distributionSpec instanceof DistributionSpecHash)) {
return false;
}
- DistributionSpecHash rightHash = (DistributionSpecHash) rightDistributionSpec;
- return rightHash.getShuffleType() == ShuffleType.STORAGE_BUCKETED;
+ DistributionSpecHash rightHash = (DistributionSpecHash) distributionSpec;
+ if (rightHash.getShuffleType() == ShuffleType.STORAGE_BUCKETED) {
+ return true;
+ }
+ return false;
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
index 8525d2629e3..289119dc0d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
@@ -466,4 +466,18 @@ public class Utils {
}
return firstChar + input.substring(1);
}
+
+ /** addLinePrefix */
+ public static String addLinePrefix(String str, String prefix) {
+ StringBuilder newStr = new StringBuilder((int) (str.length() * 1.2));
+ String[] lines = str.split("\n");
+ for (int i = 0; i < lines.length; i++) {
+ String line = lines[i];
+ newStr.append(prefix).append(line);
+ if (i + 1 < lines.length) {
+ newStr.append("\n");
+ }
+ }
+ return newStr.toString();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
index 1c760adb94a..60fce4df148 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
@@ -59,6 +59,10 @@ public class DataGenScanNode extends ExternalScanNode {
super.init(analyzer);
}
+ public DataGenTableValuedFunction getTvf() {
+ return tvf;
+ }
+
@Override
public List<TScanRangeLocations> getScanRangeLocations(long maxScanRangeLength) {
return scanRangeLocations;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 5849aa71d6a..0e403123364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -169,6 +169,8 @@ public class OlapScanNode extends ScanNode {
private int selectedPartitionNum = 0;
private Collection<Long> selectedPartitionIds = Lists.newArrayList();
private long totalBytes = 0;
+ // tablet id to single replica bytes
+ private Map<Long, Long> tabletBytes = Maps.newLinkedHashMap();
private SortInfo sortInfo = null;
private Set<Integer> outputColumnUniqueIds = new HashSet<>();
@@ -191,6 +193,7 @@ public class OlapScanNode extends ScanNode {
// a bucket seq may map to many tablets, and each tablet has a
// TScanRangeLocations.
public ArrayListMultimap<Integer, TScanRangeLocations> bucketSeq2locations = ArrayListMultimap.create();
+ public Map<Integer, Long> bucketSeq2Bytes = Maps.newLinkedHashMap();
boolean isFromPrepareStmt = false;
// For point query
@@ -748,6 +751,10 @@ public class OlapScanNode extends ScanNode {
}
}
+ public Long getTabletSingleReplicaSize(Long tabletId) {
+ return tabletBytes.get(tabletId);
+ }
+
private void addScanRangeLocations(Partition partition,
List<Tablet> tablets) throws UserException {
long visibleVersion = Partition.PARTITION_INIT_VERSION;
@@ -877,6 +884,9 @@ public class OlapScanNode extends ScanNode {
boolean tabletIsNull = true;
boolean collectedStat = false;
List<String> errs = Lists.newArrayList();
+
+ int replicaInTablet = 0;
+ long oneReplicaBytes = 0;
for (Replica replica : replicas) {
Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendId());
if (backend == null || !backend.isAlive()) {
@@ -916,7 +926,13 @@ public class OlapScanNode extends ScanNode {
// for CBO
if (!collectedStat && replica.getRowCount() != -1) {
- totalBytes += replica.getDataSize();
+ long dataSize = replica.getDataSize();
+ if (replicaInTablet == 0) {
+ oneReplicaBytes = dataSize;
+ tabletBytes.put(tabletId, dataSize);
+ }
+ replicaInTablet++;
+ totalBytes += dataSize;
collectedStat = true;
}
scanBackendIds.add(backend.getId());
@@ -934,8 +950,9 @@ public class OlapScanNode extends ScanNode {
scanRange.setPaloScanRange(paloRange);
locations.setScanRange(scanRange);
- bucketSeq2locations.put(tabletId2BucketSeq.get(tabletId), locations);
-
+ Integer bucketSeq = tabletId2BucketSeq.get(tabletId);
+ bucketSeq2locations.put(bucketSeq, locations);
+ bucketSeq2Bytes.merge(bucketSeq, oneReplicaBytes, Long::sum);
scanRangeLocations.add(locations);
}
@@ -1403,6 +1420,20 @@ public class OlapScanNode extends ScanNode {
return shouldColoScan;
}
+ public int getBucketNum() {
+ // In bucket shuffle join, we have 2 situation.
+ // 1. Only one partition: in this case, we use scanNode.getTotalTabletsNum() to get the right bucket num
+ // because when table turn on dynamic partition, the bucket number in default distribution info
+ // is not correct.
+ // 2. Table is colocated: in this case, table could have more than one partition, but all partition's
+ // bucket number must be same, so we use default bucket num is ok.
+ if (olapTable.isColocateTable()) {
+ return olapTable.getDefaultDistributionInfo().getBucketNum();
+ } else {
+ return (int) totalTabletsNum;
+ }
+ }
+
@Override
// If scan is key search, should not enable the shared scan opt to prevent the performance problem
// 1. where contain the eq or in expr of key column slot
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
index 7418e15bdc8..8a26ec4af66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
@@ -27,6 +27,8 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.common.TreeNode;
+import org.apache.doris.nereids.trees.plans.distribute.NereidsSpecifyInstances;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPartitionType;
@@ -34,6 +36,7 @@ import org.apache.doris.thrift.TPlanFragment;
import org.apache.doris.thrift.TResultSinkType;
import com.google.common.base.Preconditions;
+import com.google.common.base.Suppliers;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
@@ -43,7 +46,9 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
@@ -148,9 +153,12 @@ public class PlanFragment extends TreeNode<PlanFragment> {
// has colocate plan node
protected boolean hasColocatePlanNode = false;
+ protected final Supplier<Boolean> hasBucketShuffleJoin;
private TResultSinkType resultSinkType = TResultSinkType.MYSQL_PROTOCAL;
+ public Optional<NereidsSpecifyInstances<ScanSource>> specifyInstances = Optional.empty();
+
/**
* C'tor for fragment with specific partition; the output is by default broadcast.
*/
@@ -162,6 +170,7 @@ public class PlanFragment extends TreeNode<PlanFragment> {
this.transferQueryStatisticsWithEveryBatch = false;
this.builderRuntimeFilterIds = new HashSet<>();
this.targetRuntimeFilterIds = new HashSet<>();
+ this.hasBucketShuffleJoin = buildHasBucketShuffleJoin();
setParallelExecNumIfExists();
setFragmentInPlanTree(planRoot);
}
@@ -178,6 +187,18 @@ public class PlanFragment extends TreeNode<PlanFragment> {
this.targetRuntimeFilterIds = new HashSet<>(targetRuntimeFilterIds);
}
+ private Supplier<Boolean> buildHasBucketShuffleJoin() {
+ return Suppliers.memoize(() -> {
+ List<HashJoinNode> hashJoinNodes = getPlanRoot().collectInCurrentFragment(HashJoinNode.class::isInstance);
+ for (HashJoinNode hashJoinNode : hashJoinNodes) {
+ if (hashJoinNode.isBucketShuffle()) {
+ return true;
+ }
+ }
+ return false;
+ });
+ }
+
/**
* Assigns 'this' as fragment of all PlanNodes in the plan tree rooted at node.
* Does not traverse the children of ExchangeNodes because those must belong to a
@@ -240,6 +261,10 @@ public class PlanFragment extends TreeNode<PlanFragment> {
this.hasColocatePlanNode = hasColocatePlanNode;
}
+ public boolean hasBucketShuffleJoin() {
+ return hasBucketShuffleJoin.get();
+ }
+
public void setResultSinkType(TResultSinkType resultSinkType) {
this.resultSinkType = resultSinkType;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 198d5171e26..37f121a5c23 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -59,6 +59,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
/**
@@ -1256,4 +1258,27 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
public void addIntermediateProjectList(List<Expr> exprs) {
intermediateProjectListList.add(exprs);
}
+
+ public <T extends PlanNode> List<T> collectInCurrentFragment(Predicate<PlanNode> predicate) {
+ List<PlanNode> result = Lists.newArrayList();
+ foreachDownInCurrentFragment(child -> {
+ if (predicate.test(child)) {
+ result.add(child);
+ }
+ });
+ return (List) result;
+ }
+
+ /** foreachDownInCurrentFragment */
+ public void foreachDownInCurrentFragment(Consumer<PlanNode> visitor) {
+ int currentFragmentId = getFragmentId().asInt();
+ foreachDown(child -> {
+ PlanNode childNode = (PlanNode) child;
+ if (childNode.getFragmentId().asInt() != currentFragmentId) {
+ return false;
+ }
+ visitor.accept(childNode);
+ return true;
+ });
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 342ec3c9149..f53f69505da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -570,9 +570,12 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
@Override
public String toString() {
- return MoreObjects.toStringHelper(this).add("tid", desc.getId().asInt()).add("tblName",
- desc.getTable().getName()).add("keyRanges", "").addValue(
- super.debugString()).toString();
+ return MoreObjects.toStringHelper(this)
+ .add("id", getId().asInt())
+ .add("tid", desc.getId().asInt())
+ .add("tblName", desc.getTable().getName())
+ .add("keyRanges", "")
+ .addValue(super.debugString()).toString();
}
// Some of scan node(eg, DataGenScanNode) does not need to check column priv
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 6b6278db7d6..71a10d21171 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -89,6 +89,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
@@ -118,6 +119,7 @@ public class ConnectContext {
protected volatile LoadTaskInfo streamLoadInfo;
protected volatile TUniqueId queryId = null;
+ protected volatile AtomicInteger instanceIdGenerator = new AtomicInteger();
protected volatile String traceId;
// id for this connection
protected volatile int connectionId;
@@ -870,6 +872,10 @@ public class ConnectContext {
return queryId;
}
+ public TUniqueId nextInstanceId() {
+ return new TUniqueId(queryId.hi, queryId.lo + instanceIdGenerator.incrementAndGet());
+ }
+
public String getSqlHash() {
return sqlHash;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index a29f679303e..2c2d4437441 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -41,6 +41,8 @@ import org.apache.doris.metric.MetricRepo;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.stats.StatsErrorEstimator;
+import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
import org.apache.doris.nereids.trees.plans.physical.TopnFilter;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.planner.DataSink;
@@ -186,9 +188,10 @@ public class Coordinator implements CoordInterface {
// copied from TQueryExecRequest; constant across all fragments
private final TDescriptorTable descTable;
+ private FragmentIdMapping<DistributedPlan> distributedPlans;
// scan node id -> TFileScanRangeParams
- private Map<Integer, TFileScanRangeParams> fileScanRangeParamsMap = Maps.newHashMap();
+ protected Map<Integer, TFileScanRangeParams> fileScanRangeParamsMap = Maps.newHashMap();
// Why do we use query global?
// When `NOW()` function is in sql, we need only one now(),
@@ -212,7 +215,7 @@ public class Coordinator implements CoordInterface {
private boolean returnedAllResults = false;
// populated in computeFragmentExecParams()
- private final Map<PlanFragmentId, FragmentExecParams> fragmentExecParamsMap = Maps.newHashMap();
+ protected final Map<PlanFragmentId, FragmentExecParams> fragmentExecParamsMap = Maps.newHashMap();
private final List<PlanFragment> fragments;
@@ -336,6 +339,8 @@ public class Coordinator implements CoordInterface {
if (!useNereids) {
// Enable local shuffle on pipelineX engine only if Nereids planner is applied.
queryOptions.setEnableLocalShuffle(false);
+ } else {
+ distributedPlans = ((NereidsPlanner) planner).getDistributedPlans();
}
setFromUserProperty(context);
@@ -1287,7 +1292,7 @@ public class Coordinator implements CoordInterface {
}
}
- private void computeFragmentExecParams() throws Exception {
+ protected void computeFragmentExecParams() throws Exception {
// fill hosts field in fragmentExecParams
computeFragmentHosts();
@@ -1303,10 +1308,16 @@ public class Coordinator implements CoordInterface {
for (int j = 0; j < params.instanceExecParams.size(); ++j) {
// we add instance_num to query_id.lo to create a
// globally-unique instance id
+ FInstanceExecParam instanceExecParam = params.instanceExecParams.get(j);
+
+ // already set by nereids coordinator?
+ if (instanceExecParam.instanceId != null) {
+ continue;
+ }
TUniqueId instanceId = new TUniqueId();
instanceId.setHi(queryId.hi);
instanceId.setLo(queryId.lo + instanceIds.size() + 1);
- params.instanceExecParams.get(j).instanceId = instanceId;
+ instanceExecParam.instanceId = instanceId;
instanceIds.add(instanceId);
}
}
@@ -1679,7 +1690,7 @@ public class Coordinator implements CoordInterface {
// For each fragment in fragments, computes hosts on which to run the instances
// and stores result in fragmentExecParams.hosts.
- private void computeFragmentHosts() throws Exception {
+ protected void computeFragmentHosts() throws Exception {
// compute hosts of producer fragment before those of consumer fragment(s),
// the latter might inherit the set of hosts from the former
// compute hosts *bottom up*.
@@ -2102,12 +2113,7 @@ public class Coordinator implements CoordInterface {
fragmentIdTobucketSeqToScanRangeMap.put(scanNode.getFragmentId(), new BucketSeqToScanRange());
// Same as bucket shuffle.
- int bucketNum = 0;
- if (scanNode.getOlapTable().isColocateTable()) {
- bucketNum = scanNode.getOlapTable().getDefaultDistributionInfo().getBucketNum();
- } else {
- bucketNum = (int) (scanNode.getTotalTabletsNum());
- }
+ int bucketNum = scanNode.getBucketNum();
scanNode.getFragment().setBucketNum(bucketNum);
}
Map<Integer, TNetworkAddress> bucketSeqToAddress = fragmentIdToSeqToAddressMap.get(scanNode.getFragmentId());
@@ -2460,14 +2466,14 @@ public class Coordinator implements CoordInterface {
class BucketShuffleJoinController {
// fragment_id -> < bucket_seq -> < scannode_id -> scan_range_params >>
- private final Map<PlanFragmentId, BucketSeqToScanRange> fragmentIdBucketSeqToScanRangeMap = Maps.newHashMap();
+ protected final Map<PlanFragmentId, BucketSeqToScanRange> fragmentIdBucketSeqToScanRangeMap = Maps.newHashMap();
// fragment_id -> < bucket_seq -> be_addresss >
private final Map<PlanFragmentId, Map<Integer, TNetworkAddress>> fragmentIdToSeqToAddressMap
= Maps.newHashMap();
// fragment_id -> < be_id -> bucket_count >
private final Map<PlanFragmentId, Map<Long, Integer>> fragmentIdToBuckendIdBucketCountMap = Maps.newHashMap();
// fragment_id -> bucket_num
- private final Map<PlanFragmentId, Integer> fragmentIdToBucketNumMap = Maps.newHashMap();
+ protected final Map<PlanFragmentId, Integer> fragmentIdToBucketNumMap = Maps.newHashMap();
// cache the bucketShuffleFragmentIds
private final Set<Integer> bucketShuffleFragmentIds = new HashSet<>();
@@ -2480,7 +2486,7 @@ public class Coordinator implements CoordInterface {
}
// check whether the node fragment is bucket shuffle join fragment
- private boolean isBucketShuffleJoin(int fragmentId, PlanNode node) {
+ protected boolean isBucketShuffleJoin(int fragmentId, PlanNode node) {
if (ConnectContext.get() != null) {
if (!ConnectContext.get().getSessionVariable().isEnableBucketShuffleJoin()
&& !ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()) {
@@ -2565,18 +2571,7 @@ public class Coordinator implements CoordInterface {
Map<TNetworkAddress, Long> addressToBackendID,
Map<TNetworkAddress, Long> replicaNumPerHost) throws Exception {
if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) {
- // In bucket shuffle join, we have 2 situation.
- // 1. Only one partition: in this case, we use scanNode.getTotalTabletsNum() to get the right bucket num
- // because when table turn on dynamic partition, the bucket number in default distribution info
- // is not correct.
- // 2. Table is colocated: in this case, table could have more than one partition, but all partition's
- // bucket number must be same, so we use default bucket num is ok.
- int bucketNum = 0;
- if (scanNode.getOlapTable().isColocateTable()) {
- bucketNum = scanNode.getOlapTable().getDefaultDistributionInfo().getBucketNum();
- } else {
- bucketNum = (int) (scanNode.getTotalTabletsNum());
- }
+ int bucketNum = scanNode.getBucketNum();
fragmentIdToBucketNumMap.put(scanNode.getFragmentId(), bucketNum);
fragmentIdToSeqToAddressMap.put(scanNode.getFragmentId(), new HashMap<>());
fragmentIdBucketSeqToScanRangeMap.put(scanNode.getFragmentId(), new BucketSeqToScanRange());
@@ -2731,11 +2726,11 @@ public class Coordinator implements CoordInterface {
}
private final Map<PlanFragmentId, BucketSeqToScanRange> fragmentIdTobucketSeqToScanRangeMap = Maps.newHashMap();
- private final Map<PlanFragmentId, Map<Integer, TNetworkAddress>> fragmentIdToSeqToAddressMap = Maps.newHashMap();
+ protected final Map<PlanFragmentId, Map<Integer, TNetworkAddress>> fragmentIdToSeqToAddressMap = Maps.newHashMap();
// cache the fragment id to its scan node ids. Used for colocate join.
private final Map<PlanFragmentId, Set<Integer>> fragmentIdToScanNodeIds = Maps.newHashMap();
private final Set<Integer> colocateFragmentIds = new HashSet<>();
- private final BucketShuffleJoinController bucketShuffleJoinController
+ protected final BucketShuffleJoinController bucketShuffleJoinController
= new BucketShuffleJoinController(fragmentIdToScanNodeIds);
public class PipelineExecContext {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
new file mode 100644
index 00000000000..4f5af3762c5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.qe;
+
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.datasource.FileQueryScanNode;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.stats.StatsErrorEstimator;
+import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping;
+import org.apache.doris.nereids.trees.plans.distribute.PipelineDistributedPlan;
+import org.apache.doris.nereids.trees.plans.distribute.worker.DistributedPlanWorker;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.AssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.BucketScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanRanges;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.ScanSource;
+import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.planner.Planner;
+import org.apache.doris.planner.ScanNode;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TScanRangeParams;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+/** NereidsCoordinator */
+public class NereidsCoordinator extends Coordinator {
+ private NereidsPlanner nereidsPlanner;
+ private FragmentIdMapping<DistributedPlan> distributedPlans;
+
+ public NereidsCoordinator(ConnectContext context, Analyzer analyzer,
+ Planner planner, StatsErrorEstimator statsErrorEstimator, NereidsPlanner nereidsPlanner) {
+ super(context, analyzer, planner, statsErrorEstimator);
+ this.nereidsPlanner = Objects.requireNonNull(nereidsPlanner, "nereidsPlanner can not be null");
+ this.distributedPlans = Objects.requireNonNull(
+ nereidsPlanner.getDistributedPlans(), "distributedPlans can not be null"
+ );
+ }
+
+ @Override
+ protected void processFragmentAssignmentAndParams() throws Exception {
+ // prepare information
+ prepare();
+
+ computeFragmentExecParams();
+ }
+
+ @Override
+ protected void computeFragmentHosts() {
+ // translate distributed plan to params
+ for (DistributedPlan distributedPlan : distributedPlans.values()) {
+ UnassignedJob fragmentJob = distributedPlan.getFragmentJob();
+ PlanFragment fragment = fragmentJob.getFragment();
+
+ bucketShuffleJoinController
+ .isBucketShuffleJoin(fragment.getFragmentId().asInt(), fragment.getPlanRoot());
+
+ setFileScanParams(distributedPlan);
+
+ FragmentExecParams fragmentExecParams = fragmentExecParamsMap.computeIfAbsent(
+ fragment.getFragmentId(), id -> new FragmentExecParams(fragment)
+ );
+ List<AssignedJob> instanceJobs = ((PipelineDistributedPlan) distributedPlan).getInstanceJobs();
+ boolean useLocalShuffle = useLocalShuffle(distributedPlan);
+ if (useLocalShuffle) {
+ fragmentExecParams.ignoreDataDistribution = true;
+ fragmentExecParams.parallelTasksNum = 1;
+ } else {
+ fragmentExecParams.parallelTasksNum = instanceJobs.size();
+ }
+
+ for (AssignedJob instanceJob : instanceJobs) {
+ DistributedPlanWorker worker = instanceJob.getAssignedWorker();
+ TNetworkAddress address = new TNetworkAddress(worker.host(), worker.port());
+ FInstanceExecParam instanceExecParam = new FInstanceExecParam(
+ null, address, 0, fragmentExecParams);
+ instanceExecParam.instanceId = instanceJob.instanceId();
+ fragmentExecParams.instanceExecParams.add(instanceExecParam);
+ addressToBackendID.put(address, worker.id());
+ ScanSource scanSource = instanceJob.getScanSource();
+ if (scanSource instanceof BucketScanSource) {
+ setForBucketScanSource(instanceExecParam, (BucketScanSource) scanSource, useLocalShuffle);
+ } else {
+ setForDefaultScanSource(instanceExecParam, (DefaultScanSource) scanSource, useLocalShuffle);
+ }
+ }
+ }
+ }
+
+ private void setFileScanParams(DistributedPlan distributedPlan) {
+ for (ScanNode scanNode : distributedPlan.getFragmentJob().getScanNodes()) {
+ if (scanNode instanceof FileQueryScanNode) {
+ fileScanRangeParamsMap.put(
+ scanNode.getId().asInt(),
+ ((FileQueryScanNode) scanNode).getFileScanRangeParams()
+ );
+ }
+ }
+ }
+
+ private boolean useLocalShuffle(DistributedPlan distributedPlan) {
+ List<AssignedJob> instanceJobs = ((PipelineDistributedPlan) distributedPlan).getInstanceJobs();
+ for (AssignedJob instanceJob : instanceJobs) {
+ if (instanceJob instanceof LocalShuffleAssignedJob) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void setForDefaultScanSource(
+ FInstanceExecParam instanceExecParam, DefaultScanSource scanSource, boolean isShareScan) {
+ for (Entry<ScanNode, ScanRanges> scanNodeIdToReplicaIds : scanSource.scanNodeToScanRanges.entrySet()) {
+ ScanNode scanNode = scanNodeIdToReplicaIds.getKey();
+ ScanRanges scanReplicas = scanNodeIdToReplicaIds.getValue();
+ instanceExecParam.perNodeScanRanges.put(scanNode.getId().asInt(), scanReplicas.params);
+ instanceExecParam.perNodeSharedScans.put(scanNode.getId().asInt(), isShareScan);
+ }
+ }
+
+ private void setForBucketScanSource(FInstanceExecParam instanceExecParam,
+ BucketScanSource bucketScanSource, boolean isShareScan) {
+ for (Entry<Integer, Map<ScanNode, ScanRanges>> bucketIndexToScanTablets :
+ bucketScanSource.bucketIndexToScanNodeToTablets.entrySet()) {
+ Integer bucketIndex = bucketIndexToScanTablets.getKey();
+ instanceExecParam.addBucketSeq(bucketIndex);
+ Map<ScanNode, ScanRanges> scanNodeToRangeMap = bucketIndexToScanTablets.getValue();
+ for (Entry<ScanNode, ScanRanges> scanNodeToRange : scanNodeToRangeMap.entrySet()) {
+ ScanNode scanNode = scanNodeToRange.getKey();
+ ScanRanges scanRanges = scanNodeToRange.getValue();
+ List<TScanRangeParams> scanBucketTablets = instanceExecParam.perNodeScanRanges.computeIfAbsent(
+ scanNode.getId().asInt(), id -> Lists.newArrayList());
+ scanBucketTablets.addAll(scanRanges.params);
+ instanceExecParam.perNodeSharedScans.put(scanNode.getId().asInt(), isShareScan);
+
+ if (scanNode instanceof OlapScanNode) {
+ OlapScanNode olapScanNode = (OlapScanNode) scanNode;
+ if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) {
+ int bucketNum = olapScanNode.getBucketNum();
+ fragmentIdToSeqToAddressMap.put(olapScanNode.getFragmentId(), new HashMap<>());
+ bucketShuffleJoinController.fragmentIdBucketSeqToScanRangeMap
+ .put(scanNode.getFragmentId(), new BucketSeqToScanRange());
+ bucketShuffleJoinController.fragmentIdToBucketNumMap
+ .put(scanNode.getFragmentId(), bucketNum);
+ olapScanNode.getFragment().setBucketNum(bucketNum);
+ }
+ } else if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) {
+ int bucketNum = 1;
+ fragmentIdToSeqToAddressMap.put(scanNode.getFragmentId(), new HashMap<>());
+ bucketShuffleJoinController.fragmentIdBucketSeqToScanRangeMap
+ .put(scanNode.getFragmentId(), new BucketSeqToScanRange());
+ bucketShuffleJoinController.fragmentIdToBucketNumMap
+ .put(scanNode.getFragmentId(), bucketNum);
+ scanNode.getFragment().setBucketNum(bucketNum);
+ }
+
+ BucketSeqToScanRange bucketSeqToScanRange = bucketShuffleJoinController
+ .fragmentIdBucketSeqToScanRangeMap.get(scanNode.getFragmentId());
+
+ Map<Integer, List<TScanRangeParams>> scanNodeIdToReplicas
+ = bucketSeqToScanRange.computeIfAbsent(bucketIndex, set -> Maps.newLinkedHashMap());
+ List<TScanRangeParams> tablets = scanNodeIdToReplicas.computeIfAbsent(
+ scanNode.getId().asInt(), id -> new ArrayList<>());
+ tablets.addAll(scanRanges.params);
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index f49e4f19d31..d835dc86128 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -18,6 +18,7 @@
package org.apache.doris.qe;
import org.apache.doris.analysis.SetVar;
+import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
@@ -26,10 +27,14 @@ import org.apache.doris.common.VariableAnnotation;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.analyzer.UnboundResultSink;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.metrics.Event;
import org.apache.doris.nereids.metrics.EventSwitchParser;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.planner.GroupCommitBlockSink;
import org.apache.doris.qe.VariableMgr.VarAttr;
import org.apache.doris.thrift.TGroupCommitMode;
@@ -304,6 +309,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String NTH_OPTIMIZED_PLAN = "nth_optimized_plan";
public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner";
+ public static final String ENABLE_NEREIDS_DISTRIBUTE_PLANNER = "enable_nereids_distribute_planner";
public static final String DISABLE_NEREIDS_RULES = "disable_nereids_rules";
public static final String ENABLE_NEREIDS_RULES = "enable_nereids_rules";
public static final String ENABLE_NEW_COST_MODEL = "enable_new_cost_model";
@@ -1231,7 +1237,7 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = DISABLE_NEREIDS_RULES, needForward = true)
private String disableNereidsRules = "";
- @VariableMgr.VarAttr(name = ENABLE_NEREIDS_RULES, needForward = true, varType = VariableAnnotation.REMOVED)
+ @VariableMgr.VarAttr(name = ENABLE_NEREIDS_RULES, needForward = true)
public String enableNereidsRules = "";
@VariableMgr.VarAttr(name = ENABLE_NEW_COST_MODEL, needForward = true)
@@ -1243,6 +1249,17 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = NEREIDS_STAR_SCHEMA_SUPPORT)
private boolean nereidsStarSchemaSupport = true;
+ @VariableMgr.VarAttr(name = ENABLE_NEREIDS_DISTRIBUTE_PLANNER, needForward = true,
+ fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
+ "使用新的nereids的分布式规划器的开关，这个分布式规划器可以规划出一些更高效的查询计划，比如在某些情况下，"
+ + "可以把左表shuffle到右表去做bucket shuffle join",
+ "The switch to use new DistributedPlanner of nereids, this planner can planning some "
+ + "more efficient query plans, e.g. in certain situations, shuffle left side to "
+ + "right side to do bucket shuffle join"
+ }
+ )
+ private boolean enableNereidsDistributePlanner = false;
+
@VariableMgr.VarAttr(name = REWRITE_OR_TO_IN_PREDICATE_THRESHOLD, fuzzy = true)
private int rewriteOrToInPredicateThreshold = 2;
@@ -3068,6 +3085,41 @@ public class SessionVariable implements Serializable, Writable {
this.enableNereidsPlanner = enableNereidsPlanner;
}
+ /** canUseNereidsDistributePlanner */
+ public static boolean canUseNereidsDistributePlanner() {
+ // TODO: support cloud mode
+ if (Config.isCloudMode()) {
+ return false;
+ }
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext == null) {
+ return false;
+ }
+ StatementContext statementContext = connectContext.getStatementContext();
+ if (statementContext == null) {
+ return false;
+ }
+ StatementBase parsedStatement = statementContext.getParsedStatement();
+ if (!(parsedStatement instanceof LogicalPlanAdapter)) {
+ return false;
+ }
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStatement).getLogicalPlan();
+ SessionVariable sessionVariable = connectContext.getSessionVariable();
+ // TODO: support other sink
+ if (logicalPlan instanceof UnboundResultSink && sessionVariable.enableNereidsDistributePlanner) {
+ return true;
+ }
+ return false;
+ }
+
+ public boolean isEnableNereidsDistributePlanner() {
+ return enableNereidsDistributePlanner;
+ }
+
+ public void setEnableNereidsDistributePlanner(boolean enableNereidsDistributePlanner) {
+ this.enableNereidsDistributePlanner = enableNereidsDistributePlanner;
+ }
+
public int getNthOptimizedPlan() {
return nthOptimizedPlan;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 021c5fa36ff..8114df770d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1876,9 +1876,17 @@ public class StmtExecutor {
// this branch is for legacy planner, to be removed
coordBase = new PointQueryExec(planner, analyzer,
context.getSessionVariable().getMaxMsgSizeOfResultReceiver());
+ } else if (planner instanceof NereidsPlanner && ((NereidsPlanner) planner).getDistributedPlans() != null) {
+ coord = new NereidsCoordinator(context, analyzer,
+ planner, context.getStatsErrorEstimator(),
+ (NereidsPlanner) planner);
+ profile.addExecutionProfile(coord.getExecutionProfile());
+ QeProcessorImpl.INSTANCE.registerQuery(context.queryId(),
+ new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord));
+ coordBase = coord;
} else {
- coord = EnvFactory.getInstance().createCoordinator(context, analyzer,
- planner, context.getStatsErrorEstimator());
+ coord = EnvFactory.getInstance().createCoordinator(
+ context, analyzer, planner, context.getStatsErrorEstimator());
profile.addExecutionProfile(coord.getExecutionProfile());
QeProcessorImpl.INSTANCE.registerQuery(context.queryId(),
new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
index 0414b5d1489..86c460e6bfd 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java
@@ -517,8 +517,13 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase {
init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG));
StatementBase parseStmt = parseAndAnalyzeStmt("select * from hms_ctl.hms_db.hms_tbl", connectContext);
List<ScanNode> scanNodes = Arrays.asList(hiveScanNode1);
+
+ CacheAnalyzer ca2 = new CacheAnalyzer(connectContext, parseStmt, scanNodes);
+ ca2.checkCacheMode(0);
+ long latestPartitionTime = ca2.getLatestTable().latestPartitionTime;
+
CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes);
- ca.checkCacheMode(0);
+ ca.checkCacheMode(latestPartitionTime);
Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode());
}
@@ -527,8 +532,13 @@ public class HmsQueryCacheTest extends AnalyzeCheckTestBase {
init((HMSExternalCatalog) mgr.getCatalog(HMS_CATALOG));
StatementBase parseStmt = analyzeAndGetStmtByNereids("select * from hms_ctl.hms_db.hms_tbl", connectContext);
List<ScanNode> scanNodes = Arrays.asList(hiveScanNode1);
+
+ CacheAnalyzer ca2 = new CacheAnalyzer(connectContext, parseStmt, scanNodes);
+ ca2.checkCacheModeForNereids(0);
+ long latestPartitionTime = ca2.getLatestTable().latestPartitionTime;
+
CacheAnalyzer ca = new CacheAnalyzer(connectContext, parseStmt, scanNodes);
- ca.checkCacheModeForNereids(0);
+ ca.checkCacheModeForNereids(latestPartitionTime);
Assert.assertEquals(CacheAnalyzer.CacheMode.None, ca.getCacheMode());
}
diff --git a/regression-test/data/nereids_syntax_p0/distribute/colocate_union_numbers.out b/regression-test/data/nereids_syntax_p0/distribute/colocate_union_numbers.out
new file mode 100644
index 00000000000..6d15eb60047
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/colocate_union_numbers.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !union_all --
+0
+0
+1
+1
+2
+2
+3
+
diff --git a/regression-test/data/nereids_syntax_p0/distribute/local_shuffle.out b/regression-test/data/nereids_syntax_p0/distribute/local_shuffle.out
new file mode 100644
index 00000000000..82ed128f756
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/local_shuffle.out
@@ -0,0 +1,36 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !read_single_olap_table --
+1 1
+2 2
+
+-- !broadcast_join --
+2 2 2 2
+
+-- !shuffle_join --
+2 2 2 2
+
+-- !bucket_shuffle_join --
+2 2 2 2
+
+-- !colocate_join --
+2 2 2 2
+
+-- !bucket_shuffle_with_prune_tablets --
+\N \N 2 2
+\N \N 3 3
+
+-- !bucket_shuffle_with_prune_tablets2 --
+2 2 \N \N
+3 3 \N \N
+
+-- !bucket_shuffle_with_prune_tablets3 --
+1 1 \N \N
+
+-- !fillup_bucket --
+\N 2
+\N 3
+
+-- !shuffle_left --
+1 1 1
+2 2 2
+
diff --git a/regression-test/data/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.out b/regression-test/data/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.out
new file mode 100644
index 00000000000..acbf30fb3b1
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !fillup_bucket --
+\N 2.000
+\N 3.000
+
diff --git a/regression-test/data/nereids_syntax_p0/distribute/query_constant.out b/regression-test/data/nereids_syntax_p0/distribute/query_constant.out
new file mode 100644
index 00000000000..ffdd0b7bd9b
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/query_constant.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query_one_row --
+100 abc
+
+-- !union_all --
+100 hello
+200 world
+
+-- !union --
+100 hello
+200 world
+
diff --git a/regression-test/data/nereids_syntax_p0/distribute/shuffle.out b/regression-test/data/nereids_syntax_p0/distribute/shuffle.out
new file mode 100644
index 00000000000..e94e38f150e
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/shuffle.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !4_phase_agg --
+1 2 2
+2 2 2
+3 2 2
+4 2 2
+5 2 2
+
diff --git a/regression-test/data/nereids_syntax_p0/distribute/shuffle_left_join.out b/regression-test/data/nereids_syntax_p0/distribute/shuffle_left_join.out
new file mode 100644
index 00000000000..99d095d87f7
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/distribute/shuffle_left_join.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !shuffle_left_and_right --
+1 1 1 1
+2 2 2 2
+
+-- !shuffle_left --
+1 1 1 1
+2 2 2 2
+
diff --git a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
index 9270b80aaba..9473b88fd65 100644
--- a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
+++ b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
@@ -47,7 +47,7 @@ suite("test_forbid_unknown_col_stats") {
test{
sql "select * from test_forbid_unknown_col_stats_tbl;"
- exception "tables with unknown column stats: OlapScanNode{tid=0, tblName=test_forbid_unknown_col_stats_tbl, keyRanges=, preds= limit=-1}"
+ exception "tables with unknown column stats: OlapScanNode{id=0, tid=0, tblName=test_forbid_unknown_col_stats_tbl, keyRanges=, preds= limit=-1}"
}
sql "select count() from __internal_schema.column_statistics"
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy b/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
new file mode 100644
index 00000000000..b247c5bde25
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("colocate_union_numbers") {
+ multi_sql """
+ set enable_nereids_distribute_planner=false;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ set enable_local_shuffle=false;
+ set force_to_local_shuffle=false;
+ """
+
+ def extractFragment = { String sqlStr, String containsString, Closure<Integer> checkExchangeNum ->
+ explain {
+ sql sqlStr
+ check { result ->
+ log.info("Explain result:\n${result}")
+
+ assertTrue(result.contains(containsString))
+
+ def fragmentContainsJoin = result.split("PLAN FRAGMENT")
+ .toList()
+ .stream()
+ .filter { it.contains(containsString) }
+ .findFirst()
+ .get()
+
+ log.info("Fragment:\n${fragmentContainsJoin}")
+
+ checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+ }
+ }
+ }
+
+ def sqlStr = """
+ select * from numbers('number'='3')a
+ union all
+ select * from numbers('number'='4')b
+ """
+
+ extractFragment(sqlStr, "VUNION") { exchangeNum ->
+ assertTrue(exchangeNum == 2)
+ }
+
+ multi_sql """
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ """
+
+ extractFragment(sqlStr, "VUNION") { exchangeNum ->
+ assertTrue(exchangeNum == 0)
+ }
+
+ order_qt_union_all sqlStr
+}
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy b/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy
new file mode 100644
index 00000000000..997230b1a06
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("local_shuffle") {
+ multi_sql """
+ drop table if exists test_local_shuffle1;
+ drop table if exists test_local_shuffle2;
+
+ CREATE TABLE `test_local_shuffle1` (
+ id int,
+ id2 int
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "colocate_with" = "test_local_shuffle_with_colocate"
+ );
+
+ CREATE TABLE `test_local_shuffle2` (
+ id int,
+ id2 int
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "colocate_with" = "test_local_shuffle_with_colocate"
+ );
+
+ insert into test_local_shuffle1 values (1, 1), (2, 2);
+ insert into test_local_shuffle2 values (2, 2), (3, 3);
+
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ set enable_local_shuffle=true;
+ set force_to_local_shuffle=true;
+ """
+
+ order_qt_read_single_olap_table "select * from test_local_shuffle1"
+
+ order_qt_broadcast_join """
+ select *
+ from test_local_shuffle1
+ join [broadcast]
+ test_local_shuffle2
+ on test_local_shuffle1.id=test_local_shuffle2.id
+ """
+
+ order_qt_shuffle_join """
+ select *
+ from test_local_shuffle1
+ join [shuffle]
+ test_local_shuffle2
+ on test_local_shuffle1.id2=test_local_shuffle2.id2
+ """
+
+ order_qt_bucket_shuffle_join """
+ select *
+ from test_local_shuffle1
+ join [shuffle]
+ test_local_shuffle2
+ on test_local_shuffle1.id2=test_local_shuffle2.id2
+ """
+
+ order_qt_colocate_join """
+ select *
+ from test_local_shuffle1
+ join [shuffle]
+ test_local_shuffle2
+ on test_local_shuffle1.id=test_local_shuffle2.id
+ """
+
+ order_qt_bucket_shuffle_with_prune_tablets """
+ select *
+ from
+ (
+ select *
+ from test_local_shuffle1
+ where id=1
+ ) a
+ right outer join [shuffle]
+ test_local_shuffle2
+ on a.id=test_local_shuffle2.id2
+ """
+
+ order_qt_bucket_shuffle_with_prune_tablets2 """
+ select *
+ from
+ test_local_shuffle2
+ left outer join [shuffle]
+ (
+ select *
+ from test_local_shuffle1
+ where id=1
+ ) a
+ on a.id=test_local_shuffle2.id2
+ """
+
+ order_qt_bucket_shuffle_with_prune_tablets3 """
+ select *
+ from
+ (
+ select *
+ from test_local_shuffle1
+ where id=1
+ ) a
+ left outer join [shuffle]
+ test_local_shuffle2
+ on a.id=test_local_shuffle2.id2
+ """
+
+ multi_sql """
+ drop table if exists test_local_shuffle3;
+ CREATE TABLE IF NOT EXISTS test_local_shuffle3 (
+ c0 int
+ )
+ DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1");
+
+ drop table if exists test_local_shuffle4;
+ CREATE TABLE IF NOT EXISTS test_local_shuffle4 (
+ c0 int
+ )
+ DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1");
+ INSERT INTO test_local_shuffle3 (c0) VALUES (1), (3);
+ INSERT INTO test_local_shuffle4 (c0) VALUES (2), (3);
+
+ sync;
+
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ set enable_local_shuffle=true;
+ set force_to_local_shuffle=true;
+ """
+
+ order_qt_fillup_bucket """
+ SELECT cast(a.c0 as int), cast(b.c0 as int) FROM
+ (select * from test_local_shuffle3 where c0 =1)a
+ RIGHT OUTER JOIN
+ (select * from test_local_shuffle4)b
+ ON a.c0 = b.c0
+ """
+
+ multi_sql """
+ drop table if exists test_shuffle_left_with_local_shuffle;
+ CREATE TABLE `test_shuffle_left_with_local_shuffle` (
+ id int,
+ id2 int
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ insert into test_shuffle_left_with_local_shuffle values (1, 1), (2, 2), (3, 4);
+ """
+
+ order_qt_shuffle_left """
+ select *
+ from
+ (
+ select id2
+ from test_shuffle_left_with_local_shuffle
+ group by id2
+ ) a
+ inner join [shuffle]
+ test_shuffle_left_with_local_shuffle b
+ on a.id2=b.id;
+ """
+}
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
new file mode 100644
index 00000000000..3ce412a0075
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("prune_bucket_with_bucket_shuffle_join") {
+ multi_sql """
+ drop table if exists test_outer_join1;
+ CREATE TABLE IF NOT EXISTS test_outer_join1 (
+ c0 DECIMALV3(8,3)
+ )
+ DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1");
+
+ drop table if exists test_outer_join2;
+ CREATE TABLE IF NOT EXISTS test_outer_join2 (
+ c0 DECIMALV3(8,3)
+ )
+ DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1");
+ INSERT INTO test_outer_join1 (c0) VALUES (1), (3);
+ INSERT INTO test_outer_join2 (c0) VALUES (2), (3);
+
+ sync;
+
+ set enable_nereids_distribute_planner=false;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ set enable_local_shuffle=false;
+ set force_to_local_shuffle=false;
+ """
+
+ def extractFragment = { String sqlStr, String containsString, Closure<Integer> checkExchangeNum ->
+ explain {
+ sql sqlStr
+ check { result ->
+ log.info("Explain result:\n${result}")
+
+ assertTrue(result.contains(containsString))
+
+ def fragmentContainsJoin = result.split("PLAN FRAGMENT")
+ .toList()
+ .stream()
+ .filter { it.contains(containsString) }
+ .findFirst()
+ .get()
+
+ log.info("Fragment:\n${fragmentContainsJoin}")
+
+ checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+ }
+ }
+ }
+
+ String sqlStr = """
+ SELECT * FROM
+ (select * from test_outer_join1 where c0 =1)a
+ RIGHT OUTER JOIN
+ (select * from test_outer_join2)b
+ ON a.c0 = b.c0
+ """
+
+ extractFragment(sqlStr, "RIGHT OUTER JOIN(PARTITIONED)") { exchangeNum ->
+ assertTrue(exchangeNum > 1)
+ }
+
+ multi_sql """
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ """
+
+ extractFragment(sqlStr, "RIGHT OUTER JOIN(BUCKET_SHUFFLE)") { exchangeNum ->
+ assertTrue(exchangeNum == 1)
+ }
+
+ explain {
+ sql "distributed plan ${sqlStr}"
+ check { explainStr ->
+ log.info("Distributed plan:\n${explainStr}")
+
+ // some tablets of left table are pruned
+ assertTrue(explainStr.count("tablet ") < 20)
+ }
+ }
+
+ order_qt_fillup_bucket sqlStr
+}
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy b/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy
new file mode 100644
index 00000000000..19521294975
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("query_constant") {
+ multi_sql """
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set enable_local_shuffle=false;
+ set force_to_local_shuffle=false;
+ """
+
+ order_qt_query_one_row "select 100 id, 'abc' name"
+
+ order_qt_union_all """
+ select 100 id, 'hello' name
+ union all
+ select 200 id, 'world' name
+ """
+
+ order_qt_union """
+ select 100 id, 'hello' name
+ union
+ select 200 id, 'world' name
+ """
+}
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy b/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy
new file mode 100644
index 00000000000..42ec0df0545
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("shuffle") {
+ createTestTable "test_shuffle"
+
+ multi_sql """
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set enable_local_shuffle=false;
+ set force_to_local_shuffle=false;
+ """
+
+ order_qt_4_phase_agg """
+ select /*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT,TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI,THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI,THREE_PHASE_AGGREGATE_WITH_DISTINCT,FOUR_PHASE_AGGREGATE_WITH_DISTINCT')*/
+ id, count(distinct value)
+ from test_shuffle
+ group by id
+ """
+}
diff --git a/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy
new file mode 100644
index 00000000000..8c56c257b0e
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy
@@ -0,0 +1,117 @@
+import java.util.stream.Collectors
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("shuffle_left_join") {
+ multi_sql """
+ drop table if exists test_shuffle_left;
+
+ CREATE TABLE `test_shuffle_left` (
+ id int,
+ id2 int
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ insert into test_shuffle_left values (1, 1), (2, 2), (3, 4);
+
+ sync;
+
+ set enable_nereids_distribute_planner=false;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ set enable_local_shuffle=false;
+ set force_to_local_shuffle=false;
+ """
+
+ def extractFragment = { String sqlStr, String containsString, Closure<Integer> checkExchangeNum ->
+ explain {
+ sql sqlStr
+ check { result ->
+ log.info("Explain result:\n${result}")
+
+ assertTrue(result.contains(containsString))
+
+ def fragmentContainsJoin = result.split("PLAN FRAGMENT")
+ .toList()
+ .stream()
+ .filter { it.contains(containsString) }
+ .findFirst()
+ .get()
+
+ log.info("Fragment:\n${fragmentContainsJoin}")
+
+ checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE"))
+ }
+ }
+ }
+
+ def sqlStr = """
+ select *
+ from
+ (
+ select id2
+ from test_shuffle_left
+ group by id2
+ ) a
+ inner join [shuffle]
+ test_shuffle_left b
+ on a.id2=b.id;
+ """
+
+ extractFragment(sqlStr, "INNER JOIN(PARTITIONED)") { exchangeNum ->
+ assertTrue(exchangeNum == 2)
+ }
+
+ order_qt_shuffle_left_and_right sqlStr
+
+ multi_sql """
+ set enable_nereids_distribute_planner=true;
+ set enable_pipeline_x_engine=true;
+ set disable_join_reorder=true;
+ """
+
+ def variables = sql "show variables"
+ def variableString = variables.stream()
+ .map { it.toString() }
+ .collect(Collectors.joining("\n"))
+ logger.info("Variables:\n${variableString}")
+
+ extractFragment(sqlStr, "INNER JOIN(BUCKET_SHUFFLE)") { exchangeNum ->
+ assertTrue(exchangeNum == 1)
+ }
+
+ explain {
+ sql "plan $sqlStr"
+ check { explainStr ->
+ log.info("explain plan:\n${explainStr}")
+ }
+ }
+
+ def rows = sql sqlStr
+ def rowsString = rows.stream()
+ .map { it.toString() }
+ .collect(Collectors.joining("\n"))
+ logger.info("Rows:\n${rowsString}")
+
+
+ order_qt_shuffle_left sqlStr
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 11/21: [improvement](jdbc catalog) Modify the maximum number of connections in the connection pool to 30 by default (#36720)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1cbc322d0c548d04cde2dd52ed7a462072a36af4
Author: zy-kkk <zh...@gmail.com>
AuthorDate: Fri Jun 28 20:13:34 2024 +0800
[improvement](jdbc catalog) Modify the maximum number of connections in the connection pool to 30 by default (#36720)
In many cases, we found that users would use JDBC Catalog to perform a
large number of queries, which resulted in the maximum of 10 connections
being insufficient, so I adjusted it to 30, which covered most needs.
---
.../src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java | 2 +-
.../src/test/java/org/apache/doris/catalog/JdbcResourceTest.java | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java
index 5fdbc211ab0..a99377add25 100644
--- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java
+++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSourceConfig.java
@@ -31,7 +31,7 @@ public class JdbcDataSourceConfig {
private TJdbcOperation op;
private TOdbcTableType tableType;
private int connectionPoolMinSize = 1;
- private int connectionPoolMaxSize = 10;
+ private int connectionPoolMaxSize = 30;
private int connectionPoolMaxWaitTime = 5000;
private int connectionPoolMaxLifeTime = 1800000;
private boolean connectionPoolKeepAlive = false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index 3878500f917..1db801b024a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -140,7 +140,7 @@ public class JdbcResource extends Resource {
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(INCLUDE_DATABASE_LIST, "");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(EXCLUDE_DATABASE_LIST, "");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MIN_SIZE, "1");
- OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_SIZE, "10");
+ OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_SIZE, "30");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_LIFE_TIME, "1800000");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_WAIT_TIME, "5000");
OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_KEEP_ALIVE, "false");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java
index 8e004d4b236..81c2157686a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java
@@ -87,7 +87,7 @@ public class JdbcResourceTest {
// Verify the default properties were applied during the replay
Map<String, String> properties = jdbcResource.getCopiedProperties();
Assert.assertEquals("1", properties.get("connection_pool_min_size"));
- Assert.assertEquals("10", properties.get("connection_pool_max_size"));
+ Assert.assertEquals("30", properties.get("connection_pool_max_size"));
Assert.assertEquals("1800000", properties.get("connection_pool_max_life_time"));
Assert.assertEquals("5000", properties.get("connection_pool_max_wait_time"));
Assert.assertEquals("false", properties.get("connection_pool_keep_alive"));
@@ -110,7 +110,7 @@ public class JdbcResourceTest {
// Verify the default properties were applied during the replay
Map<String, String> properties = replayedResource.getCopiedProperties();
Assert.assertEquals("1", properties.get("connection_pool_min_size"));
- Assert.assertEquals("10", properties.get("connection_pool_max_size"));
+ Assert.assertEquals("30", properties.get("connection_pool_max_size"));
Assert.assertEquals("1800000", properties.get("connection_pool_max_life_time"));
Assert.assertEquals("5000", properties.get("connection_pool_max_wait_time"));
Assert.assertEquals("false", properties.get("connection_pool_keep_alive"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 15/21: [bugfix](testcase)add java error log output (#35998)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 66ca7e94c834a0ebdd3ea9976c7685bd87d932d5
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Fri Jun 28 21:56:43 2024 +0800
[bugfix](testcase)add java error log output (#35998)
In the testcase pipeline, this spark-connector case has a 4% chance of
error, but since there is no error log, the cause of the error cannot be
located.
Therefore, an error log is added to facilitate problem location later.
---
.../suites/connector_p0/spark_connector/spark_connector.groovy | 10 ++++++++--
.../connector_p0/spark_connector/spark_connector_arrow.groovy | 10 ++++++++--
.../spark_connector/spark_connector_read_type.groovy | 10 ++++++++--
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy b/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
index ecd4e6dfc14..2bd618fcc3c 100644
--- a/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
+++ b/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
@@ -28,7 +28,13 @@ suite("spark_connector", "connector") {
logger.info("finish download spark doris demo ...")
def run_cmd = "java -jar spark-doris-demo.jar $context.config.feHttpAddress $context.config.feHttpUser regression_test_connector_p0_spark_connector.$tableName"
logger.info("run_cmd : $run_cmd")
- def run_spark_jar = run_cmd.execute().getText()
- logger.info("result: $run_spark_jar")
+ def proc = run_cmd.execute()
+ def sout = new StringBuilder()
+ def serr = new StringBuilder()
+ proc.consumeProcessOutput(sout, serr)
+ proc.waitForOrKill(1200_000)
+ if (proc.exitValue() != 0) {
+ logger.warn("failed to execute jar: code=${proc.exitValue()}, " + "output: ${sout.toString()}, error: ${serr.toString()}")
+ }
qt_select """ select * from $tableName order by order_id"""
}
diff --git a/regression-test/suites/connector_p0/spark_connector/spark_connector_arrow.groovy b/regression-test/suites/connector_p0/spark_connector/spark_connector_arrow.groovy
index 1cd2ed31d2e..a5fbc3b2835 100644
--- a/regression-test/suites/connector_p0/spark_connector/spark_connector_arrow.groovy
+++ b/regression-test/suites/connector_p0/spark_connector/spark_connector_arrow.groovy
@@ -136,8 +136,14 @@ suite("spark_connector_for_arrow", "connector") {
logger.info("finish download spark doris demo ...")
def run_cmd = "java -cp ${jar_name} org.apache.doris.spark.testcase.TestStreamLoadForArrowType $context.config.feHttpAddress $context.config.feHttpUser regression_test_connector_p0_spark_connector"
logger.info("run_cmd : $run_cmd")
- def run_spark_jar = run_cmd.execute().getText()
- logger.info("result: $run_spark_jar")
+ def proc = run_cmd.execute()
+ def sout = new StringBuilder()
+ def serr = new StringBuilder()
+ proc.consumeProcessOutput(sout, serr)
+ proc.waitForOrKill(1200_000)
+ if (proc.exitValue() != 0) {
+ logger.warn("failed to execute jar: code=${proc.exitValue()}, " + "output: ${sout.toString()}, error: ${serr.toString()}")
+ }
qt_q01 """ select * from spark_connector_primitive """
qt_q02 """ select * from spark_connector_array """
diff --git a/regression-test/suites/connector_p0/spark_connector/spark_connector_read_type.groovy b/regression-test/suites/connector_p0/spark_connector/spark_connector_read_type.groovy
index 32a3ebf68c7..632e5e3d401 100644
--- a/regression-test/suites/connector_p0/spark_connector/spark_connector_read_type.groovy
+++ b/regression-test/suites/connector_p0/spark_connector/spark_connector_read_type.groovy
@@ -99,8 +99,14 @@ suite("spark_connector_read_type", "connector") {
logger.info("finish download spark doris demo ...")
def run_cmd = "java -jar spark-doris-read.jar $context.config.feHttpAddress $context.config.feHttpUser regression_test_connector_p0_spark_connector.$tableReadName regression_test_connector_p0_spark_connector.$tableWriterName"
logger.info("run_cmd : $run_cmd")
- def run_spark_jar = run_cmd.execute().getText()
- logger.info("result: $run_spark_jar")
+ def proc = run_cmd.execute()
+ def sout = new StringBuilder()
+ def serr = new StringBuilder()
+ proc.consumeProcessOutput(sout, serr)
+ proc.waitForOrKill(1200_000)
+ if (proc.exitValue() != 0) {
+ logger.warn("failed to execute jar: code=${proc.exitValue()}, " + "output: ${sout.toString()}, error: ${serr.toString()}")
+ }
qt_select """ select * from $tableWriterName order by id"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/21: [Fix](multi-catalog) Fix the transaction is not removed in abnormal situations by removing transaction in finally block. (#36705)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 05f694a94051c69be719a49327bd1c1064e04ef6
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Fri Jun 28 17:41:20 2024 +0800
[Fix](multi-catalog) Fix the transaction is not removed in abnormal situations by removing transaction in finally block. (#36705)
[Fix] (multi-catalog) Fix the transaction is not removed in abnormal
situations by removing transaction in `finally` block.
---
.../java/org/apache/doris/transaction/HiveTransactionManager.java | 7 +++++--
.../org/apache/doris/transaction/IcebergTransactionManager.java | 7 +++++--
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java
index 838d135fa45..c48210ad452 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java
@@ -59,8 +59,11 @@ public class HiveTransactionManager implements TransactionManager {
@Override
public void rollback(long id) {
- getTransactionWithException(id).rollback();
- transactions.remove(id);
+ try {
+ getTransactionWithException(id).rollback();
+ } finally {
+ transactions.remove(id);
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java
index f4b802aaa99..f373c133685 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java
@@ -51,8 +51,11 @@ public class IcebergTransactionManager implements TransactionManager {
@Override
public void rollback(long id) {
- getTransactionWithException(id).rollback();
- transactions.remove(id);
+ try {
+ getTransactionWithException(id).rollback();
+ } finally {
+ transactions.remove(id);
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 07/21: [chore](profile) rm useless code of profile (#36915)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 13e2a19bbb59eb4532320ee6085b0f70e844c9cd
Author: zhiqiang <se...@163.com>
AuthorDate: Fri Jun 28 18:44:06 2024 +0800
[chore](profile) rm useless code of profile (#36915)
---
.../doris/common/profile/ExecutionProfile.java | 23 ----------------------
1 file changed, 23 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index 3c683fced9d..ebe41c1146f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -76,8 +76,6 @@ public class ExecutionProfile {
private Map<Integer, RuntimeProfile> fragmentProfiles;
// Profile for load channels. Only for load job.
private RuntimeProfile loadChannelProfile;
- // FragmentId -> InstanceId -> RuntimeProfile
- private Map<PlanFragmentId, Map<TUniqueId, RuntimeProfile>> fragmentInstancesProfiles;
// use to merge profile from multi be
private Map<Integer, Map<TNetworkAddress, List<RuntimeProfile>>> multiBeProfile = null;
@@ -85,8 +83,6 @@ public class ExecutionProfile {
// Not serialize this property, it is only used to get profile id.
private SummaryProfile summaryProfile;
- // BE only has instance id, does not have fragmentid, so should use this map to find fragmentid.
- private Map<TUniqueId, PlanFragmentId> instanceIdToFragmentId;
private Map<Integer, Integer> fragmentIdBeNum;
private Map<Integer, Integer> seqNoToFragmentId;
@@ -112,8 +108,6 @@ public class ExecutionProfile {
}
loadChannelProfile = new RuntimeProfile("LoadChannels");
root.addChild(loadChannelProfile);
- fragmentInstancesProfiles = Maps.newHashMap();
- instanceIdToFragmentId = Maps.newHashMap();
}
private List<List<RuntimeProfile>> getMultiBeProfile(int fragmentId) {
@@ -299,23 +293,6 @@ public class ExecutionProfile {
multiBeProfile.get(params.fragment_id).put(backend.getHeartbeatAddress(), taskProfile);
}
- // MultiInstances may update the profile concurrently
- public synchronized void addInstanceProfile(PlanFragmentId fragmentId, TUniqueId instanceId,
- RuntimeProfile instanceProfile) {
- Map<TUniqueId, RuntimeProfile> instanceProfiles = fragmentInstancesProfiles.get(fragmentId);
- if (instanceProfiles == null) {
- instanceProfiles = Maps.newHashMap();
- fragmentInstancesProfiles.put(fragmentId, instanceProfiles);
- }
- RuntimeProfile existingInstanceProfile = instanceProfiles.get(instanceId);
- if (existingInstanceProfile == null) {
- instanceProfiles.put(instanceId, instanceProfile);
- instanceIdToFragmentId.put(instanceId, fragmentId);
- fragmentProfiles.get(fragmentId.asInt()).addChild(instanceProfile);
- return;
- }
- }
-
public synchronized void addFragmentBackend(PlanFragmentId fragmentId, Long backendId) {
fragmentIdBeNum.put(fragmentId.asInt(), fragmentIdBeNum.get(fragmentId.asInt()) + 1);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/21: [Refactor](meta) Delete useless classes and write methods (#36894)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f04c185cc6c780483977918e19e6a14a35b3dc16
Author: Peyz <30...@users.noreply.github.com>
AuthorDate: Fri Jun 28 17:08:24 2024 +0800
[Refactor](meta) Delete useless classes and write methods (#36894)
Co-authored-by: zhangpeicheng <zh...@meituan.com>
---
.../java/org/apache/doris/common/io/DeepCopy.java | 1 +
.../java/org/apache/doris/backup/BackupMeta.java | 1 +
.../apache/doris/catalog/CatalogRecycleBin.java | 2 +
.../main/java/org/apache/doris/catalog/Column.java | 30 +-----
.../java/org/apache/doris/catalog/ColumnStats.java | 26 +----
.../org/apache/doris/catalog/DistributionInfo.java | 1 +
.../apache/doris/catalog/HashDistributionInfo.java | 1 +
.../apache/doris/catalog/ListPartitionInfo.java | 1 +
.../apache/doris/catalog/MaterializedIndex.java | 11 +-
.../java/org/apache/doris/catalog/OlapTable.java | 3 +-
.../java/org/apache/doris/catalog/Partition.java | 2 +
.../org/apache/doris/catalog/PartitionInfo.java | 20 +---
.../doris/catalog/RandomDistributionInfo.java | 1 +
.../apache/doris/catalog/RangePartitionInfo.java | 1 +
.../java/org/apache/doris/catalog/Replica.java | 22 +---
.../apache/doris/catalog/SinglePartitionInfo.java | 1 +
.../main/java/org/apache/doris/catalog/Tablet.java | 12 +--
.../org/apache/doris/catalog/TempPartitions.java | 24 +----
.../apache/doris/cloud/catalog/CloudPartition.java | 9 +-
.../apache/doris/cloud/catalog/CloudReplica.java | 17 +--
.../apache/doris/persist/BackendIdsUpdateInfo.java | 63 ------------
.../java/org/apache/doris/persist/ClusterInfo.java | 114 ---------------------
.../apache/doris/transaction/TabletCommitInfo.java | 11 +-
.../org/apache/doris/catalog/ColumnStatTest.java | 21 ++--
.../java/org/apache/doris/catalog/ColumnTest.java | 18 ++--
.../doris/catalog/MaterializedIndexTest.java | 6 +-
.../doris/catalog/RangePartitionInfoTest.java | 6 +-
.../java/org/apache/doris/catalog/ReplicaTest.java | 12 +--
.../java/org/apache/doris/catalog/TabletTest.java | 6 +-
.../apache/doris/catalog/TempPartitionTest.java | 4 +-
30 files changed, 74 insertions(+), 373 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java
index 106205f2256..282e6a0e16a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java
@@ -39,6 +39,7 @@ public class DeepCopy {
// deep copy orig to dest.
// the param "c" is the implementation class of "dest".
// And the "dest" class must has method "readFields(DataInput)"
+ @Deprecated
public static boolean copy(Writable orig, Writable dest, Class c, int metaVersion) {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(metaVersion);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
index 45fdd1261ce..e27c8d19a84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
@@ -140,6 +140,7 @@ public class BackupMeta implements Writable, GsonPostProcessable {
Text.writeString(out, GsonUtils.GSON.toJson(this));
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
int size = in.readInt();
for (int i = 0; i < size; i++) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index 6b6fe5284a4..17cc5cd148b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -1305,6 +1305,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos
updateDbInfoForLowerVersion();
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
int count = in.readInt();
for (int i = 0; i < count; i++) {
@@ -1439,6 +1440,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable, GsonPos
return table;
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
dbId = in.readLong();
table = Table.read(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 8d453b1b0f0..0aca7e4e28e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -29,7 +29,6 @@ import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -48,7 +47,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
@@ -60,7 +58,7 @@ import java.util.Set;
/**
* This class represents the column-related metadata.
*/
-public class Column implements Writable, GsonPostProcessable {
+public class Column implements GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(Column.class);
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";
@@ -1065,33 +1063,7 @@ public class Column implements Writable, GsonPostProcessable {
return ok;
}
- @Override
- public void write(DataOutput out) throws IOException {
- String json = GsonUtils.GSON.toJson(this);
- Text.writeString(out, json);
- }
-
@Deprecated
- private void readFields(DataInput in) throws IOException {
- name = Text.readString(in);
- type = ColumnType.read(in);
- boolean notNull = in.readBoolean();
- if (notNull) {
- aggregationType = AggregateType.valueOf(Text.readString(in));
- isAggregationTypeImplicit = in.readBoolean();
- }
- isKey = in.readBoolean();
- isAllowNull = in.readBoolean();
- notNull = in.readBoolean();
- if (notNull) {
- defaultValue = Text.readString(in);
- realDefaultValue = defaultValue;
- }
- stats = ColumnStats.read(in);
-
- comment = Text.readString(in);
- }
-
public static Column read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, Column.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java
index 8ba694b42db..198ca02f51b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnStats.java
@@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
-import org.apache.doris.common.io.Writable;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
@@ -27,15 +26,12 @@ import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
import java.util.Objects;
/**
* Statistics for a single column.
*/
-public class ColumnStats implements Writable {
+public class ColumnStats {
private static final Logger LOG = LogManager.getLogger(ColumnStats.class);
@SerializedName(value = "avgSerializedSize")
@@ -120,26 +116,6 @@ public class ColumnStats implements Writable {
"numNulls", numNulls).toString();
}
- public void write(DataOutput out) throws IOException {
- out.writeLong(numDistinctValues);
- out.writeFloat(avgSerializedSize);
- out.writeLong(maxSize);
- out.writeLong(numNulls);
- }
-
- public void readFields(DataInput in) throws IOException {
- numDistinctValues = in.readLong();
- avgSerializedSize = in.readFloat();
- maxSize = in.readLong();
- numNulls = in.readLong();
- }
-
- public static ColumnStats read(DataInput in) throws IOException {
- ColumnStats columnStats = new ColumnStats();
- columnStats.readFields(in);
- return columnStats;
- }
-
@Override
public int hashCode() {
return Objects.hash(avgSerializedSize, maxSize, numDistinctValues, numNulls);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
index b8265ccae40..9b05f8ca709 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DistributionInfo.java
@@ -84,6 +84,7 @@ public abstract class DistributionInfo {
throw new NotImplementedException("toDistributionDesc not implemented");
}
+ @Deprecated
public void readFields(DataInput in) throws IOException {
type = DistributionInfoType.valueOf(Text.readString(in));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
index 0967a3b912b..2de384961dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HashDistributionInfo.java
@@ -66,6 +66,7 @@ public class HashDistributionInfo extends DistributionInfo {
this.bucketNum = bucketNum;
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
index c2b3451cf67..da48ac980bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ListPartitionInfo.java
@@ -67,6 +67,7 @@ public class ListPartitionInfo extends PartitionInfo {
}
}
+ @Deprecated
public static PartitionInfo read(DataInput in) throws IOException {
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_136) {
return GsonUtils.GSON.fromJson(Text.readString(in), ListPartitionInfo.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index 1d4d2d7e45e..d56f043873a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -27,7 +26,6 @@ import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -37,7 +35,7 @@ import java.util.Map;
/**
* The OlapTraditional table is a materialized table which stored as rowcolumnar file or columnar file
*/
-public class MaterializedIndex extends MetaObject implements Writable, GsonPostProcessable {
+public class MaterializedIndex extends MetaObject implements GsonPostProcessable {
public enum IndexState {
NORMAL,
@Deprecated
@@ -208,11 +206,7 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
return -1;
}
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, GsonUtils.GSON.toJson(this));
- }
-
+ @Deprecated
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -232,6 +226,7 @@ public class MaterializedIndex extends MetaObject implements Writable, GsonPostP
rollupFinishedVersion = in.readLong();
}
+ @Deprecated
public static MaterializedIndex read(DataInput in) throws IOException {
if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_136) {
MaterializedIndex mi = new MaterializedIndex();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index b0103e0d68b..f5e3cc90e85 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1559,6 +1559,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
Text.writeString(out, GsonUtils.GSON.toJson(this));
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -1796,7 +1797,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
t.readFields(in);
return t;
}
- return (OlapTable) GsonUtils.GSON.fromJson(Text.readString(in), OlapTable.class);
+ return GsonUtils.GSON.fromJson(Text.readString(in), OlapTable.class);
}
/*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
index 22762cbaeae..775ee5530c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
@@ -341,12 +341,14 @@ public class Partition extends MetaObject {
return true;
}
+ @Deprecated
public static Partition read(DataInput in) throws IOException {
Partition partition = EnvFactory.getInstance().createPartition();
partition.readFields(in);
return partition;
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 10c01f5f2de..8deaa6d5f41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -29,8 +29,6 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTabletType;
@@ -42,7 +40,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -56,7 +53,7 @@ import java.util.stream.Collectors;
/*
* Repository of a partition's related infos
*/
-public class PartitionInfo implements Writable {
+public class PartitionInfo {
private static final Logger LOG = LogManager.getLogger(PartitionInfo.class);
@SerializedName("Type")
@@ -361,16 +358,6 @@ public class PartitionInfo implements Writable {
idToInMemory.put(partitionId, isInMemory);
}
- public static PartitionInfo read(DataInput in) throws IOException {
- if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_136) {
- PartitionInfo partitionInfo = new PartitionInfo();
- partitionInfo.readFields(in);
- return partitionInfo;
- } else {
- return GsonUtils.GSON.fromJson(Text.readString(in), PartitionInfo.class);
- }
- }
-
public boolean isMultiColumnPartition() {
return isMultiColumnPartition;
}
@@ -431,11 +418,6 @@ public class PartitionInfo implements Writable {
}
}
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, GsonUtils.GSON.toJson(this));
- }
-
@Deprecated
public void readFields(DataInput in) throws IOException {
type = PartitionType.valueOf(Text.readString(in));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
index 6ec9200c8ce..d225adbf0a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RandomDistributionInfo.java
@@ -62,6 +62,7 @@ public class RandomDistributionInfo extends DistributionInfo {
return builder.toString();
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
index a35f97537d1..87e1d5d19b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java
@@ -206,6 +206,7 @@ public class RangePartitionInfo extends PartitionInfo {
RangeUtils.checkRangeConflict(list1, list2);
}
+ @Deprecated
public static PartitionInfo read(DataInput in) throws IOException {
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_136) {
return GsonUtils.GSON.fromJson(Text.readString(in), RangePartitionInfo.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
index 7c22489285b..22813ef5118 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TTabletInfo;
@@ -30,14 +29,13 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.Comparator;
/**
* This class represents the olap replica related metadata.
*/
-public class Replica implements Writable {
+public class Replica {
private static final Logger LOG = LogManager.getLogger(Replica.class);
public static final VersionComparator<Replica> VERSION_DESC_COMPARATOR = new VersionComparator<Replica>();
public static final LastSuccessVersionComparator<Replica> LAST_SUCCESS_VERSION_COMPARATOR =
@@ -710,22 +708,7 @@ public class Replica implements Writable {
return strBuffer.toString();
}
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(id);
- out.writeLong(backendId);
- out.writeLong(version);
- out.writeLong(versionHash);
- out.writeLong(dataSize);
- out.writeLong(rowCount);
- Text.writeString(out, state.name());
-
- out.writeLong(lastFailedVersion);
- out.writeLong(lastFailedVersionHash);
- out.writeLong(lastSuccessVersion);
- out.writeLong(lastSuccessVersionHash);
- }
-
+ @Deprecated
public void readFields(DataInput in) throws IOException {
id = in.readLong();
backendId = in.readLong();
@@ -740,6 +723,7 @@ public class Replica implements Writable {
lastSuccessVersionHash = in.readLong();
}
+ @Deprecated
public static Replica read(DataInput in) throws IOException {
Replica replica = EnvFactory.getInstance().createReplica();
replica.readFields(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java
index 97cda77fb32..495d57c61f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java
@@ -29,6 +29,7 @@ public class SinglePartitionInfo extends PartitionInfo {
super(PartitionType.UNPARTITIONED);
}
+ @Deprecated
public static PartitionInfo read(DataInput in) throws IOException {
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_136) {
return GsonUtils.GSON.fromJson(Text.readString(in), SinglePartitionInfo.class);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index cdf59fe41b1..899eb94b8d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -27,7 +27,6 @@ import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.Backend;
@@ -44,7 +43,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -60,7 +58,7 @@ import java.util.stream.LongStream;
/**
* This class represents the olap tablet related metadata.
*/
-public class Tablet extends MetaObject implements Writable {
+public class Tablet extends MetaObject {
private static final Logger LOG = LogManager.getLogger(Tablet.class);
// if current version count of replica is mor than
// QUERYABLE_TIMES_OF_MIN_VERSION_COUNT times the minimum version count,
@@ -394,12 +392,7 @@ public class Tablet extends MetaObject implements Writable {
return "tabletId=" + this.id;
}
- @Override
- public void write(DataOutput out) throws IOException {
- String json = GsonUtils.GSON.toJson(this);
- Text.writeString(out, json);
- }
-
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -418,6 +411,7 @@ public class Tablet extends MetaObject implements Writable {
isConsistent = in.readBoolean();
}
+ @Deprecated
public static Tablet read(DataInput in) throws IOException {
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_115) {
String json = Text.readString(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java
index f28eb870f77..083f69b741d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TempPartitions.java
@@ -18,7 +18,6 @@
package org.apache.doris.catalog;
import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -28,7 +27,6 @@ import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -40,7 +38,7 @@ import java.util.Set;
// user can load data into some of the temp partitions,
// and then replace the formal partitions with these temp partitions
// to make a overwrite load.
-public class TempPartitions implements Writable, GsonPostProcessable {
+public class TempPartitions implements GsonPostProcessable {
@SerializedName(value = "idToPartition")
private Map<Long, Partition> idToPartition = Maps.newHashMap();
private Map<String, Partition> nameToPartition = Maps.newHashMap();
@@ -124,30 +122,12 @@ public class TempPartitions implements Writable, GsonPostProcessable {
}
}
- @Override
- public void write(DataOutput out) throws IOException {
- String json = GsonUtils.GSON.toJson(this);
- Text.writeString(out, json);
- }
-
+ @Deprecated
public static TempPartitions read(DataInput in) throws IOException {
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, TempPartitions.class);
}
- @Deprecated
- private void readFields(DataInput in) throws IOException {
- int size = in.readInt();
- for (int i = 0; i < size; i++) {
- Partition partition = Partition.read(in);
- idToPartition.put(partition.getId(), partition);
- nameToPartition.put(partition.getName(), partition);
- }
- if (in.readBoolean()) {
- partitionInfo = (RangePartitionInfo) RangePartitionInfo.read(in);
- }
- }
-
@Override
public void gsonPostProcess() {
for (Partition partition : idToPartition.values()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
index 42327b6039d..b3f868ebe12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
@@ -36,7 +36,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -359,6 +358,7 @@ public class CloudPartition extends Partition {
return null;
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -366,13 +366,6 @@ public class CloudPartition extends Partition {
this.tableId = in.readLong();
}
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- out.writeLong(this.dbId);
- out.writeLong(this.tableId);
- }
-
public boolean equals(Object obj) {
if (!super.equals(obj)) {
return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
index b18dbb9970e..d2e173dfd53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java
@@ -36,7 +36,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -347,6 +346,7 @@ public class CloudReplica extends Replica {
return true;
}
+ @Deprecated
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
@@ -375,21 +375,6 @@ public class CloudReplica extends Replica {
}
}
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- out.writeLong(dbId);
- out.writeLong(tableId);
- out.writeLong(partitionId);
- out.writeLong(indexId);
- out.writeLong(idx);
- out.writeInt(clusterToBackends.size());
- for (Map.Entry<String, List<Long>> entry : clusterToBackends.entrySet()) {
- Text.writeString(out, entry.getKey());
- out.writeLong(entry.getValue().get(0));
- }
- }
-
public long getDbId() {
return dbId;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/BackendIdsUpdateInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/BackendIdsUpdateInfo.java
deleted file mode 100644
index ac2e0ea9d23..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/BackendIdsUpdateInfo.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.persist;
-
-import org.apache.doris.common.io.Writable;
-
-import com.google.common.collect.Lists;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.List;
-
-public class BackendIdsUpdateInfo implements Writable {
- private List<Long> backendIds;
-
- public BackendIdsUpdateInfo() {
- this.backendIds = Lists.newArrayList();
- }
-
- public BackendIdsUpdateInfo(List<Long> backends) {
- this.backendIds = backends;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(backendIds.size());
- for (Long id : backendIds) {
- out.writeLong(id);
- }
- }
-
- public void readFields(DataInput in) throws IOException {
- int backendCount = in.readInt();
- while (backendCount-- > 0) {
- backendIds.add(in.readLong());
- }
- }
-
- public List<Long> getBackendList() {
- return backendIds;
- }
-
- public void setBackendList(List<Long> backendList) {
- this.backendIds = backendList;
- }
-
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ClusterInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ClusterInfo.java
deleted file mode 100644
index aad46a232f5..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/ClusterInfo.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.persist;
-
-import org.apache.doris.common.io.Text;
-import org.apache.doris.common.io.Writable;
-
-import com.google.common.collect.Lists;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.List;
-
-public class ClusterInfo implements Writable {
-
- private String clusterName;
- private long clusterId;
- private int instanceNum;
-
- private String newClusterName;
- private long newClusterId;
- private int newInstanceNum;
-
- private List<Long> expandBackendIds = Lists.newArrayList();
-
- public ClusterInfo() {
- this.clusterName = "";
- this.clusterId = 0L;
- this.instanceNum = 0;
- this.newClusterName = "";
- this.newClusterId = 0L;
- this.newInstanceNum = 0;
- }
-
- public ClusterInfo(String clusterName, long clusterId) {
- this.clusterName = clusterName;
- this.clusterId = clusterId;
- this.instanceNum = 0;
- this.newClusterName = "";
- this.newClusterId = 0L;
- this.newInstanceNum = 0;
- }
-
- public ClusterInfo(String clusterName, long clusterId, List<Long> expandBackendIds) {
- this.clusterName = clusterName;
- this.clusterId = clusterId;
- this.expandBackendIds = expandBackendIds;
- this.instanceNum = 0;
- this.newClusterName = "";
- this.newClusterId = 0L;
- this.newInstanceNum = 0;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Text.writeString(out, clusterName);
- out.writeLong(clusterId);
- out.writeInt(instanceNum);
- out.writeInt(expandBackendIds.size());
- for (long id : expandBackendIds) {
- out.writeLong(id);
- }
- }
-
- public void readFields(DataInput in) throws IOException {
- clusterName = Text.readString(in);
- clusterId = in.readLong();
- instanceNum = in.readInt();
- int count = in.readInt();
- while (count-- > 0) {
- expandBackendIds.add(in.readLong());
- }
- }
-
- public String getClusterName() {
- return clusterName;
- }
-
- public long getClusterId() {
- return clusterId;
- }
-
- public int getInstanceNum() {
- return instanceNum;
- }
-
- public String getNewClusterName() {
- return newClusterName;
- }
-
- public int getNewInstanceNum() {
- return newInstanceNum;
- }
-
- public List<Long> getBackendIdList() {
- return expandBackendIds;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletCommitInfo.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletCommitInfo.java
index ed8aa01320e..8ca75a3bd7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletCommitInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletCommitInfo.java
@@ -17,18 +17,16 @@
package org.apache.doris.transaction;
-import org.apache.doris.common.io.Writable;
import org.apache.doris.thrift.TTabletCommitInfo;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
-public class TabletCommitInfo implements Writable {
+public class TabletCommitInfo {
private long tabletId;
private long backendId;
@@ -55,12 +53,7 @@ public class TabletCommitInfo implements Writable {
return commitInfos;
}
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeLong(tabletId);
- out.writeLong(backendId);
- }
-
+ @Deprecated
public void readFields(DataInput in) throws IOException {
tabletId = in.readLong();
backendId = in.readLong();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java
index 9fe3ac8b59b..5e48f91069e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnStatTest.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.persist.gson.GsonUtils;
+
import org.junit.Assert;
import org.junit.Test;
@@ -35,42 +38,40 @@ public class ColumnStatTest {
DataOutputStream dos = new DataOutputStream(Files.newOutputStream(path));
ColumnStats stats1 = new ColumnStats();
- stats1.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(stats1));
ColumnStats stats2 = new ColumnStats();
stats2.setAvgSerializedSize(1.1f);
stats2.setNumDistinctValues(100L);
stats2.setMaxSize(1000L);
stats2.setNumNulls(10000L);
- stats2.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(stats2));
ColumnStats stats3 = new ColumnStats();
stats3.setAvgSerializedSize(3.3f);
stats3.setNumDistinctValues(200L);
stats3.setMaxSize(2000L);
stats3.setNumNulls(20000L);
- stats3.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(stats3));
ColumnStats stats4 = new ColumnStats(stats3);
- stats4.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(stats4));
dos.flush();
dos.close();
// 2. Read objects from file
DataInputStream dis = new DataInputStream(Files.newInputStream(path));
- ColumnStats rStats1 = new ColumnStats();
- rStats1.readFields(dis);
+ ColumnStats rStats1 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
Assert.assertEquals(rStats1, stats1);
- ColumnStats rStats2 = new ColumnStats();
- rStats2.readFields(dis);
+ ColumnStats rStats2 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
Assert.assertEquals(rStats2, stats2);
- ColumnStats rStats3 = ColumnStats.read(dis);
+ ColumnStats rStats3 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
Assert.assertEquals(rStats3, stats3);
- ColumnStats rStats4 = ColumnStats.read(dis);
+ ColumnStats rStats4 = GsonUtils.GSON.fromJson(Text.readString(dis), ColumnStats.class);
Assert.assertEquals(rStats4, stats4);
Assert.assertEquals(rStats4, stats3);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
index 45e4df06589..431ce02ae20 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColumnTest.java
@@ -19,7 +19,9 @@ package org.apache.doris.catalog;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.io.Text;
import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.persist.gson.GsonUtils;
import org.junit.Assert;
import org.junit.Before;
@@ -53,26 +55,26 @@ public class ColumnTest {
Column column1 = new Column("user",
ScalarType.createChar(20), false, AggregateType.SUM, "", "");
- column1.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(column1));
Column column2 = new Column("age",
ScalarType.createType(PrimitiveType.INT), false, AggregateType.REPLACE, "20", "");
- column2.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(column2));
Column column3 = new Column("name", PrimitiveType.BIGINT);
column3.setIsKey(true);
- column3.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(column3));
Column column4 = new Column("age",
ScalarType.createType(PrimitiveType.INT), false, AggregateType.REPLACE, "20",
"");
- column4.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(column4));
dos.flush();
dos.close();
// 2. Read objects from file
DataInputStream dis = new DataInputStream(Files.newInputStream(path));
- Column rColumn1 = Column.read(dis);
+ Column rColumn1 = GsonUtils.GSON.fromJson(Text.readString(dis), Column.class);
Assert.assertEquals("user", rColumn1.getName());
Assert.assertEquals(PrimitiveType.CHAR, rColumn1.getDataType());
Assert.assertEquals(AggregateType.SUM, rColumn1.getAggregationType());
@@ -83,16 +85,16 @@ public class ColumnTest {
Assert.assertFalse(rColumn1.isAllowNull());
// 3. Test read()
- Column rColumn2 = Column.read(dis);
+ Column rColumn2 = GsonUtils.GSON.fromJson(Text.readString(dis), Column.class);
Assert.assertEquals("age", rColumn2.getName());
Assert.assertEquals(PrimitiveType.INT, rColumn2.getDataType());
Assert.assertEquals(AggregateType.REPLACE, rColumn2.getAggregationType());
Assert.assertEquals("20", rColumn2.getDefaultValue());
- Column rColumn3 = Column.read(dis);
+ Column rColumn3 = GsonUtils.GSON.fromJson(Text.readString(dis), Column.class);
Assert.assertEquals(rColumn3, column3);
- Column rColumn4 = Column.read(dis);
+ Column rColumn4 = GsonUtils.GSON.fromJson(Text.readString(dis), Column.class);
Assert.assertEquals(rColumn4, column4);
Assert.assertEquals(rColumn2.toString(), column2.toString());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java
index ccc8f8925eb..25b94bc59e8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MaterializedIndexTest.java
@@ -19,6 +19,8 @@ package org.apache.doris.catalog;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.persist.gson.GsonUtils;
import mockit.Mocked;
import org.junit.Assert;
@@ -70,14 +72,14 @@ public class MaterializedIndexTest {
Path path = Files.createFile(Paths.get("./index"));
DataOutputStream dos = new DataOutputStream(Files.newOutputStream(path));
- index.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(index));
dos.flush();
dos.close();
// 2. Read objects from file
DataInputStream dis = new DataInputStream(Files.newInputStream(path));
- MaterializedIndex rIndex = MaterializedIndex.read(dis);
+ MaterializedIndex rIndex = GsonUtils.GSON.fromJson(Text.readString(dis), MaterializedIndex.class);
Assert.assertEquals(index, rIndex);
// 3. delete files
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java
index e1112c8cb38..3121397a772 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java
@@ -23,6 +23,8 @@ import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.collect.Lists;
import org.junit.Assert;
@@ -457,14 +459,14 @@ public class RangePartitionInfoTest {
partitionInfo = new RangePartitionInfo(partitionColumns);
- partitionInfo.write(out);
+ Text.writeString(out, GsonUtils.GSON.toJson(partitionInfo));
out.flush();
out.close();
// 2. Read objects from file
DataInputStream in = new DataInputStream(Files.newInputStream(path));
- RangePartitionInfo partitionInfo2 = (RangePartitionInfo) PartitionInfo.read(in);
+ RangePartitionInfo partitionInfo2 = GsonUtils.GSON.fromJson(Text.readString(in), RangePartitionInfo.class);
Assert.assertEquals(partitionInfo.getType(), partitionInfo2.getType());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaTest.java
index eb7dbca0775..63f61cc8747 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaTest.java
@@ -18,6 +18,8 @@
package org.apache.doris.catalog;
import org.apache.doris.catalog.Replica.ReplicaState;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.persist.gson.GsonUtils;
import mockit.Mocked;
import org.junit.Assert;
@@ -85,20 +87,19 @@ public class ReplicaTest {
Replica olapReplica = new Replica(100L * count, 100L * count, 100L * count, 0,
100L * count, 0, 100 * count, ReplicaState.NORMAL, 0, 100L * count);
list1.add(olapReplica);
- olapReplica.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(olapReplica));
}
Replica replica = new Replica(10L, 20L, 0, null);
list1.add(replica);
- replica.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(replica));
dos.flush();
dos.close();
// 2. Read a object from file
DataInputStream dis = new DataInputStream(Files.newInputStream(path));
for (int count = 0; count < 10; ++count) {
- Replica olapReplica = new Replica();
- olapReplica.readFields(dis);
+ Replica olapReplica = GsonUtils.GSON.fromJson(Text.readString(dis), Replica.class);
Assert.assertEquals(100 * count, olapReplica.getId());
Assert.assertEquals(100 * count, olapReplica.getBackendId());
Assert.assertEquals(100 * count, olapReplica.getVersion());
@@ -107,8 +108,7 @@ public class ReplicaTest {
Assert.assertEquals(Replica.ReplicaState.NORMAL, olapReplica.getState());
list2.add(olapReplica);
}
- Replica olapReplica = new Replica();
- olapReplica.readFields(dis);
+ Replica olapReplica = GsonUtils.GSON.fromJson(Text.readString(dis), Replica.class);
list2.add(olapReplica);
// 3. Check equal
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
index d7fdb2694a8..2b15d962884 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
@@ -20,6 +20,8 @@ package org.apache.doris.catalog;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.collect.Sets;
@@ -117,13 +119,13 @@ public class TabletTest {
public void testSerialization() throws Exception {
final Path path = Files.createTempFile("olapTabletTest", "tmp");
DataOutputStream dos = new DataOutputStream(Files.newOutputStream(path));
- tablet.write(dos);
+ Text.writeString(dos, GsonUtils.GSON.toJson(tablet));
dos.flush();
dos.close();
// 2. Read a object from file
DataInputStream dis = new DataInputStream(Files.newInputStream(path));
- Tablet rTablet1 = Tablet.read(dis);
+ Tablet rTablet1 = GsonUtils.GSON.fromJson(Text.readString(dis), Tablet.class);
Assert.assertEquals(1, rTablet1.getId());
Assert.assertEquals(3, rTablet1.getReplicas().size());
Assert.assertEquals(rTablet1.getReplicas().get(0).getVersion(), rTablet1.getReplicas().get(1).getVersion());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java
index 19b191a4cd2..04e6bb8a151 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TempPartitionTest.java
@@ -30,8 +30,10 @@ import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.meta.MetaContext;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.qe.ShowResultSet;
@@ -1299,7 +1301,7 @@ public class TempPartitionTest {
file.createNewFile();
DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
- tempPartitionsInstance.write(out);
+ Text.writeString(out, GsonUtils.GSON.toJson(tempPartitionsInstance));
out.flush();
out.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/21: [Fix](hive-writer) Fixed the issue where `uncompletedMpuPendingUploads` did not remove objects correctly. (#36905)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a89a43c2a97910b5f907490feb72faad94b34553
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Fri Jun 28 17:41:02 2024 +0800
[Fix](hive-writer) Fixed the issue where `uncompletedMpuPendingUploads` did not remove objects correctly. (#36905)
[Fix](hive-writer) Fixed the issue where `uncompletedMpuPendingUploads`
did not remove objects correctly.
---
.../apache/doris/datasource/hive/HMSTransaction.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
index 824af6996a9..d883b9dc786 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
@@ -103,6 +103,24 @@ public class HMSTransaction implements Transaction {
this.s3MPUPendingUpload = s3MPUPendingUpload;
this.path = path;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ UncompletedMpuPendingUpload that = (UncompletedMpuPendingUpload) o;
+ return Objects.equals(s3MPUPendingUpload, that.s3MPUPendingUpload) && Objects.equals(path,
+ that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(s3MPUPendingUpload, path);
+ }
}
private Set<UncompletedMpuPendingUpload> uncompletedMpuPendingUploads = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 08/21: [chore](upgrade) turn off fallback_to_original_planner when upgrade (#37005)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fee6a24ef645c6db7cefeffe6b96c4496e4054ae
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Fri Jun 28 18:56:32 2024 +0800
[chore](upgrade) turn off fallback_to_original_planner when upgrade (#37005)
upgrade from 2.1 or ealier version to 3.0.x will
- enable_nereids_planner to true
- enable_nereids_dml to true
- enable_fallback_to_original_planner to false
---
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index a3da3482f77..cda6f3b4602 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1590,8 +1590,15 @@ public class Env {
SessionVariable.NEREIDS_TIMEOUT_SECOND, "30");
}
}
- if (journalVersion <= FeMetaVersion.VERSION_133) {
- VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1",
+ if (journalVersion <= FeMetaVersion.VERSION_129) {
+ VariableMgr.refreshDefaultSessionVariables("2.1 to 3.0", SessionVariable.ENABLE_NEREIDS_PLANNER,
+ "true");
+ VariableMgr.refreshDefaultSessionVariables("2.1 to 3.0", SessionVariable.ENABLE_NEREIDS_DML,
+ "true");
+ VariableMgr.refreshDefaultSessionVariables("2.1 to 3.0",
+ SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
+ "false");
+ VariableMgr.refreshDefaultSessionVariables("2.1 to 3.0",
SessionVariable.ENABLE_MATERIALIZED_VIEW_REWRITE,
"true");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 17/21: (cloud-merge) Get fileCacheSize by RPC always (#36857)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9255c366ec145a2692bd79731b7974f0f1048aa4
Author: Lightman <31...@users.noreply.github.com>
AuthorDate: Sat Jun 29 09:25:40 2024 +0800
(cloud-merge) Get fileCacheSize by RPC always (#36857)
Get fileCacheSize every time when do warm up job.
---
.../apache/doris/cloud/CacheHotspotManager.java | 39 ++++++++++------------
.../main/java/org/apache/doris/system/Backend.java | 10 ------
2 files changed, 18 insertions(+), 31 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
index 304f76dee48..4f359446aad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
@@ -152,7 +152,6 @@ public class CacheHotspotManager extends MasterDaemon {
TGetTopNHotPartitionsResponse resp = respPair.first;
if (resp.isSetHotTables()) {
resp.getHotTables().forEach((THotTableMessage hotTable) -> {
- respPair.second.setfileCacheCapacityBytes(resp.file_cache_size);
if (hotTable.isSetHotPartitions()) {
hotTable.hot_partitions.forEach((THotPartition partition) -> {
insertIntoTable(clusterToBeList.getKey(), hotTable.table_id,
@@ -337,26 +336,24 @@ public class CacheHotspotManager extends MasterDaemon {
.getBackendsByClusterName(clusterName);
Long totalFileCache = 0L;
for (Backend backend : backends) {
- Long fileCacheSize = backend.getfileCacheCapactiyBytes();
- if (fileCacheSize == 0) {
- boolean ok = false;
- BackendService.Client client = null;
- TNetworkAddress address = null;
- try {
- address = new TNetworkAddress(backend.getHost(), backend.getBePort());
- client = ClientPool.backendPool.borrowObject(address);
- TGetTopNHotPartitionsResponse resp = client.getTopNHotPartitions(
- new TGetTopNHotPartitionsRequest());
- fileCacheSize = resp.file_cache_size;
- ok = true;
- } catch (Exception e) {
- throw new RuntimeException(e);
- } finally {
- if (ok) {
- ClientPool.backendPool.returnObject(address, client);
- } else {
- ClientPool.backendPool.invalidateObject(address, client);
- }
+ Long fileCacheSize = 0L;
+ boolean ok = false;
+ BackendService.Client client = null;
+ TNetworkAddress address = null;
+ try {
+ address = new TNetworkAddress(backend.getHost(), backend.getBePort());
+ client = ClientPool.backendPool.borrowObject(address);
+ TGetTopNHotPartitionsResponse resp = client.getTopNHotPartitions(
+ new TGetTopNHotPartitionsRequest());
+ fileCacheSize = resp.file_cache_size;
+ ok = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (ok) {
+ ClientPool.backendPool.returnObject(address, client);
+ } else {
+ ClientPool.backendPool.invalidateObject(address, client);
}
}
totalFileCache += fileCacheSize;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
index 902b90843b6..a366aca5d6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
@@ -148,8 +148,6 @@ public class Backend implements Writable {
// send some queries to this BE, it is not an important problem.
private AtomicBoolean isShutDown = new AtomicBoolean(false);
- private long fileCacheCapactiyBytes = 0;
-
public Backend() {
this.host = "";
this.version = "";
@@ -241,14 +239,6 @@ public class Backend implements Writable {
return heartbeatPort;
}
- public void setfileCacheCapacityBytes(long fileCacheCapactiyBytes) {
- this.fileCacheCapactiyBytes = fileCacheCapactiyBytes;
- }
-
- public long getfileCacheCapactiyBytes() {
- return fileCacheCapactiyBytes;
- }
-
public int getHttpPort() {
return httpPort;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 21/21: [improvement](segmentcache) limit segment cache by memory or segment num (#37026)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3df52cbf7610dc662ec525281b3a04eba360d219
Author: Yongqiang YANG <98...@users.noreply.github.com>
AuthorDate: Sun Jun 30 12:57:14 2024 +0800
[improvement](segmentcache) limit segment cache by memory or segment num (#37026)
Also enlarge columns per segment.
---
be/src/common/config.cpp | 2 +-
be/src/olap/lru_cache.cpp | 5 +++++
be/src/olap/lru_cache.h | 4 +++-
be/src/olap/segment_loader.h | 13 ++++++++-----
be/src/runtime/exec_env_init.cpp | 8 ++++----
be/test/testutil/run_all_tests.cpp | 2 +-
6 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 580793d36ab..9df75b97bd6 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1068,7 +1068,7 @@ DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
// max number of segment cache, default -1 for backward compatibility fd_number*2/5
DEFINE_mInt32(segment_cache_capacity, "-1");
-DEFINE_mInt32(estimated_num_columns_per_segment, "30");
+DEFINE_mInt32(estimated_num_columns_per_segment, "200");
DEFINE_mInt32(estimated_mem_per_column_reader, "1024");
// The value is calculate by storage_page_cache_limit * index_page_cache_percentage
DEFINE_mInt32(segment_cache_memory_percentage, "2");
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 031082f6da8..741c2423915 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -22,6 +22,7 @@ namespace doris {
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(cache_capacity, MetricUnit::BYTES);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(cache_usage, MetricUnit::BYTES);
+DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(cache_element_count, MetricUnit::NOUNIT);
DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(cache_usage_ratio, MetricUnit::NOUNIT);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(cache_lookup_count, MetricUnit::OPERATIONS);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(cache_hit_count, MetricUnit::OPERATIONS);
@@ -542,6 +543,7 @@ ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity,
_entity->register_hook(name, std::bind(&ShardedLRUCache::update_cache_metrics, this));
INT_GAUGE_METRIC_REGISTER(_entity, cache_capacity);
INT_GAUGE_METRIC_REGISTER(_entity, cache_usage);
+ INT_GAUGE_METRIC_REGISTER(_entity, cache_element_count);
INT_DOUBLE_METRIC_REGISTER(_entity, cache_usage_ratio);
INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count);
INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_hit_count);
@@ -640,15 +642,18 @@ void ShardedLRUCache::update_cache_metrics() const {
size_t total_usage = 0;
size_t total_lookup_count = 0;
size_t total_hit_count = 0;
+ size_t total_element_count = 0;
for (int i = 0; i < _num_shards; i++) {
total_capacity += _shards[i]->get_capacity();
total_usage += _shards[i]->get_usage();
total_lookup_count += _shards[i]->get_lookup_count();
total_hit_count += _shards[i]->get_hit_count();
+ total_element_count += _shards[i]->get_element_count();
}
cache_capacity->set_value(total_capacity);
cache_usage->set_value(total_usage);
+ cache_element_count->set_value(total_element_count);
cache_lookup_count->set_value(total_lookup_count);
cache_hit_count->set_value(total_hit_count);
cache_usage_ratio->set_value(total_capacity == 0 ? 0 : ((double)total_usage / total_capacity));
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 50676921044..059020deab5 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -60,7 +60,7 @@ enum LRUCacheType {
};
static constexpr LRUCacheType DEFAULT_LRU_CACHE_TYPE = LRUCacheType::SIZE;
-static constexpr uint32_t DEFAULT_LRU_CACHE_NUM_SHARDS = 16;
+static constexpr uint32_t DEFAULT_LRU_CACHE_NUM_SHARDS = 32;
static constexpr size_t DEFAULT_LRU_CACHE_ELEMENT_COUNT_CAPACITY = 0;
class CacheKey {
@@ -349,6 +349,7 @@ public:
uint64_t get_hit_count() const { return _hit_count; }
size_t get_usage() const { return _usage; }
size_t get_capacity() const { return _capacity; }
+ size_t get_element_count() const { return _table.element_count(); }
private:
void _lru_remove(LRUHandle* e);
@@ -433,6 +434,7 @@ private:
std::shared_ptr<MetricEntity> _entity;
IntGauge* cache_capacity = nullptr;
IntGauge* cache_usage = nullptr;
+ IntGauge* cache_element_count = nullptr;
DoubleGauge* cache_usage_ratio = nullptr;
IntAtomicCounter* cache_lookup_count = nullptr;
IntAtomicCounter* cache_hit_count = nullptr;
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 4d1f3f7a910..5bb8fae3c41 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -80,10 +80,11 @@ public:
segment_v2::SegmentSharedPtr segment;
};
- SegmentCache(size_t capacity)
- : LRUCachePolicyTrackingManual(CachePolicy::CacheType::SEGMENT_CACHE, capacity,
- LRUCacheType::SIZE,
- config::tablet_rowset_stale_sweep_time_sec) {}
+ SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)
+ : LRUCachePolicyTrackingManual(CachePolicy::CacheType::SEGMENT_CACHE,
+ memory_bytes_limit, LRUCacheType::SIZE,
+ config::tablet_rowset_stale_sweep_time_sec,
+ DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {}
// Lookup the given segment in the cache.
// If the segment is found, the cache entry will be written into handle.
@@ -110,7 +111,9 @@ public:
// After the estimation of segment memory usage is provided later, it is recommended
// to use Memory as the capacity limit of the cache.
- SegmentLoader(size_t capacity) { _segment_cache = std::make_unique<SegmentCache>(capacity); }
+ SegmentLoader(size_t memory_limit_bytes, size_t segment_num_count) {
+ _segment_cache = std::make_unique<SegmentCache>(memory_limit_bytes, segment_num_count);
+ }
// Load segments of "rowset", return the "cache_handle" which contains segments.
// If use_cache is true, it will be loaded from _cache.
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 6c8ac1e4aa9..bbf012b3a63 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -515,8 +515,8 @@ Status ExecEnv::_init_mem_env() {
// SegmentLoader caches segments in rowset granularity. So the size of
// opened files will greater than segment_cache_capacity.
int64_t segment_cache_capacity = config::segment_cache_capacity;
- if (segment_cache_capacity < 0 || segment_cache_capacity > fd_number * 2 / 5) {
- segment_cache_capacity = fd_number * 2 / 5;
+ if (segment_cache_capacity < 0 || segment_cache_capacity > fd_number * 1 / 5) {
+ segment_cache_capacity = fd_number * 1 / 5;
}
int64_t segment_cache_mem_limit =
@@ -526,8 +526,8 @@ Status ExecEnv::_init_mem_env() {
min(segment_cache_mem_limit, segment_cache_capacity *
config::estimated_num_columns_per_segment *
config::estimated_mem_per_column_reader);
- _segment_loader = new SegmentLoader(min_segment_cache_mem_limit);
- LOG(INFO) << "segment_cache_capacity <= fd_number * 2 / 5, fd_number: " << fd_number
+ _segment_loader = new SegmentLoader(min_segment_cache_mem_limit, segment_cache_capacity);
+ LOG(INFO) << "segment_cache_capacity <= fd_number * 1 / 5, fd_number: " << fd_number
<< " segment_cache_capacity: " << segment_cache_capacity
<< " min_segment_cache_mem_limit " << min_segment_cache_mem_limit;
diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp
index 68b7352419c..1cf749d0f7f 100644
--- a/be/test/testutil/run_all_tests.cpp
+++ b/be/test/testutil/run_all_tests.cpp
@@ -53,7 +53,7 @@ int main(int argc, char** argv) {
doris::ExecEnv::GetInstance()->set_dummy_lru_cache(std::make_shared<doris::DummyLRUCache>());
doris::ExecEnv::GetInstance()->set_storage_page_cache(
doris::StoragePageCache::create_global_cache(1 << 30, 10, 0));
- doris::ExecEnv::GetInstance()->set_segment_loader(new doris::SegmentLoader(1000));
+ doris::ExecEnv::GetInstance()->set_segment_loader(new doris::SegmentLoader(1000, 1000));
std::string conf = std::string(getenv("DORIS_HOME")) + "/conf/be.conf";
auto st = doris::config::init(conf.c_str(), false);
doris::ExecEnv::GetInstance()->set_tablet_schema_cache(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 12/21: [fix](statistics)Escape fetch partition stats sql. (#36941)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 74eaf05cb855d4b3a1c0b2b18082839a503715da
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Jun 28 20:51:52 2024 +0800
[fix](statistics)Escape fetch partition stats sql. (#36941)
Sql to fetch partition stats need to be escaped.
---
.../apache/doris/statistics/AnalysisManager.java | 2 +-
.../PartitionColumnStatisticCacheLoader.java | 2 +-
.../doris/statistics/StatisticsRepository.java | 6 ++---
.../suites/statistics/test_partition_stats.groovy | 27 ++++++++++++++++++++++
4 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 22bc11971c2..1e95adf1714 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -788,7 +788,7 @@ public class AnalysisManager implements Writable {
StringBuilder partNamePredicate = new StringBuilder();
while (iterator.hasNext()) {
partNamePredicate.append("'");
- partNamePredicate.append(iterator.next());
+ partNamePredicate.append(StatisticsUtil.escapeSQL(iterator.next()));
partNamePredicate.append("'");
partNamePredicate.append(",");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/PartitionColumnStatisticCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/PartitionColumnStatisticCacheLoader.java
index ac807a0d5f7..c365f6b1a74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/PartitionColumnStatisticCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/PartitionColumnStatisticCacheLoader.java
@@ -57,7 +57,7 @@ public class PartitionColumnStatisticCacheLoader extends
private Optional<PartitionColumnStatistic> loadFromPartitionStatsTable(PartitionColumnStatisticCacheKey key) {
List<ResultRow> partitionResults;
try {
- String partName = "'" + key.partId + "'";
+ String partName = "'" + StatisticsUtil.escapeSQL(key.partId) + "'";
partitionResults = StatisticsRepository.loadPartitionColumnStats(
key.catalogId, key.dbId, key.tableId, key.idxId, partName, key.colName);
} catch (InternalQueryExecutionException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index bbcc9de3a28..81192762f99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -144,12 +144,12 @@ public class StatisticsRepository {
params.put("tableId", String.valueOf(table.getId()));
StringJoiner sj = new StringJoiner(",");
for (String colName : columnNames) {
- sj.add("'" + colName + "'");
+ sj.add("'" + StatisticsUtil.escapeSQL(colName) + "'");
}
params.put("columnInfo", sj.toString());
sj = new StringJoiner(",");
for (String part : partitionNames) {
- sj.add("'" + part + "'");
+ sj.add("'" + StatisticsUtil.escapeSQL(part) + "'");
}
params.put("partitionInfo", sj.toString());
return StatisticsUtil.executeQuery(FETCH_PARTITIONS_STATISTIC_TEMPLATE, params);
@@ -418,7 +418,7 @@ public class StatisticsRepository {
params.put("tableId", String.valueOf(tableId));
params.put("indexId", String.valueOf(idxId));
params.put("partName", partName);
- params.put("columnId", colName);
+ params.put("columnId", StatisticsUtil.escapeSQL(colName));
return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
.replace(FETCH_PARTITION_STATISTIC_TEMPLATE));
}
diff --git a/regression-test/suites/statistics/test_partition_stats.groovy b/regression-test/suites/statistics/test_partition_stats.groovy
index 7658d50c47e..05216477323 100644
--- a/regression-test/suites/statistics/test_partition_stats.groovy
+++ b/regression-test/suites/statistics/test_partition_stats.groovy
@@ -928,6 +928,33 @@ suite("test_partition_stats") {
assertEquals("1", result[0][7])
assertEquals("20004", result[0][8])
+ // Test escape special col name.
+ sql """
+ create table part9(
+ k int null,
+ v variant null
+ )
+ duplicate key (k)
+ PARTITION BY RANGE(`k`)
+ (
+ PARTITION p1 VALUES [("0"), ("2")),
+ PARTITION p2 VALUES [("2"), ("4")),
+ PARTITION p3 VALUES [("4"), ("6"))
+ )
+ distributed BY hash(k) buckets 3
+ properties("replication_num" = "1");
+ """
+ sql """insert into part9 select 1,'{"k1" : 1, "k2" : 1, "k3" : "a"}';"""
+ sql """insert into part9 select 2,'{"k1" : 2, "k2" : 2, "k3" : "b"}';"""
+ sql """insert into part9 select 3,'{"k1" : 3, "k2" : null, "k3" : "c"}';"""
+ sql """insert into part9 select 4,'{"k1" : 4, "k2" : null, "k4" : {"k44" : 456}}';"""
+ createMV("create materialized view mv1 as select abs(cast(v['k4']['k44'] as int)), sum(abs(cast(v['k2'] as int)+2)+3) from part9 group by abs(cast(v['k4']['k44'] as int));")
+ sql """analyze table part9 with sync"""
+ result = sql """show column cached stats part9 partition(*)"""
+ assertEquals(9, result.size())
+ result = sql """show column stats part9 partition(*)"""
+ assertEquals(9, result.size())
+
sql """drop database test_partition_stats"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 09/21: [enhancement](cloud) batching get visible version from MetaService (#34615)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c0591389cc706ea46dac72438e48e1ab42bd587e
Author: zhengyu <fr...@gmail.com>
AuthorDate: Fri Jun 28 19:05:58 2024 +0800
[enhancement](cloud) batching get visible version from MetaService (#34615)
Get visible versions one by one from MetaService is costly. Batching
them into one RPC will not only reduce the workload of RPC service but
also reduce the lag.
---
cloud/src/meta-service/meta_service.cpp | 3 +++
cloud/src/meta-service/meta_service_txn.cpp | 6 +++++-
.../java/org/apache/doris/catalog/OlapTable.java | 15 ++++++++++++++
.../java/org/apache/doris/catalog/Partition.java | 7 +++++++
.../apache/doris/cloud/catalog/CloudPartition.java | 24 ++++++++++++++++------
.../cloud/datasource/CloudInternalCatalog.java | 4 ----
.../transaction/CloudGlobalTransactionMgr.java | 4 +++-
.../doris/common/NereidsSqlCacheManager.java | 9 ++++----
.../org/apache/doris/nereids/SqlCacheContext.java | 1 -
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 11 ++++++----
gensrc/proto/cloud.proto | 3 +++
11 files changed, 66 insertions(+), 21 deletions(-)
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index b234a48d88a..0bb1fb5f277 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -269,6 +269,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller,
return;
}
response->set_version(version_pb.version());
+ response->add_version_update_time_ms(version_pb.update_time_ms());
}
{ TEST_SYNC_POINT_CALLBACK("get_version_code", &code); }
return;
@@ -373,6 +374,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
if (!value.has_value()) {
// return -1 if the target version is not exists.
response->add_versions(-1);
+ response->add_version_update_time_ms(-1);
} else if (is_table_version) {
int64_t version = 0;
if (!txn->decode_atomic_int(*value, &version)) {
@@ -389,6 +391,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr
break;
}
response->add_versions(version_pb.version());
+ response->add_version_update_time_ms(version_pb.update_time_ms());
}
}
}
diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp
index 729696f8e2f..69fecc80788 100644
--- a/cloud/src/meta-service/meta_service_txn.cpp
+++ b/cloud/src/meta-service/meta_service_txn.cpp
@@ -1046,10 +1046,14 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller,
}
// Save versions
+ int64_t version_update_time_ms =
+ duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+ response->set_version_update_time_ms(version_update_time_ms);
for (auto& i : new_versions) {
std::string ver_val;
VersionPB version_pb;
version_pb.set_version(i.second);
+ version_pb.set_update_time_ms(version_update_time_ms);
if (!version_pb.SerializeToString(&ver_val)) {
code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
ss << "failed to serialize version_pb when saving, txn_id=" << txn_id;
@@ -1059,7 +1063,7 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller,
txn->put(i.first, ver_val);
LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second
- << " txn_id=" << txn_id;
+ << " txn_id=" << txn_id << " update_time=" << version_update_time_ms;
std::string_view ver_key = i.first;
ver_key.remove_prefix(1); // Remove key space
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index f5e3cc90e85..d5b1c258c5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1149,6 +1149,21 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
return partition;
}
+ public void getVersionInBatchForCloudMode(Collection<Long> partitionIds) {
+ if (Config.isCloudMode()) { // do nothing for non-cloud mode
+ List<CloudPartition> partitions = partitionIds.stream()
+ .sorted()
+ .map(this::getPartition)
+ .map(partition -> (CloudPartition) partition)
+ .collect(Collectors.toList());
+ try {
+ CloudPartition.getSnapshotVisibleVersion(partitions);
+ } catch (RpcException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
// select the non-empty partition ids belonging to this table.
//
// ATTN: partitions not belonging to this table will be filtered.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
index 775ee5530c5..ef94da902b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
@@ -163,6 +163,13 @@ public class Partition extends MetaObject {
this.setVisibleVersionAndTime(visibleVersion, visibleVersionTime);
}
+ /* fromCache is only used in CloudPartition
+ * make it overrided here to avoid rewrite all the usages with ugly Config.isCloudConfig() branches
+ */
+ public long getVisibleVersion(Boolean fromCache) {
+ return visibleVersion;
+ }
+
public long getVisibleVersion() {
return visibleVersion;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
index b3f868ebe12..1246c5b640b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java
@@ -91,17 +91,22 @@ public class CloudPartition extends Partition {
return;
}
- public void setCachedVisibleVersion(long version) {
+ public void setCachedVisibleVersion(long version, Long versionUpdateTimeMs) {
// we only care the version should increase monotonically and ignore the readers
LOG.debug("setCachedVisibleVersion use CloudPartition {}, version: {}, old version: {}",
super.getId(), version, super.getVisibleVersion());
lock.lock();
if (version > super.getVisibleVersion()) {
- super.setVisibleVersion(version);
+ super.setVisibleVersionAndTime(version, versionUpdateTimeMs);
}
lock.unlock();
}
+ @Override
+ public long getVisibleVersion(Boolean fromCache) {
+ return super.getVisibleVersion();
+ }
+
@Override
public long getVisibleVersion() {
if (Env.isCheckpointThread() || Config.enable_check_compatibility_mode) {
@@ -125,7 +130,8 @@ public class CloudPartition extends Partition {
if (resp.getStatus().getCode() == MetaServiceCode.OK) {
version = resp.getVersion();
// Cache visible version, see hasData() for details.
- setCachedVisibleVersion(version);
+ assert resp.getVersionUpdateTimeMsList().size() == 1;
+ setCachedVisibleVersion(version, resp.getVersionUpdateTimeMs(0));
} else {
assert resp.getStatus().getCode() == MetaServiceCode.VERSION_NOT_FOUND;
version = Partition.PARTITION_INIT_VERSION;
@@ -185,20 +191,21 @@ public class CloudPartition extends Partition {
List<Long> dbIds = new ArrayList<>();
List<Long> tableIds = new ArrayList<>();
List<Long> partitionIds = new ArrayList<>();
+ List<Long> versionUpdateTimesMs = new ArrayList<>();
for (CloudPartition partition : partitions) {
dbIds.add(partition.getDbId());
tableIds.add(partition.getTableId());
partitionIds.add(partition.getId());
}
- List<Long> versions = getSnapshotVisibleVersion(dbIds, tableIds, partitionIds);
+ List<Long> versions = getSnapshotVisibleVersion(dbIds, tableIds, partitionIds, versionUpdateTimesMs);
// Cache visible version, see hasData() for details.
int size = versions.size();
for (int i = 0; i < size; ++i) {
Long version = versions.get(i);
if (version > Partition.PARTITION_INIT_VERSION) {
- partitions.get(i).setCachedVisibleVersion(versions.get(i));
+ partitions.get(i).setCachedVisibleVersion(versions.get(i), versionUpdateTimesMs.get(i));
}
}
@@ -208,7 +215,8 @@ public class CloudPartition extends Partition {
// Get visible versions for the specified partitions.
//
// Return the visible version in order of the specified partition ids, -1 means version NOT FOUND.
- public static List<Long> getSnapshotVisibleVersion(List<Long> dbIds, List<Long> tableIds, List<Long> partitionIds)
+ public static List<Long> getSnapshotVisibleVersion(List<Long> dbIds, List<Long> tableIds, List<Long> partitionIds,
+ List<Long> versionUpdateTimesMs)
throws RpcException {
assert dbIds.size() == partitionIds.size() :
"partition ids size: " + partitionIds.size() + " should equals to db ids size: " + dbIds.size();
@@ -251,6 +259,10 @@ public class CloudPartition extends Partition {
return news;
}
+ if (versionUpdateTimesMs != null) {
+ versionUpdateTimesMs.addAll(resp.getVersionUpdateTimeMsList());
+ }
+
return versions;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 879621faef9..3ebc9d13808 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -125,10 +125,6 @@ public class CloudInternalCatalog extends InternalCatalog {
indexMap.put(indexId, rollup);
}
- // version and version hash
- if (versionInfo != null) {
- partition.updateVisibleVersion(versionInfo);
- }
long version = partition.getVisibleVersion();
final String storageVaultName = tbl.getStorageVaultName();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
index d3cd92e52f5..b433d8cfbe7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java
@@ -385,7 +385,9 @@ public class CloudGlobalTransactionMgr implements GlobalTransactionMgrIface {
if (partition == null) {
continue;
}
- partition.setCachedVisibleVersion(version);
+ partition.setCachedVisibleVersion(version, commitTxnResponse.getVersionUpdateTimeMs());
+ LOG.info("Update Partition. transactionId:{}, table_id:{}, partition_id:{}, version:{}, update time:{}",
+ txnId, tableId, partition.getId(), version, commitTxnResponse.getVersionUpdateTimeMs());
}
env.getAnalysisManager().setNewPartitionLoaded(
tablePartitionMap.keySet().stream().collect(Collectors.toList()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
index cf6280650f0..b661196dd3b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java
@@ -62,6 +62,7 @@ import org.apache.commons.collections.CollectionUtils;
import java.lang.reflect.Field;
import java.time.Duration;
+import java.util.Collection;
import java.util.List;
import java.util.Map.Entry;
import java.util.Objects;
@@ -253,16 +254,16 @@ public class NereidsSqlCacheManager {
return true;
}
OlapTable olapTable = (OlapTable) tableIf;
- long currentTableTime = olapTable.getVisibleVersionTime();
- long cacheTableTime = scanTable.latestTimestamp;
long currentTableVersion = olapTable.getVisibleVersion();
long cacheTableVersion = scanTable.latestVersion;
// some partitions have been dropped, or delete or updated or replaced, or insert rows into new partition?
- if (currentTableTime > cacheTableTime
- || (currentTableTime == cacheTableTime && currentTableVersion > cacheTableVersion)) {
+ if (currentTableVersion != cacheTableVersion) {
return true;
}
+ Collection<Long> partitionIds = scanTable.getScanPartitions();
+ olapTable.getVersionInBatchForCloudMode(partitionIds);
+
for (Long scanPartitionId : scanTable.getScanPartitions()) {
Partition partition = olapTable.getPartition(scanPartitionId);
// partition == null: is this partition truncated?
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
index f3fa61cecaa..f8d5a00b9f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java
@@ -422,7 +422,6 @@ public class SqlCacheContext {
@lombok.AllArgsConstructor
public static class ScanTable {
public final FullTableName fullTableName;
- public final long latestTimestamp;
public final long latestVersion;
public final List<Long> scanPartitions = Lists.newArrayList();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index c88f3bd13a1..17be5c42e22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -693,15 +693,19 @@ public class CacheAnalyzer {
CatalogIf catalog = database.getCatalog();
ScanTable scanTable = new ScanTable(
new FullTableName(catalog.getName(), database.getFullName(), olapTable.getName()),
- olapTable.getVisibleVersionTime(), olapTable.getVisibleVersion());
+ olapTable.getVisibleVersion());
scanTables.add(scanTable);
+
+ Collection<Long> partitionIds = node.getSelectedPartitionIds();
+ olapTable.getVersionInBatchForCloudMode(partitionIds);
+
for (Long partitionId : node.getSelectedPartitionIds()) {
Partition partition = olapTable.getPartition(partitionId);
scanTable.addScanPartition(partitionId);
if (partition.getVisibleVersionTime() >= cacheTable.latestPartitionTime) {
cacheTable.latestPartitionId = partition.getId();
cacheTable.latestPartitionTime = partition.getVisibleVersionTime();
- cacheTable.latestPartitionVersion = partition.getVisibleVersion();
+ cacheTable.latestPartitionVersion = partition.getVisibleVersion(true);
}
}
return cacheTable;
@@ -716,8 +720,7 @@ public class CacheAnalyzer {
DatabaseIf database = tableIf.getDatabase();
CatalogIf catalog = database.getCatalog();
ScanTable scanTable = new ScanTable(new FullTableName(
- catalog.getName(), database.getFullName(), tableIf.getName()
- ), cacheTable.latestPartitionTime, 0);
+ catalog.getName(), database.getFullName(), tableIf.getName()), 0);
scanTables.add(scanTable);
return cacheTable;
}
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index f285ad3f260..ab69deaa212 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -403,6 +403,7 @@ message TxnRunningPB {
message VersionPB {
optional int64 version = 1;
+ optional int64 update_time_ms = 2;
}
message RecycleTxnPB {
@@ -685,6 +686,7 @@ message CommitTxnResponse {
repeated int64 partition_ids = 4;
repeated int64 versions = 5;
repeated TableStatsPB table_stats = 6;
+ optional int64 version_update_time_ms = 7;
}
message AbortTxnRequest {
@@ -818,6 +820,7 @@ message GetVersionResponse {
repeated int64 table_ids = 4;
repeated int64 partition_ids = 5;
repeated int64 versions = 6;
+ repeated int64 version_update_time_ms = 7;
};
message GetObjStoreInfoRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 13/21: [fix](mtmv)fix mtmv dead lock (#37009)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 05ca6c66f51d265d321ffbed6bae37d9183f1b89
Author: zhangdong <49...@qq.com>
AuthorDate: Fri Jun 28 21:09:35 2024 +0800
[fix](mtmv)fix mtmv dead lock (#37009)
Cause of occurrence：
- dropBaseTable: Holding db's writeLock ,notify mtmv alter status to
schema_change,need mv's writeLock
- task(insert overwrite):Holding mv's readLock,when generage plan need
db's readLock
fix:
- mtmv alter status to schema_change need mv's writeMvLock instead of
mv's writeLock
---
fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index a029f604dfd..26045f8527a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -916,7 +916,7 @@ public class Alter {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb());
mtmv = (MTMV) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW);
- mtmv.writeLock();
+ mtmv.writeMvLock();
switch (alterMTMV.getOpType()) {
case ALTER_REFRESH_INFO:
mtmv.alterRefreshInfo(alterMTMV.getRefreshInfo());
@@ -945,7 +945,7 @@ public class Alter {
LOG.warn(e);
} finally {
if (mtmv != null) {
- mtmv.writeUnlock();
+ mtmv.writeMvUnlock();
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/21: [feature](insert)support external hive truncate table DDL (#36801)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 018ff0ae94cb16df3f120b08e135e078f25ab741
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Fri Jun 28 17:27:44 2024 +0800
[feature](insert)support external hive truncate table DDL (#36801)
## Proposed changes
Issue Number: #31442
1. support hive3 'truncate table'
2. forbiden hive2 'truncate table' because of it is not supported by HMS
Client API on hive2
---
.../apache/doris/analysis/TruncateTableStmt.java | 3 -
.../main/java/org/apache/doris/catalog/Env.java | 6 +-
.../org/apache/doris/datasource/CatalogIf.java | 3 +
.../apache/doris/datasource/ExternalCatalog.java | 23 +++++
.../doris/datasource/hive/HMSCachedClient.java | 2 +
.../doris/datasource/hive/HiveMetadataOps.java | 16 ++++
.../hive/PostgreSQLJdbcHMSCachedClient.java | 4 +
.../datasource/hive/ThriftHMSCachedClient.java | 19 +++-
.../datasource/iceberg/IcebergMetadataOps.java | 5 ++
.../datasource/operations/ExternalMetadataOps.java | 8 ++
.../doris/datasource/TestHMSCachedClient.java | 3 +
.../hive/ddl/test_hive_truncate_table.out | 24 +++++
.../hive/ddl/test_hive_truncate_table.groovy | 100 +++++++++++++++++++++
13 files changed, 210 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
index a275879692c..1a9fbd3bafd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
@@ -23,7 +23,6 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.InternalDatabaseUtil;
-import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -44,8 +43,6 @@ public class TruncateTableStmt extends DdlStmt {
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
tblRef.getName().analyze(analyzer);
- // disallow external catalog
- Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName());
if (tblRef.hasExplicitAlias()) {
throw new AnalysisException("Not support truncate table with alias");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 207a16a2678..a3da3482f77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -5577,8 +5577,10 @@ public class Env {
* otherwise, it will only truncate those specified partitions.
*
*/
- public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException {
- getInternalCatalog().truncateTable(truncateTableStmt);
+ public void truncateTable(TruncateTableStmt stmt) throws DdlException {
+ CatalogIf<?> catalogIf = catalogMgr.getCatalogOrException(stmt.getTblRef().getName().getCtl(),
+ catalog -> new DdlException(("Unknown catalog " + catalog)));
+ catalogIf.truncateTable(stmt);
}
public void replayTruncateTable(TruncateTableInfo info) throws MetaNotFoundException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index ceee1a68157..a79897c67df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
@@ -194,4 +195,6 @@ public interface CatalogIf<T extends DatabaseIf> {
boolean createTable(CreateTableStmt stmt) throws UserException;
void dropTable(DropTableStmt stmt) throws DdlException;
+
+ void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlException;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 929378ae80f..b643176aaf1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -22,6 +22,8 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TableRef;
+import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
@@ -832,4 +834,25 @@ public abstract class ExternalCatalog
}
return ret;
}
+
+ @Override
+ public void truncateTable(TruncateTableStmt stmt) throws DdlException {
+ makeSureInitialized();
+ if (metadataOps == null) {
+ throw new UnsupportedOperationException("Truncate table not supported in " + getName());
+ }
+ try {
+ TableRef tableRef = stmt.getTblRef();
+ TableName tableName = tableRef.getName();
+ // delete all table data if null
+ List<String> partitions = null;
+ if (tableRef.getPartitionNames() != null) {
+ partitions = tableRef.getPartitionNames().getPartitionNames();
+ }
+ metadataOps.truncateTable(tableName.getDb(), tableName.getTbl(), partitions);
+ } catch (Exception e) {
+ LOG.warn("Failed to drop a table", e);
+ throw e;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
index c9d0ce1736b..1ac77972053 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCachedClient.java
@@ -94,6 +94,8 @@ public interface HMSCachedClient {
void dropTable(String dbName, String tableName);
+ void truncateTable(String dbName, String tblName, List<String> partitions);
+
void createTable(TableMetadata catalogTable, boolean ignoreIfExists);
void updateTableStatistics(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
index 70c61875b8a..7267297c93e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
@@ -251,6 +251,22 @@ public class HiveMetadataOps implements ExternalMetadataOps {
}
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException {
+ ExternalDatabase<?> db = catalog.getDbNullable(dbName);
+ if (db == null) {
+ throw new DdlException("Failed to get database: '" + dbName + "' in catalog: " + catalog.getName());
+ }
+ try {
+ client.truncateTable(dbName, tblName, partitions);
+ } catch (Exception e) {
+ throw new DdlException(e.getMessage(), e);
+ }
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(catalog.getId(), dbName, tblName);
+ db.setLastUpdateTime(System.currentTimeMillis());
+ db.setUnInitialized(true);
+ }
+
@Override
public List<String> listTableNames(String dbName) {
return client.getAllTables(dbName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
index 8e41b48bfdd..0cdb3e469c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
@@ -546,6 +546,10 @@ public class PostgreSQLJdbcHMSCachedClient extends JdbcHMSCachedClient {
throw new NotImplementedException("PostgreSQL dropDatabase not implemented");
}
+ public void truncateTable(String dbName, String tblName, List<String> partitions) {
+ throw new NotImplementedException("PostgreSQL truncateTable not implemented");
+ }
+
public void createTable(TableMetadata hiveTable, boolean ignoreIfExists) {
throw new NotImplementedException("PostgreSQL createTable not implemented");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
index bc4da6762ca..fcabcc2565a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java
@@ -244,6 +244,23 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
}
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String> partitions) {
+ try (ThriftHMSClient client = getClient()) {
+ try {
+ ugiDoAs(() -> {
+ client.client.truncateTable(dbName, tblName, partitions);
+ return null;
+ });
+ } catch (Exception e) {
+ client.setThrowable(e);
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new HMSClientException("failed to truncate table %s in db %s.", e, tblName, dbName);
+ }
+ }
+
@Override
public boolean tableExists(String dbName, String tblName) {
try (ThriftHMSClient client = getClient()) {
@@ -272,7 +289,7 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
throw e;
}
} catch (Exception e) {
- throw new HMSClientException("failed to check if table %s in db %s exists", e, tblName, dbName);
+ throw new HMSClientException("failed to list partitions in table '%s.%s'.", e, dbName, tblName);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 7bd8409ca2c..3236367afc0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -87,6 +87,7 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
.collect(Collectors.toList());
}
+
@Override
public List<String> listTableNames(String dbName) {
List<TableIdentifier> tableIdentifiers = catalog.listTables(Namespace.of(dbName));
@@ -178,4 +179,8 @@ public class IcebergMetadataOps implements ExternalMetadataOps {
db.setUnInitialized(true);
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String> partitions) {
+ throw new UnsupportedOperationException("Truncate Iceberg table is not supported.");
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
index 9426442cebb..0333124b352 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java
@@ -60,6 +60,14 @@ public interface ExternalMetadataOps {
*/
void dropTable(DropTableStmt stmt) throws DdlException;
+ /**
+ *
+ * @param dbName
+ * @param tblName
+ * @param partitions
+ */
+ void truncateTable(String dbName, String tblName, List<String> partitions) throws DdlException;
+
/**
*
* @return
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
index 6f969257245..f3cb918d6f5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/TestHMSCachedClient.java
@@ -235,6 +235,9 @@ public class TestHMSCachedClient implements HMSCachedClient {
this.partitions.remove(new HMSTransaction.DatabaseTableName(dbName, tableName));
}
+ @Override
+ public void truncateTable(String dbName, String tblName, List<String> partitions) {}
+
@Override
public void createTable(TableMetadata tbl, boolean ignoreIfExists) {
String dbName = tbl.getDbName();
diff --git a/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out b/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
new file mode 100644
index 00000000000..24d6cade5d2
--- /dev/null
+++ b/regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !truncate_01_pre --
+222 aoe
+3234424 44
+
+-- !truncate_01 --
+
+-- !truncate_02 --
+
+-- !truncate_03 --
+
+-- !truncate_04_pre --
+33 awe wuu 2023-02-04
+5535 3 dre 2023-04-04
+
+-- !truncate_04 --
+
+-- !truncate_06 --
+44 etg wuweu 2022-02-04
+555 etgf wet 2021-05-06
+88 etg wuweu 2022-01-04
+95 etgf hiyr 2021-05-06
+
+-- !truncate_09 --
diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
new file mode 100644
index 00000000000..94082d0287f
--- /dev/null
+++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_truncate_table", "p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String hms_port = context.config.otherConfigs.get("hive3HmsPort")
+ String hdfs_port = context.config.otherConfigs.get("hive3HdfsPort")
+ String catalog_name = "test_hive3_truncate_table"
+ sql """drop catalog if exists ${catalog_name};"""
+
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'use_meta_cache' = 'true',
+ 'hive.version'='3.0'
+ );
+ """
+
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """create database if not exists `hive_truncate`;"""
+ sql """use `hive_truncate`;"""
+
+ // 1. test no partition table
+ sql """create table if not exists `table_no_pars`(col1 bigint, col2 string) engine=hive; """
+ sql """truncate table table_no_pars;"""
+
+ sql """insert into `table_no_pars` values(3234424, '44'); """
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ order_qt_truncate_01_pre """ select * from table_no_pars; """
+ sql """truncate table table_no_pars;"""
+ order_qt_truncate_01 """ select * from table_no_pars; """
+
+ sql """insert into `table_no_pars` values(3234424, '44'); """
+ sql """truncate table hive_truncate.table_no_pars;"""
+ order_qt_truncate_02 """ select * from table_no_pars; """
+
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ sql """truncate table ${catalog_name}.hive_truncate.table_no_pars;"""
+ order_qt_truncate_03 """ select * from table_no_pars; """
+ sql """insert into `table_no_pars` values(222, 'aoe'); """
+ sql """drop table table_no_pars """
+
+ // 2. test partition table
+ sql """create table if not exists `table_with_pars`(col1 bigint, col2 string, pt1 varchar, pt2 date) engine=hive
+ partition by list(pt1, pt2)() """
+ sql """truncate table table_with_pars;"""
+
+ sql """insert into `table_with_pars` values(33, 'awe', 'wuu', '2023-02-04') """
+ sql """insert into `table_with_pars` values(5535, '3', 'dre', '2023-04-04') """
+ order_qt_truncate_04_pre """ select * from table_with_pars; """
+ sql """truncate table table_with_pars;"""
+ order_qt_truncate_04 """ select * from table_with_pars; """
+
+ // 3. test partition table and truncate partitions
+ sql """insert into `table_with_pars` values(44, 'etg', 'wuweu', '2022-02-04') """
+ sql """insert into `table_with_pars` values(88, 'etg', 'wuweu', '2022-01-04') """
+ sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr', '2021-05-06') """
+ sql """insert into `table_with_pars` values(555, 'etgf', 'wet', '2021-05-06') """
+ // sql """truncate table hive_truncate.table_with_pars partition pt1;"""
+ // order_qt_truncate_05 """ select * from table_with_pars; """
+ // sql """truncate table hive_truncate.table_with_pars partition pt2;"""
+ order_qt_truncate_06 """ select * from table_with_pars; """
+
+ sql """insert into `table_with_pars` values(22, 'ttt', 'gggw', '2022-02-04')"""
+ sql """insert into `table_with_pars` values(44, 'etg', 'wuweu', '2022-02-04') """
+ sql """insert into `table_with_pars` values(88, 'etg', 'wuweu', '2022-01-04') """
+ sql """insert into `table_with_pars` values(095, 'etgf', 'hiyr', '2021-05-06') """
+ sql """insert into `table_with_pars` values(555, 'etgf', 'wet', '2021-05-06') """
+ // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt1;"""
+ // order_qt_truncate_07 """ select * from table_with_pars; """
+ // sql """truncate table ${catalog_name}.hive_truncate.table_with_pars partition pt2;"""
+ // order_qt_truncate_08 """ select * from table_with_pars; """
+ sql """truncate table table_with_pars"""
+ order_qt_truncate_09 """ select * from table_with_pars; """
+
+ sql """drop table table_with_pars """
+ sql """drop database hive_truncate;"""
+ sql """drop catalog ${catalog_name};"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org