You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/07/31 07:42:03 UTC
(doris) branch branch-3.0 updated (879a6577bfe -> 397cc68da32)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 879a6577bfe [Feature](multi-catalog) Add memory tracker for orc reader/writer and arrow parquet writer。 (#37234)
new df07690109a [Improvement][Audit] add sql type for query audit (#37790)
new a9e4ffdbd96 [Feature][external catalog/lakesoul] LakeSoul Catalog support Filter Pushdown & Cdc data handling & S3 data access (#37979)
new 397cc68da32 [chore](bash) support read env variables from doris_cloud.conf (#38060)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
cloud/script/start.sh | 14 +
fe/be-java-extensions/lakesoul-scanner/pom.xml | 22 +-
.../apache/doris/lakesoul/LakeSoulJniScanner.java | 82 +++-
.../org/apache/doris/lakesoul/LakeSoulUtils.java | 30 +-
.../apache/doris/lakesoul/arrow/ArrowUtils.java | 8 +-
.../lakesoul/arrow/LakeSoulArrowJniScanner.java | 44 +-
fe/fe-core/pom.xml | 44 +-
.../doris/analysis/AlterCatalogCommentStmt.java | 5 +
.../doris/analysis/AlterCatalogNameStmt.java | 5 +
.../doris/analysis/AlterCatalogPropertyStmt.java | 5 +
.../doris/analysis/AlterColocateGroupStmt.java | 5 +
.../doris/analysis/AlterColumnStatsStmt.java | 5 +
.../doris/analysis/AlterDatabasePropertyStmt.java | 5 +
.../doris/analysis/AlterDatabaseQuotaStmt.java | 5 +
.../apache/doris/analysis/AlterDatabaseRename.java | 5 +
.../apache/doris/analysis/AlterJobStatusStmt.java | 5 +
.../org/apache/doris/analysis/AlterPolicyStmt.java | 5 +
.../apache/doris/analysis/AlterRepositoryStmt.java | 5 +
.../apache/doris/analysis/AlterResourceStmt.java | 5 +
.../org/apache/doris/analysis/AlterRoleStmt.java | 5 +
.../doris/analysis/AlterRoutineLoadStmt.java | 6 +
.../doris/analysis/AlterSqlBlockRuleStmt.java | 5 +
.../org/apache/doris/analysis/AlterSystemStmt.java | 5 +
.../org/apache/doris/analysis/AlterTableStmt.java | 5 +
.../org/apache/doris/analysis/AlterUserStmt.java | 5 +
.../org/apache/doris/analysis/AlterViewStmt.java | 5 +
.../doris/analysis/AlterWorkloadGroupStmt.java | 5 +
.../org/apache/doris/analysis/AnalyzeStmt.java | 5 +
.../java/org/apache/doris/analysis/BackupStmt.java | 6 +
.../doris/analysis/CancelAlterSystemStmt.java | 5 +
.../doris/analysis/CancelAlterTableStmt.java | 5 +
.../apache/doris/analysis/CancelBackupStmt.java | 5 +
.../apache/doris/analysis/CancelExportStmt.java | 5 +
.../apache/doris/analysis/CancelJobTaskStmt.java | 5 +
.../org/apache/doris/analysis/CancelLoadStmt.java | 5 +
.../org/apache/doris/analysis/CleanLabelStmt.java | 5 +
.../apache/doris/analysis/CleanProfileStmt.java | 5 +
.../apache/doris/analysis/CleanQueryStatsStmt.java | 5 +
.../apache/doris/analysis/CreateCatalogStmt.java | 5 +
.../doris/analysis/CreateDataSyncJobStmt.java | 5 +
.../org/apache/doris/analysis/CreateDbStmt.java | 5 +
.../doris/analysis/CreateEncryptKeyStmt.java | 5 +
.../org/apache/doris/analysis/CreateFileStmt.java | 5 +
.../apache/doris/analysis/CreateFunctionStmt.java | 5 +
.../doris/analysis/CreateMaterializedViewStmt.java | 5 +
.../apache/doris/analysis/CreatePolicyStmt.java | 5 +
.../doris/analysis/CreateRepositoryStmt.java | 5 +
.../apache/doris/analysis/CreateResourceStmt.java | 5 +
.../org/apache/doris/analysis/CreateRoleStmt.java | 5 +
.../doris/analysis/CreateRoutineLoadStmt.java | 5 +
.../doris/analysis/CreateSqlBlockRuleStmt.java | 5 +
.../doris/analysis/CreateTableAsSelectStmt.java | 5 +
.../apache/doris/analysis/CreateTableLikeStmt.java | 5 +
.../org/apache/doris/analysis/CreateTableStmt.java | 5 +
.../org/apache/doris/analysis/CreateUserStmt.java | 5 +
.../org/apache/doris/analysis/CreateViewStmt.java | 5 +
.../doris/analysis/CreateWorkloadGroupStmt.java | 5 +
.../analysis/CreateWorkloadSchedPolicyStmt.java | 5 +
.../java/org/apache/doris/analysis/DdlStmt.java | 5 +
.../java/org/apache/doris/analysis/DeleteStmt.java | 5 +
.../apache/doris/analysis/DropAnalyzeJobStmt.java | 5 +
.../org/apache/doris/analysis/DropCatalogStmt.java | 5 +
.../java/org/apache/doris/analysis/DropDbStmt.java | 5 +
.../apache/doris/analysis/DropEncryptKeyStmt.java | 5 +
.../org/apache/doris/analysis/DropFileStmt.java | 5 +
.../apache/doris/analysis/DropFunctionStmt.java | 5 +
.../doris/analysis/DropMaterializedViewStmt.java | 5 +
.../org/apache/doris/analysis/DropPolicyStmt.java | 5 +
.../apache/doris/analysis/DropRepositoryStmt.java | 5 +
.../apache/doris/analysis/DropResourceStmt.java | 5 +
.../org/apache/doris/analysis/DropRoleStmt.java | 5 +
.../doris/analysis/DropSqlBlockRuleStmt.java | 5 +
.../org/apache/doris/analysis/DropStatsStmt.java | 5 +
.../org/apache/doris/analysis/DropTableStmt.java | 5 +
.../org/apache/doris/analysis/DropUserStmt.java | 5 +
.../doris/analysis/DropWorkloadGroupStmt.java | 5 +
.../analysis/DropWorkloadSchedPolicyStmt.java | 5 +
.../org/apache/doris/analysis/ExecuteStmt.java | 5 +
.../java/org/apache/doris/analysis/ExportStmt.java | 5 +
.../java/org/apache/doris/analysis/GrantStmt.java | 5 +
.../doris/analysis/InsertOverwriteTableStmt.java | 6 +
.../java/org/apache/doris/analysis/InsertStmt.java | 5 +
.../apache/doris/analysis/InstallPluginStmt.java | 5 +
.../apache/doris/analysis/KillAnalysisJobStmt.java | 5 +
.../java/org/apache/doris/analysis/KillStmt.java | 5 +
.../java/org/apache/doris/analysis/LoadStmt.java | 4 +
.../doris/analysis/PauseRoutineLoadStmt.java | 5 +
.../apache/doris/analysis/PauseSyncJobStmt.java | 5 +
.../org/apache/doris/analysis/RecoverDbStmt.java | 5 +
.../doris/analysis/RecoverPartitionStmt.java | 5 +
.../apache/doris/analysis/RecoverTableStmt.java | 5 +
.../apache/doris/analysis/RefreshCatalogStmt.java | 4 +
.../org/apache/doris/analysis/RefreshDbStmt.java | 5 +
.../org/apache/doris/analysis/RefreshLdapStmt.java | 5 +
.../apache/doris/analysis/RefreshTableStmt.java | 5 +
.../org/apache/doris/analysis/RestoreStmt.java | 5 +
.../doris/analysis/ResumeRoutineLoadStmt.java | 5 +
.../apache/doris/analysis/ResumeSyncJobStmt.java | 5 +
.../java/org/apache/doris/analysis/RevokeStmt.java | 5 +
.../java/org/apache/doris/analysis/SelectStmt.java | 5 +
.../apache/doris/analysis/SetOperationStmt.java | 7 +
.../java/org/apache/doris/analysis/SetStmt.java | 5 +
.../apache/doris/analysis/SetUserPropertyStmt.java | 5 +
.../java/org/apache/doris/analysis/ShowStmt.java | 5 +
.../org/apache/doris/analysis/StatementBase.java | 4 +
.../{KillAnalysisJobStmt.java => StmtType.java} | 48 +-
.../apache/doris/analysis/StopRoutineLoadStmt.java | 5 +
.../org/apache/doris/analysis/StopSyncJobStmt.java | 5 +
.../java/org/apache/doris/analysis/SwitchStmt.java | 5 +
.../java/org/apache/doris/analysis/SyncStmt.java | 5 +
.../org/apache/doris/analysis/TransactionStmt.java | 5 +
.../apache/doris/analysis/TruncateTableStmt.java | 5 +
.../apache/doris/analysis/UninstallPluginStmt.java | 5 +
.../apache/doris/analysis/UnsetVariableStmt.java | 5 +
.../java/org/apache/doris/analysis/UpdateStmt.java | 5 +
.../java/org/apache/doris/analysis/UseStmt.java | 5 +
.../org/apache/doris/catalog/InternalSchema.java | 1 +
.../lakesoul/LakeSoulExternalCatalog.java | 22 +-
.../datasource/lakesoul/LakeSoulExternalTable.java | 22 +-
.../doris/datasource/lakesoul/LakeSoulUtils.java | 535 +++++++++++++++++++++
.../lakesoul/source/LakeSoulScanNode.java | 158 +++++-
.../doris/nereids/analyzer/UnboundResultSink.java | 6 +
.../doris/nereids/rules/analysis/BindRelation.java | 1 +
.../trees/plans/commands/AlterMTMVCommand.java | 5 +
.../trees/plans/commands/AlterViewCommand.java | 6 +
.../nereids/trees/plans/commands/CallCommand.java | 5 +
.../plans/commands/CancelMTMVTaskCommand.java | 6 +
.../trees/plans/commands/CreateMTMVCommand.java | 6 +
.../trees/plans/commands/CreatePolicyCommand.java | 6 +
.../plans/commands/CreateProcedureCommand.java | 6 +
.../trees/plans/commands/CreateTableCommand.java | 6 +
.../plans/commands/CreateTableLikeCommand.java | 6 +
.../trees/plans/commands/CreateViewCommand.java | 6 +
.../trees/plans/commands/DeleteFromCommand.java | 6 +
.../plans/commands/DeleteFromUsingCommand.java | 6 +
.../trees/plans/commands/DropMTMVCommand.java | 6 +
.../trees/plans/commands/DropProcedureCommand.java | 6 +
.../trees/plans/commands/ExecuteCommand.java | 6 +
.../trees/plans/commands/ExplainCommand.java | 6 +
.../trees/plans/commands/ExportCommand.java | 6 +
.../nereids/trees/plans/commands/LoadCommand.java | 6 +
.../trees/plans/commands/PauseMTMVCommand.java | 6 +
.../trees/plans/commands/PrepareCommand.java | 6 +
.../trees/plans/commands/RefreshMTMVCommand.java | 6 +
.../trees/plans/commands/ResumeMTMVCommand.java | 6 +
.../plans/commands/ShowConstraintsCommand.java | 6 +
.../plans/commands/ShowCreateProcedureCommand.java | 6 +
.../plans/commands/ShowProcedureStatusCommand.java | 6 +
.../trees/plans/commands/UpdateCommand.java | 6 +
.../insert/BatchInsertIntoTableCommand.java | 6 +
.../commands/insert/InsertIntoTableCommand.java | 6 +
.../insert/InsertOverwriteTableCommand.java | 6 +
.../nereids/trees/plans/logical/LogicalPlan.java | 5 +
.../org/apache/doris/plugin/audit/AuditEvent.java | 7 +
.../doris/plugin/audit/AuditLoaderPlugin.java | 1 +
.../java/org/apache/doris/qe/AuditLogHelper.java | 18 +
.../doris/alter/InternalSchemaAlterTest.java | 14 +
.../org/apache/doris/analysis/QueryStmtTest.java | 29 ++
.../datasource/lakesoul/LakeSoulPredicateTest.java | 280 +++++++++++
.../doris/nereids/parser/NereidsParserTest.java | 21 +
.../apache/doris/qe/AuditEventProcessorTest.java | 2 +
.../org/apache/doris/utframe/UtFrameUtils.java | 20 +
fe/pom.xml | 4 +-
.../apache/doris/plugin/audit/AuditPluginDemo.java | 1 -
.../src/main/assembly/plugin.properties | 2 +-
.../audit/{ => custom}/AuditLoaderPlugin.java | 15 +-
.../audit/{ => custom}/DorisStreamLoader.java | 17 +-
.../dialect/spark/SparkSql3LogicalPlanBuilder.java | 2 +-
.../spark/SparkSqlDialectConverterPlugin.java | 2 +-
.../dialect/trino/TrinoLogicalPlanBuilder.java | 2 +-
regression-test/conf/regression-conf.groovy | 11 +
.../lakesoul/test_lakesoul_filter.out | 8 +
.../pipeline/external/conf/regression-conf.groovy | 4 +
.../lakesoul/test_lakesoul_catalog.groovy | 23 +-
.../lakesoul/test_lakesoul_filter.groovy | 58 +++
.../lakesoul/test_external_table_lakesoul.groovy | 24 +-
176 files changed, 2185 insertions(+), 136 deletions(-)
copy fe/fe-core/src/main/java/org/apache/doris/analysis/{KillAnalysisJobStmt.java => StmtType.java} (63%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lakesoul/LakeSoulPredicateTest.java
rename fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/{ => custom}/AuditLoaderPlugin.java (97%)
rename fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/{ => custom}/DorisStreamLoader.java (92%)
create mode 100644 regression-test/data/external_table_p0/lakesoul/test_lakesoul_filter.out
create mode 100644 regression-test/suites/external_table_p0/lakesoul/test_lakesoul_filter.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/03: [chore](bash) support read env variables from doris_cloud.conf (#38060)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 397cc68da3231eda1baa1bbc07eebdec7a85e5fc
Author: yagagagaga <zh...@gmail.com>
AuthorDate: Tue Jul 23 20:20:31 2024 +0800
[chore](bash) support read env variables from doris_cloud.conf (#38060)
---
cloud/script/start.sh | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/cloud/script/start.sh b/cloud/script/start.sh
index e119e193d82..94638957a86 100644
--- a/cloud/script/start.sh
+++ b/cloud/script/start.sh
@@ -40,6 +40,20 @@ for arg; do
done
# echo "$@" "daemonized=${daemonized}"}
+# export env variables from doris_cloud.conf
+# read from doris_cloud.conf
+while read -r line; do
+ envline="$(echo "${line}" |
+ sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
+ sed 's/^[[:blank:]]*//g' |
+ grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
+ true)"
+ envline="$(eval "echo ${envline}")"
+ if [[ "${envline}" == *"="* ]]; then
+ eval 'export "${envline}"'
+ fi
+done <"${DORIS_HOME}/conf/doris_cloud.conf"
+
process=doris_cloud
if [[ -f "${DORIS_HOME}/bin/${process}.pid" ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/03: [Improvement][Audit] add sql type for query audit (#37790)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit df07690109a68040ff378d977a4f7d6bd736d9ce
Author: shee <13...@users.noreply.github.com>
AuthorDate: Fri Jul 26 14:37:51 2024 +0800
[Improvement][Audit] add sql type for query audit (#37790)
Co-authored-by: garenshi <ga...@tencent.com>
---
.../doris/analysis/AlterCatalogCommentStmt.java | 5 ++
.../doris/analysis/AlterCatalogNameStmt.java | 5 ++
.../doris/analysis/AlterCatalogPropertyStmt.java | 5 ++
.../doris/analysis/AlterColocateGroupStmt.java | 5 ++
.../doris/analysis/AlterColumnStatsStmt.java | 5 ++
.../doris/analysis/AlterDatabasePropertyStmt.java | 5 ++
.../doris/analysis/AlterDatabaseQuotaStmt.java | 5 ++
.../apache/doris/analysis/AlterDatabaseRename.java | 5 ++
.../apache/doris/analysis/AlterJobStatusStmt.java | 5 ++
.../org/apache/doris/analysis/AlterPolicyStmt.java | 5 ++
.../apache/doris/analysis/AlterRepositoryStmt.java | 5 ++
.../apache/doris/analysis/AlterResourceStmt.java | 5 ++
.../org/apache/doris/analysis/AlterRoleStmt.java | 5 ++
.../doris/analysis/AlterRoutineLoadStmt.java | 6 +++
.../doris/analysis/AlterSqlBlockRuleStmt.java | 5 ++
.../org/apache/doris/analysis/AlterSystemStmt.java | 5 ++
.../org/apache/doris/analysis/AlterTableStmt.java | 5 ++
.../org/apache/doris/analysis/AlterUserStmt.java | 5 ++
.../org/apache/doris/analysis/AlterViewStmt.java | 5 ++
.../doris/analysis/AlterWorkloadGroupStmt.java | 5 ++
.../org/apache/doris/analysis/AnalyzeStmt.java | 5 ++
.../java/org/apache/doris/analysis/BackupStmt.java | 6 +++
.../doris/analysis/CancelAlterSystemStmt.java | 5 ++
.../doris/analysis/CancelAlterTableStmt.java | 5 ++
.../apache/doris/analysis/CancelBackupStmt.java | 5 ++
.../apache/doris/analysis/CancelExportStmt.java | 5 ++
.../apache/doris/analysis/CancelJobTaskStmt.java | 5 ++
.../org/apache/doris/analysis/CancelLoadStmt.java | 5 ++
.../org/apache/doris/analysis/CleanLabelStmt.java | 5 ++
.../apache/doris/analysis/CleanProfileStmt.java | 5 ++
.../apache/doris/analysis/CleanQueryStatsStmt.java | 5 ++
.../apache/doris/analysis/CreateCatalogStmt.java | 5 ++
.../doris/analysis/CreateDataSyncJobStmt.java | 5 ++
.../org/apache/doris/analysis/CreateDbStmt.java | 5 ++
.../doris/analysis/CreateEncryptKeyStmt.java | 5 ++
.../org/apache/doris/analysis/CreateFileStmt.java | 5 ++
.../apache/doris/analysis/CreateFunctionStmt.java | 5 ++
.../doris/analysis/CreateMaterializedViewStmt.java | 5 ++
.../apache/doris/analysis/CreatePolicyStmt.java | 5 ++
.../doris/analysis/CreateRepositoryStmt.java | 5 ++
.../apache/doris/analysis/CreateResourceStmt.java | 5 ++
.../org/apache/doris/analysis/CreateRoleStmt.java | 5 ++
.../doris/analysis/CreateRoutineLoadStmt.java | 5 ++
.../doris/analysis/CreateSqlBlockRuleStmt.java | 5 ++
.../doris/analysis/CreateTableAsSelectStmt.java | 5 ++
.../apache/doris/analysis/CreateTableLikeStmt.java | 5 ++
.../org/apache/doris/analysis/CreateTableStmt.java | 5 ++
.../org/apache/doris/analysis/CreateUserStmt.java | 5 ++
.../org/apache/doris/analysis/CreateViewStmt.java | 5 ++
.../doris/analysis/CreateWorkloadGroupStmt.java | 5 ++
.../analysis/CreateWorkloadSchedPolicyStmt.java | 5 ++
.../java/org/apache/doris/analysis/DdlStmt.java | 5 ++
.../java/org/apache/doris/analysis/DeleteStmt.java | 5 ++
.../apache/doris/analysis/DropAnalyzeJobStmt.java | 5 ++
.../org/apache/doris/analysis/DropCatalogStmt.java | 5 ++
.../java/org/apache/doris/analysis/DropDbStmt.java | 5 ++
.../apache/doris/analysis/DropEncryptKeyStmt.java | 5 ++
.../org/apache/doris/analysis/DropFileStmt.java | 5 ++
.../apache/doris/analysis/DropFunctionStmt.java | 5 ++
.../doris/analysis/DropMaterializedViewStmt.java | 5 ++
.../org/apache/doris/analysis/DropPolicyStmt.java | 5 ++
.../apache/doris/analysis/DropRepositoryStmt.java | 5 ++
.../apache/doris/analysis/DropResourceStmt.java | 5 ++
.../org/apache/doris/analysis/DropRoleStmt.java | 5 ++
.../doris/analysis/DropSqlBlockRuleStmt.java | 5 ++
.../org/apache/doris/analysis/DropStatsStmt.java | 5 ++
.../org/apache/doris/analysis/DropTableStmt.java | 5 ++
.../org/apache/doris/analysis/DropUserStmt.java | 5 ++
.../doris/analysis/DropWorkloadGroupStmt.java | 5 ++
.../analysis/DropWorkloadSchedPolicyStmt.java | 5 ++
.../org/apache/doris/analysis/ExecuteStmt.java | 5 ++
.../java/org/apache/doris/analysis/ExportStmt.java | 5 ++
.../java/org/apache/doris/analysis/GrantStmt.java | 5 ++
.../doris/analysis/InsertOverwriteTableStmt.java | 6 +++
.../java/org/apache/doris/analysis/InsertStmt.java | 5 ++
.../apache/doris/analysis/InstallPluginStmt.java | 5 ++
.../apache/doris/analysis/KillAnalysisJobStmt.java | 5 ++
.../java/org/apache/doris/analysis/KillStmt.java | 5 ++
.../java/org/apache/doris/analysis/LoadStmt.java | 4 ++
.../doris/analysis/PauseRoutineLoadStmt.java | 5 ++
.../apache/doris/analysis/PauseSyncJobStmt.java | 5 ++
.../org/apache/doris/analysis/RecoverDbStmt.java | 5 ++
.../doris/analysis/RecoverPartitionStmt.java | 5 ++
.../apache/doris/analysis/RecoverTableStmt.java | 5 ++
.../apache/doris/analysis/RefreshCatalogStmt.java | 4 ++
.../org/apache/doris/analysis/RefreshDbStmt.java | 5 ++
.../org/apache/doris/analysis/RefreshLdapStmt.java | 5 ++
.../apache/doris/analysis/RefreshTableStmt.java | 5 ++
.../org/apache/doris/analysis/RestoreStmt.java | 5 ++
.../doris/analysis/ResumeRoutineLoadStmt.java | 5 ++
.../apache/doris/analysis/ResumeSyncJobStmt.java | 5 ++
.../java/org/apache/doris/analysis/RevokeStmt.java | 5 ++
.../java/org/apache/doris/analysis/SelectStmt.java | 5 ++
.../apache/doris/analysis/SetOperationStmt.java | 7 +++
.../java/org/apache/doris/analysis/SetStmt.java | 5 ++
.../apache/doris/analysis/SetUserPropertyStmt.java | 5 ++
.../java/org/apache/doris/analysis/ShowStmt.java | 5 ++
.../org/apache/doris/analysis/StatementBase.java | 4 ++
.../analysis/{ShowStmt.java => StmtType.java} | 55 ++++++++++++++++------
.../apache/doris/analysis/StopRoutineLoadStmt.java | 5 ++
.../org/apache/doris/analysis/StopSyncJobStmt.java | 5 ++
.../java/org/apache/doris/analysis/SwitchStmt.java | 5 ++
.../java/org/apache/doris/analysis/SyncStmt.java | 5 ++
.../org/apache/doris/analysis/TransactionStmt.java | 5 ++
.../apache/doris/analysis/TruncateTableStmt.java | 5 ++
.../apache/doris/analysis/UninstallPluginStmt.java | 5 ++
.../apache/doris/analysis/UnsetVariableStmt.java | 5 ++
.../java/org/apache/doris/analysis/UpdateStmt.java | 5 ++
.../java/org/apache/doris/analysis/UseStmt.java | 5 ++
.../org/apache/doris/catalog/InternalSchema.java | 1 +
.../doris/nereids/analyzer/UnboundResultSink.java | 6 +++
.../trees/plans/commands/AlterMTMVCommand.java | 5 ++
.../trees/plans/commands/AlterViewCommand.java | 6 +++
.../nereids/trees/plans/commands/CallCommand.java | 5 ++
.../plans/commands/CancelMTMVTaskCommand.java | 6 +++
.../trees/plans/commands/CreateMTMVCommand.java | 6 +++
.../trees/plans/commands/CreatePolicyCommand.java | 6 +++
.../plans/commands/CreateProcedureCommand.java | 6 +++
.../trees/plans/commands/CreateTableCommand.java | 6 +++
.../plans/commands/CreateTableLikeCommand.java | 6 +++
.../trees/plans/commands/CreateViewCommand.java | 6 +++
.../trees/plans/commands/DeleteFromCommand.java | 6 +++
.../plans/commands/DeleteFromUsingCommand.java | 6 +++
.../trees/plans/commands/DropMTMVCommand.java | 6 +++
.../trees/plans/commands/DropProcedureCommand.java | 6 +++
.../trees/plans/commands/ExecuteCommand.java | 6 +++
.../trees/plans/commands/ExplainCommand.java | 6 +++
.../trees/plans/commands/ExportCommand.java | 6 +++
.../nereids/trees/plans/commands/LoadCommand.java | 6 +++
.../trees/plans/commands/PauseMTMVCommand.java | 6 +++
.../trees/plans/commands/PrepareCommand.java | 6 +++
.../trees/plans/commands/RefreshMTMVCommand.java | 6 +++
.../trees/plans/commands/ResumeMTMVCommand.java | 6 +++
.../plans/commands/ShowConstraintsCommand.java | 6 +++
.../plans/commands/ShowCreateProcedureCommand.java | 6 +++
.../plans/commands/ShowProcedureStatusCommand.java | 6 +++
.../trees/plans/commands/UpdateCommand.java | 6 +++
.../insert/BatchInsertIntoTableCommand.java | 6 +++
.../commands/insert/InsertIntoTableCommand.java | 6 +++
.../insert/InsertOverwriteTableCommand.java | 6 +++
.../nereids/trees/plans/logical/LogicalPlan.java | 5 ++
.../org/apache/doris/plugin/audit/AuditEvent.java | 7 +++
.../doris/plugin/audit/AuditLoaderPlugin.java | 1 +
.../java/org/apache/doris/qe/AuditLogHelper.java | 18 +++++++
.../doris/alter/InternalSchemaAlterTest.java | 14 ++++++
.../org/apache/doris/analysis/QueryStmtTest.java | 29 ++++++++++++
.../doris/nereids/parser/NereidsParserTest.java | 21 +++++++++
.../apache/doris/qe/AuditEventProcessorTest.java | 2 +
.../org/apache/doris/utframe/UtFrameUtils.java | 20 ++++++++
.../apache/doris/plugin/audit/AuditPluginDemo.java | 1 -
.../src/main/assembly/plugin.properties | 2 +-
.../audit/{ => custom}/AuditLoaderPlugin.java | 15 ++++--
.../audit/{ => custom}/DorisStreamLoader.java | 17 +++----
.../dialect/spark/SparkSql3LogicalPlanBuilder.java | 2 +-
.../spark/SparkSqlDialectConverterPlugin.java | 2 +-
.../dialect/trino/TrinoLogicalPlanBuilder.java | 2 +-
156 files changed, 902 insertions(+), 32 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogCommentStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogCommentStmt.java
index cfc52f077d7..8249b01b2bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogCommentStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogCommentStmt.java
@@ -46,4 +46,9 @@ public class AlterCatalogCommentStmt extends AlterCatalogStmt {
public String toSql() {
return "ALTER CATALOG " + catalogName + " MODIFY COMMENT " + comment;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogNameStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogNameStmt.java
index f3c363fd5eb..abfdae6b5a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogNameStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogNameStmt.java
@@ -56,4 +56,9 @@ public class AlterCatalogNameStmt extends AlterCatalogStmt {
public String toSql() {
return "ALTER CATALOG " + catalogName + " RENAME " + newCatalogName;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java
index 3cac7056276..1f204015ad7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterCatalogPropertyStmt.java
@@ -54,4 +54,9 @@ public class AlterCatalogPropertyStmt extends AlterCatalogStmt {
public boolean needAuditEncryption() {
return true;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColocateGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColocateGroupStmt.java
index 2885d662555..98b1ec76f98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColocateGroupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColocateGroupStmt.java
@@ -79,4 +79,9 @@ public class AlterColocateGroupStmt extends DdlStmt {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
index 9e76b065921..40c8600381e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterColumnStatsStmt.java
@@ -219,4 +219,9 @@ public class AlterColumnStatsStmt extends DdlStmt {
public String getValue(StatsType statsType) {
return statsTypeToValue.get(statsType);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
index c36651855a2..af01882dc3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabasePropertyStmt.java
@@ -76,4 +76,9 @@ public class AlterDatabasePropertyStmt extends DdlStmt {
return "ALTER DATABASE " + dbName + " SET PROPERTIES ("
+ new PrintableMap<String, String>(properties, "=", true, false, ",") + ")";
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java
index 647273add40..552b701d029 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java
@@ -94,4 +94,9 @@ public class AlterDatabaseQuotaStmt extends DdlStmt {
+ quotaType.name()
+ " QUOTA " + quotaValue;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseRename.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseRename.java
index cf32745e0b3..1e4d78840d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseRename.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseRename.java
@@ -73,4 +73,9 @@ public class AlterDatabaseRename extends DdlStmt {
return "ALTER DATABASE " + dbName + " RENAME " + newDbName;
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
index 31a35aca2c2..3dd2a9f27fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
@@ -74,4 +74,9 @@ public class AlterJobStatusStmt extends DdlStmt {
throw new AnalysisException("Can't alter inner job status");
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterPolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterPolicyStmt.java
index db9c10f34f2..622feed12ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterPolicyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterPolicyStmt.java
@@ -96,4 +96,9 @@ public class AlterPolicyStmt extends DdlStmt {
return sb.toString();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java
index b0ecd6d3bd4..31c33b6b8e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRepositoryStmt.java
@@ -83,4 +83,9 @@ public class AlterRepositoryStmt extends DdlStmt {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java
index 084200f28da..bd05f478412 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterResourceStmt.java
@@ -87,4 +87,9 @@ public class AlterResourceStmt extends DdlStmt {
public boolean needAuditEncryption() {
return true;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoleStmt.java
index 75e86a4b6f0..ec428cb82f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoleStmt.java
@@ -62,4 +62,9 @@ public class AlterRoleStmt extends DdlStmt {
sb.append(" COMMENT \"").append(comment).append("\"");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
index 5a1f1ba56aa..67c103ed02c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
@@ -264,4 +264,10 @@ public class AlterRoutineLoadStmt extends DdlStmt {
dataSourceProperties.setTimezone(job.getTimezone());
dataSourceProperties.analyze();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java
index e7af4c47a65..a3cb97dd6bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSqlBlockRuleStmt.java
@@ -138,4 +138,9 @@ public class AlterSqlBlockRuleStmt extends DdlStmt {
.append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSystemStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSystemStmt.java
index a3fc515e472..47cb4fc4306 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSystemStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterSystemStmt.java
@@ -69,4 +69,9 @@ public class AlterSystemStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
index d939bb0c3fb..f5d42435532 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java
@@ -210,4 +210,9 @@ public class AlterTableStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterUserStmt.java
index 544a106b3ba..f87884c6c4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterUserStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterUserStmt.java
@@ -159,4 +159,9 @@ public class AlterUserStmt extends DdlStmt {
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterViewStmt.java
index 3323c00bac9..5a2d184e20d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterViewStmt.java
@@ -111,4 +111,9 @@ public class AlterViewStmt extends BaseViewStmt implements NotFallbackInParser {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java
index f6120338333..3a5a3c0b63f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterWorkloadGroupStmt.java
@@ -66,4 +66,9 @@ public class AlterWorkloadGroupStmt extends DdlStmt {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
index e97de804376..9b5145a7d0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
@@ -114,4 +114,9 @@ public class AnalyzeStmt extends StatementBase {
public boolean usingSqlForExternalTable() {
return analyzeProperties.usingSqlForExternalTable();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ANALYZE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BackupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BackupStmt.java
index a1eef45ec26..84b3171c2c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BackupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BackupStmt.java
@@ -127,4 +127,10 @@ public class BackupStmt extends AbstractBackupStmt {
sb.append("\n)");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.BACKUP;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterSystemStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterSystemStmt.java
index e3a465d2d90..9cfde8da4b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterSystemStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterSystemStmt.java
@@ -90,4 +90,9 @@ public class CancelAlterSystemStmt extends CancelStmt {
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterTableStmt.java
index 8c05483b770..7dac7c28ff1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelAlterTableStmt.java
@@ -104,4 +104,9 @@ public class CancelAlterTableStmt extends CancelStmt {
return toSql();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelBackupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelBackupStmt.java
index ab1fb8c4b70..54e9ba8b8ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelBackupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelBackupStmt.java
@@ -82,4 +82,9 @@ public class CancelBackupStmt extends CancelStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
index ebdd7997d41..039443becd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelExportStmt.java
@@ -169,4 +169,9 @@ public class CancelExportStmt extends DdlStmt {
return toSql();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelJobTaskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelJobTaskStmt.java
index ea6febe1630..841e43a8257 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelJobTaskStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelJobTaskStmt.java
@@ -69,4 +69,9 @@ public class CancelJobTaskStmt extends DdlStmt {
}
this.taskId = ((IntLiteral) compoundPredicate.getChildren().get(1).getChild(1)).getLongValue();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
index f38b5f84db1..dad9a0ccd2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CancelLoadStmt.java
@@ -173,4 +173,9 @@ public class CancelLoadStmt extends DdlStmt {
return toSql();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanLabelStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanLabelStmt.java
index 0be3adb9e60..84de16c4e6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanLabelStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanLabelStmt.java
@@ -68,4 +68,9 @@ public class CleanLabelStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CLEAN;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanProfileStmt.java
index 95ff420d1e2..02a562a328b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanProfileStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanProfileStmt.java
@@ -47,4 +47,9 @@ public class CleanProfileStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CLEAN;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java
index f9dce80d4d4..c0964f796e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CleanQueryStatsStmt.java
@@ -141,4 +141,9 @@ public class CleanQueryStatsStmt extends DdlStmt {
public enum Scope {
ALL, DB, TABLE
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CLEAN;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java
index 720df2dbedc..59a59522acf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateCatalogStmt.java
@@ -132,4 +132,9 @@ public class CreateCatalogStmt extends DdlStmt {
public boolean needAuditEncryption() {
return true;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDataSyncJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDataSyncJobStmt.java
index adc8e55f75a..dd7103b463d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDataSyncJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDataSyncJobStmt.java
@@ -127,4 +127,9 @@ public class CreateDataSyncJobStmt extends DdlStmt {
public DataSyncJobType getDataSyncJobType() {
return dataSyncJobType;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDbStmt.java
index 09ae61a1ee1..ad17ccd23ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateDbStmt.java
@@ -93,4 +93,9 @@ public class CreateDbStmt extends DdlStmt {
}
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateEncryptKeyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateEncryptKeyStmt.java
index 701df2bd69a..a7380865a44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateEncryptKeyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateEncryptKeyStmt.java
@@ -91,4 +91,9 @@ public class CreateEncryptKeyStmt extends DdlStmt {
.append(encryptKeyName.getKeyName()).append(" AS \"").append(keyString).append("\"");
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFileStmt.java
index cfc25b838da..11d2bf846e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFileStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFileStmt.java
@@ -162,4 +162,9 @@ public class CreateFileStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
index abc2249a00b..a45b7901fec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java
@@ -902,4 +902,9 @@ public class CreateFunctionStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 449b848321b..3a6a9718c72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -723,4 +723,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
public String toSql() {
return null;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java
index 0c3e760e576..98c71d10c16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java
@@ -153,4 +153,9 @@ public class CreatePolicyStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
index 4bd82be73e3..618b5a7d05b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
@@ -88,4 +88,9 @@ public class CreateRepositoryStmt extends DdlStmt {
public boolean needAuditEncryption() {
return true;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
index 9b4ae717ae5..08730723431 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
@@ -121,4 +121,9 @@ public class CreateResourceStmt extends DdlStmt {
public boolean needAuditEncryption() {
return true;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoleStmt.java
index 9021402d48a..1f64ceddb85 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoleStmt.java
@@ -85,4 +85,9 @@ public class CreateRoleStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
index d58b25195c7..95434a1fd19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
@@ -549,4 +549,9 @@ public class CreateRoutineLoadStmt extends DdlStmt {
this.dataSourceProperties.setTimezone(this.timezone);
this.dataSourceProperties.analyze();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java
index a53f7c944cb..f50a339991c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateSqlBlockRuleStmt.java
@@ -166,4 +166,9 @@ public class CreateSqlBlockRuleStmt extends DdlStmt {
.append(new PrintableMap<>(properties, " = ", true, true, true)).append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java
index 1f919255f96..04f2a475706 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java
@@ -126,4 +126,9 @@ public class CreateTableAsSelectStmt extends DdlStmt implements NotFallbackInPar
super.reset();
queryStmt.reset();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableLikeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableLikeStmt.java
index d8807bfae37..aab0f362afe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableLikeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableLikeStmt.java
@@ -129,4 +129,9 @@ public class CreateTableLikeStmt extends DdlStmt implements NotFallbackInParser
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index b4f1ec61091..795dfe237d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -874,4 +874,9 @@ public class CreateTableStmt extends DdlStmt implements NotFallbackInParser {
}
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java
index dfdda334451..e29a82f2e5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java
@@ -203,4 +203,9 @@ public class CreateUserStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java
index 3596470dd91..08f830a7fde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java
@@ -104,4 +104,9 @@ public class CreateViewStmt extends BaseViewStmt implements NotFallbackInParser
public void setFinalColumns(List<Column> columns) {
finalCols.addAll(columns);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java
index 92a60a94e55..9c8efcf1d1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadGroupStmt.java
@@ -85,4 +85,9 @@ public class CreateWorkloadGroupStmt extends DdlStmt {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, " = ", true, false)).append(")");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadSchedPolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadSchedPolicyStmt.java
index 001068476d4..455ca2c802e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadSchedPolicyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateWorkloadSchedPolicyStmt.java
@@ -122,4 +122,9 @@ public class CreateWorkloadSchedPolicyStmt extends DdlStmt {
return str;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DdlStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DdlStmt.java
index 2fb066c6763..596341bb374 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DdlStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DdlStmt.java
@@ -22,4 +22,9 @@ public abstract class DdlStmt extends StatementBase {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DDL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
index 369eb126597..692e53ab229 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
@@ -463,4 +463,9 @@ public class DeleteStmt extends DdlStmt implements NotFallbackInParser {
sb.append(" WHERE ").append(wherePredicate.toSql());
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DELETE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropAnalyzeJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropAnalyzeJobStmt.java
index 34a7c875eda..04562476687 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropAnalyzeJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropAnalyzeJobStmt.java
@@ -31,4 +31,9 @@ public class DropAnalyzeJobStmt extends DdlStmt {
public long getJobId() {
return jobId;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java
index f3606c07f29..7043066fc47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java
@@ -72,4 +72,9 @@ public class DropCatalogStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
index a277f3a9ce5..7ce85a29902 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
@@ -96,4 +96,9 @@ public class DropDbStmt extends DdlStmt {
return toSql();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropEncryptKeyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropEncryptKeyStmt.java
index 4b41fa71a2f..542f67b029d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropEncryptKeyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropEncryptKeyStmt.java
@@ -67,4 +67,9 @@ public class DropEncryptKeyStmt extends DdlStmt {
stringBuilder.append("DROP ENCRYPTKEY ").append(encryptKeyName.getKeyName());
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFileStmt.java
index fb2af7d9dd3..048498050bc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFileStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFileStmt.java
@@ -109,4 +109,9 @@ public class DropFileStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFunctionStmt.java
index 9f3d297b6b6..de4b5576cd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFunctionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropFunctionStmt.java
@@ -94,4 +94,9 @@ public class DropFunctionStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropMaterializedViewStmt.java
index 377cdcf4152..28db2575277 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropMaterializedViewStmt.java
@@ -90,4 +90,9 @@ public class DropMaterializedViewStmt extends DdlStmt {
stringBuilder.append("ON ").append(tableName.toSql());
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropPolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropPolicyStmt.java
index 4bd20fe8c9b..85560a08f0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropPolicyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropPolicyStmt.java
@@ -105,4 +105,9 @@ public class DropPolicyStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRepositoryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRepositoryStmt.java
index ce2ea3e8ea2..036d2b606d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRepositoryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRepositoryStmt.java
@@ -53,4 +53,9 @@ public class DropRepositoryStmt extends DdlStmt {
sb.append("REPOSITORY `").append(repoName).append("`");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceStmt.java
index e57c433bb9b..10744acceec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropResourceStmt.java
@@ -62,4 +62,9 @@ public class DropResourceStmt extends DdlStmt {
sb.append("RESOURCE `").append(resourceName).append("`");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRoleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRoleStmt.java
index 468b86579f4..e27dc4705ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRoleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropRoleStmt.java
@@ -69,4 +69,9 @@ public class DropRoleStmt extends DdlStmt {
public String toSql() {
return "DROP ROLE " + role;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java
index 1a90eec2ac7..11dccab6577 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropSqlBlockRuleStmt.java
@@ -56,4 +56,9 @@ public class DropSqlBlockRuleStmt extends DdlStmt {
sb.append("DROP SQL_BLOCK_RULE ").append(Joiner.on(",").join(ruleNames));
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
index aa83a1de186..3689af25cda 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
@@ -193,4 +193,9 @@ public class DropStatsStmt extends DdlStmt {
dbName + "." + tblName);
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java
index 5df0a82c6f2..d9a266fdf11 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStmt.java
@@ -107,4 +107,9 @@ public class DropTableStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
index e336040d7f8..e9b3e6fe794 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java
@@ -84,4 +84,9 @@ public class DropUserStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java
index d7a1703771c..1137224c12e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadGroupStmt.java
@@ -61,4 +61,9 @@ public class DropWorkloadGroupStmt extends DdlStmt {
sb.append("RESOURCE GROUP '").append(workloadGroupName).append("' ");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadSchedPolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadSchedPolicyStmt.java
index 7eeff5f3214..fae534f51f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadSchedPolicyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropWorkloadSchedPolicyStmt.java
@@ -63,4 +63,9 @@ public class DropWorkloadSchedPolicyStmt extends DdlStmt {
return sb.toString();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExecuteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExecuteStmt.java
index df5707dda67..4805f8c97e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExecuteStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExecuteStmt.java
@@ -54,4 +54,9 @@ public class ExecuteStmt extends StatementBase {
sql += ")";
return sql;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.EXECUTE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
index 855379cbc37..232a61f1edc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExportStmt.java
@@ -447,4 +447,9 @@ public class ExportStmt extends StatementBase {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.EXPORT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
index de4414ad40e..f1b3ab01d43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
@@ -374,4 +374,9 @@ public class GrantStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.GRANT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java
index cfe030c428a..bef4a38bc3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertOverwriteTableStmt.java
@@ -105,4 +105,10 @@ public class InsertOverwriteTableStmt extends DdlStmt {
getDb() + ": " + getTbl());
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSERT;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
index 3b5c923bfea..dc7b5130e3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -333,4 +333,9 @@ public abstract class InsertStmt extends DdlStmt implements NotFallbackInParser
String toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSERT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InstallPluginStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InstallPluginStmt.java
index 20385f59d72..3d5db0c8840 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InstallPluginStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InstallPluginStmt.java
@@ -87,4 +87,9 @@ public class InstallPluginStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSTALL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/KillAnalysisJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/KillAnalysisJobStmt.java
index a950c663d36..996afb8c8a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/KillAnalysisJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/KillAnalysisJobStmt.java
@@ -24,4 +24,9 @@ public class KillAnalysisJobStmt extends DdlStmt {
public KillAnalysisJobStmt(long jobId) {
this.jobId = jobId;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.KILL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/KillStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/KillStmt.java
index 77cbafe4a8f..12cd9a4daf9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/KillStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/KillStmt.java
@@ -76,4 +76,9 @@ public class KillStmt extends StatementBase {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.NO_FORWARD;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.KILL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 1990078b46c..a24d1ae72fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -689,4 +689,8 @@ public class LoadStmt extends DdlStmt {
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.LOAD;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseRoutineLoadStmt.java
index adda77df65b..745b6ecf385 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseRoutineLoadStmt.java
@@ -63,4 +63,9 @@ public class PauseRoutineLoadStmt extends DdlStmt {
db = analyzer.getDefaultDb();
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.PAUSE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseSyncJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseSyncJobStmt.java
index a032caa2622..14d38a882c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseSyncJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PauseSyncJobStmt.java
@@ -53,4 +53,9 @@ public class PauseSyncJobStmt extends DdlStmt {
stringBuilder.append(jobName.toSql());
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.PAUSE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverDbStmt.java
index 0d60664e100..cfb4ecf4087 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverDbStmt.java
@@ -84,4 +84,9 @@ public class RecoverDbStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RECOVER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
index 9b2f4626808..05db7a81e83 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverPartitionStmt.java
@@ -97,4 +97,9 @@ public class RecoverPartitionStmt extends DdlStmt {
sb.append(getTableName());
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RECOVER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverTableStmt.java
index c4133c2ed9a..56dfbbfc2c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RecoverTableStmt.java
@@ -91,4 +91,9 @@ public class RecoverTableStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RECOVER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshCatalogStmt.java
index a2e5930d142..060807b7919 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshCatalogStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshCatalogStmt.java
@@ -84,4 +84,8 @@ public class RefreshCatalogStmt extends DdlStmt {
return stringBuilder.toString();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
index e8bdc168ddc..091d55eafe0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
@@ -116,4 +116,9 @@ public class RefreshDbStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshLdapStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshLdapStmt.java
index 48c23999f15..81296a4961c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshLdapStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshLdapStmt.java
@@ -69,4 +69,9 @@ public class RefreshLdapStmt extends DdlStmt {
return stringBuilder.toString();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
index 65bf35f3059..9755e91dadf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
@@ -82,4 +82,9 @@ public class RefreshTableStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
index 21e417631dd..317de11c1f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
@@ -247,4 +247,9 @@ public class RestoreStmt extends AbstractBackupStmt {
sb.append("\n)");
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RESTORE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeRoutineLoadStmt.java
index 0087dcb0452..e21a6ca53c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeRoutineLoadStmt.java
@@ -63,4 +63,9 @@ public class ResumeRoutineLoadStmt extends DdlStmt {
db = analyzer.getDefaultDb();
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RESUME;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeSyncJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeSyncJobStmt.java
index d4b6c65c80d..2f401a09112 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeSyncJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ResumeSyncJobStmt.java
@@ -53,4 +53,9 @@ public class ResumeSyncJobStmt extends DdlStmt {
stringBuilder.append(jobName.toSql());
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RESUME;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
index ad26c2670cb..9b06623f82c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
@@ -209,4 +209,9 @@ public class RevokeStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REVOKE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 66994e99a72..b7b778ae901 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -2900,4 +2900,9 @@ public class SelectStmt extends QueryStmt {
this.selectList = selectList;
this.originSelectList = selectList.clone();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SELECT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
index 1d3c98cc7a7..3c4f8a76fd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
@@ -981,5 +981,12 @@ public class SetOperationStmt extends QueryStmt {
public SetOperand clone() {
return new SetOperand(this);
}
+
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SELECT;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetStmt.java
index 3c6d938026f..bfa0839b592 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetStmt.java
@@ -87,6 +87,11 @@ public class SetStmt extends StatementBase {
return toSql();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SET;
+ }
+
@Override
public RedirectStatus getRedirectStatus() {
if (setVars != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java
index 240fb5bfa36..7342fe7a79c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetUserPropertyStmt.java
@@ -91,4 +91,9 @@ public class SetUserPropertyStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SET;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java
index 09daa778b7b..9be50ee9dbd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java
@@ -31,4 +31,9 @@ public abstract class ShowStmt extends StatementBase {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.NO_FORWARD;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SHOW;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
index c0ebab251f8..f2f91529e8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
@@ -143,6 +143,10 @@ public abstract class StatementBase implements ParseNode {
return "";
}
+ public StmtType stmtType() {
+ return StmtType.OTHER;
+ }
+
public abstract RedirectStatus getRedirectStatus();
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
similarity index 63%
copy from fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java
copy to fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
index 09daa778b7b..426233ff7b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
@@ -17,18 +17,45 @@
package org.apache.doris.analysis;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.qe.ShowResultSetMetaData;
-
-public abstract class ShowStmt extends StatementBase {
- public abstract ShowResultSetMetaData getMetaData();
-
- public SelectStmt toSelectStmt(Analyzer analyzer) throws AnalysisException {
- return null;
- }
-
- @Override
- public RedirectStatus getRedirectStatus() {
- return RedirectStatus.NO_FORWARD;
- }
+public enum StmtType {
+ ADMIN,
+ ALTER,
+ ANALYZE,
+ BACKUP,
+ CALL,
+ CANCEL,
+ CLEAN,
+ CREATE,
+ DDL,
+ DELETE,
+ DROP,
+ EXECUTE,
+ EXPLAIN,
+ EXPORT,
+ GRANT,
+ IMPORT,
+ INSERT,
+ INSTALL,
+ KILL,
+ LOAD,
+ OTHER,
+ PAUSE,
+ PREPARE,
+ RECOVER,
+ REFRESH,
+ RESTORE,
+ RESUME,
+ REVOKE,
+ SELECT,
+ SET,
+ SHOW,
+ STOP,
+ SWITCH,
+ SYNC,
+ TRANSACTION,
+ TRUNCATE,
+ UNINSTALL,
+ UNSET,
+ UPDATE,
+ USE
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StopRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StopRoutineLoadStmt.java
index 58ed8a441e7..df18979cc7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StopRoutineLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StopRoutineLoadStmt.java
@@ -47,4 +47,9 @@ public class StopRoutineLoadStmt extends DdlStmt {
super.analyze(analyzer);
labelName.analyze(analyzer);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.STOP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StopSyncJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StopSyncJobStmt.java
index 34ab27b4516..e58835a26a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StopSyncJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StopSyncJobStmt.java
@@ -53,4 +53,9 @@ public class StopSyncJobStmt extends DdlStmt {
stringBuilder.append(jobName.toSql());
return stringBuilder.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.STOP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SwitchStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SwitchStmt.java
index fe5ac574136..d8d3dac0d22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SwitchStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SwitchStmt.java
@@ -62,4 +62,9 @@ public class SwitchStmt extends StatementBase {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.NO_FORWARD;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SWITCH;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SyncStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SyncStmt.java
index 59b9d3cd8af..e559b30f420 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SyncStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SyncStmt.java
@@ -29,4 +29,9 @@ public class SyncStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SYNC;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java
index bad6d4670a8..60c8d040f72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TransactionStmt.java
@@ -31,4 +31,9 @@ public class TransactionStmt extends StatementBase {
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.TRANSACTION;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
index 1a9fbd3bafd..46a6045d51f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
@@ -85,4 +85,9 @@ public class TruncateTableStmt extends DdlStmt {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.TRUNCATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UninstallPluginStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UninstallPluginStmt.java
index 5b87a6797e9..c87456dcb37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UninstallPluginStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UninstallPluginStmt.java
@@ -67,4 +67,9 @@ public class UninstallPluginStmt extends DdlStmt {
public RedirectStatus getRedirectStatus() {
return RedirectStatus.FORWARD_WITH_SYNC;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.UNINSTALL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UnsetVariableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UnsetVariableStmt.java
index 2b05e233338..1756e831f69 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UnsetVariableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UnsetVariableStmt.java
@@ -106,4 +106,9 @@ public class UnsetVariableStmt extends StatementBase {
return RedirectStatus.NO_FORWARD;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.UNSET;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java
index b6db99ee43b..786ad16237e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java
@@ -251,4 +251,9 @@ public class UpdateStmt extends DdlStmt implements NotFallbackInParser {
}
return sb.toString();
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.UPDATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UseStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UseStmt.java
index 6bf7ce0bc03..dd9cc582512 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UseStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UseStmt.java
@@ -65,6 +65,11 @@ public class UseStmt extends StatementBase {
return sb.toString();
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.USE;
+ }
+
@Override
public String toString() {
return toSql();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java
index a2618db2ff4..15bf65c3c73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java
@@ -140,6 +140,7 @@ public class InternalSchema {
AUDIT_SCHEMA
.add(new ColumnDef("return_rows", TypeDef.create(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE));
AUDIT_SCHEMA.add(new ColumnDef("stmt_id", TypeDef.create(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE));
+ AUDIT_SCHEMA.add(new ColumnDef("stmt_type", TypeDef.createVarchar(48), ColumnNullableType.NULLABLE));
AUDIT_SCHEMA.add(new ColumnDef("is_query", TypeDef.create(PrimitiveType.TINYINT), ColumnNullableType.NULLABLE));
AUDIT_SCHEMA.add(new ColumnDef("frontend_ip", TypeDef.createVarchar(128), ColumnNullableType.NULLABLE));
AUDIT_SCHEMA
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java
index 10a6b0121e3..5fd5c18a365 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.analyzer;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
@@ -94,4 +95,9 @@ public class UnboundResultSink<CHILD_TYPE extends Plan> extends LogicalSink<CHIL
public String toString() {
return Utils.toSqlString("UnboundResultSink[" + id.asInt() + "]");
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SELECT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java
index ab9d6e35c9d..56f1ffec182 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVInfo;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -55,4 +56,8 @@ public class AlterMTMVCommand extends Command implements ForwardWithSync, NotAll
return visitor.visitAlterMTMVCommand(this, context);
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterViewCommand.java
index 3a2b551d9bf..900986874b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterViewCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterViewCommand.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.AlterViewStmt;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.AlterViewInfo;
@@ -47,4 +48,9 @@ public class AlterViewCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitAlterViewCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.ALTER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CallCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CallCommand.java
index 29e0b17228f..81d950eab6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CallCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CallCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.call.CallFunc;
@@ -58,4 +59,8 @@ public class CallCommand extends Command implements ForwardWithSync {
return visitor.visitCallCommand(this, context);
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CALL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
index e5c075da7e4..c8b2a19067d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
@@ -47,4 +48,9 @@ public class CancelMTMVTaskCommand extends Command implements ForwardWithSync, N
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitCancelMTMVTaskCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CANCEL;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java
index 814804b8ce9..0bc3c335fd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
@@ -56,4 +57,9 @@ public class CreateMTMVCommand extends Command implements ForwardWithSync, NotAl
return visitor.visitCreateMTMVCommand(this, context);
}
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java
index 01ccb3fa33f..ee4a4bfea45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -82,4 +83,9 @@ public class CreatePolicyCommand extends Command implements ForwardWithSync {
ctx.getSessionVariable().enableFallbackToOriginalPlannerOnce();
throw new AnalysisException("Not support create policy command in Nereids now");
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateProcedureCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateProcedureCommand.java
index 39a93831848..7fb8c856b46 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateProcedureCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateProcedureCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.FuncNameInfo;
@@ -61,4 +62,9 @@ public class CreateProcedureCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitCreateProcedureCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
index 38c92631d41..72e72cbe092 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropTableStmt;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
@@ -211,4 +212,9 @@ public class CreateTableCommand extends Command implements ForwardWithSync {
public CreateTableInfo getCreateTableInfo() {
return createTableInfo;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableLikeCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableLikeCommand.java
index a271623ef08..3b59c312880 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableLikeCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableLikeCommand.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.CreateTableLikeStmt;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableLikeInfo;
@@ -46,4 +47,9 @@ public class CreateTableLikeCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitCreateTableLikeCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java
index a6698bc1f69..2f597a1a85e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.CreateViewStmt;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.CreateViewInfo;
@@ -47,4 +48,9 @@ public class CreateViewCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitCreateViewCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.CREATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index fe5552fed65..0778765f0bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.Predicate;
import org.apache.doris.analysis.SetVar;
import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
@@ -434,4 +435,9 @@ public class DeleteFromCommand extends Command implements ForwardWithSync, Expla
throw new AnalysisException("delete command on aggregate/duplicate table is not explainable");
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DELETE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
index d212e3d6c28..de374d939c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -82,4 +83,9 @@ public class DeleteFromUsingCommand extends DeleteFromCommand {
throw new AnalysisException("delete command on with using clause only supports unique key model");
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DELETE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java
index 19eedb82746..909b87ef9aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.DropMTMVInfo;
@@ -47,4 +48,9 @@ public class DropMTMVCommand extends Command implements ForwardWithSync, NotAllo
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitDropMTMVCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropProcedureCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropProcedureCommand.java
index 6aa76197d82..6d4f32e999a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropProcedureCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropProcedureCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.FuncNameInfo;
@@ -59,4 +60,9 @@ public class DropProcedureCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitDropProcedureCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.DROP;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
index d5260a72cde..7cecc6343c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.Queriable;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@@ -102,4 +103,9 @@ public class ExecuteCommand extends Command {
return "EXECUTE `" + stmtName + "`"
+ realValueExpr.stream().map(Expression::toSql).collect(Collectors.joining(", ", " USING ", ""));
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.EXECUTE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index fccd9c07b14..d4fce906693 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.plans.commands;
import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@@ -112,4 +113,9 @@ public class ExplainCommand extends Command implements NoForward {
public boolean showPlanProcess() {
return showPlanProcess;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.EXPLAIN;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
index 263bf43e355..dbf6cf7067e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.BrokerDesc;
import org.apache.doris.analysis.LoadStmt;
import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.Separator;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.BrokerMgr;
@@ -400,4 +401,9 @@ public class ExportCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitExportCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.EXPORT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
index 53ee7263b00..59eaa1728b9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
@@ -500,4 +501,9 @@ public class LoadCommand extends Command implements ForwardWithSync {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitLoadCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.LOAD;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java
index f2a8cb23360..fadb3bc0c32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
@@ -47,4 +48,9 @@ public class PauseMTMVCommand extends Command implements ForwardWithSync, NotAll
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitPauseMTMVCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.PAUSE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java
index 43778a1e005..f8788f0657f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.nereids.trees.expressions.Placeholder;
import org.apache.doris.nereids.trees.plans.PlanType;
@@ -122,4 +123,9 @@ public class PrepareCommand extends Command {
public PrepareCommand withPlaceholders(List<Placeholder> placeholders) {
return new PrepareCommand(this.name, this.logicalPlan, placeholders, this.originalStmt);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.PREPARE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java
index a918112555e..2ed3e446d7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
@@ -47,4 +48,9 @@ public class RefreshMTMVCommand extends Command implements ForwardWithSync, NotA
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitRefreshMTMVCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java
index a47c65eeb41..5fc933886c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo;
@@ -47,4 +48,9 @@ public class ResumeMTMVCommand extends Command implements ForwardWithSync, NotAl
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitResumeMTMVCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.RESUME;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConstraintsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConstraintsCommand.java
index 13d1f0c5f6d..5c3c16137bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConstraintsCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConstraintsCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -63,4 +64,9 @@ public class ShowConstraintsCommand extends Command implements NoForward {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitShowConstraintsCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SHOW;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
index 1eac3cdc867..1000100c459 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.nereids.trees.plans.PlanType;
@@ -73,4 +74,9 @@ public class ShowCreateProcedureCommand extends Command implements NoForward {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitShowCreateProcedureCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SHOW;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
index b940f104d99..c77c77c875e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -135,4 +136,9 @@ public class ShowProcedureStatusCommand extends Command implements NoForward {
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitShowProcedureStatusCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.SHOW;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
index 644c5730a37..444bc8aa309 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
@@ -257,4 +258,9 @@ public class UpdateCommand extends Command implements ForwardWithSync, Explainab
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitUpdateCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.UPDATE;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java
index cdb5a4c7eeb..c0ab37e5d08 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands.insert;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
@@ -159,4 +160,9 @@ public class BatchInsertIntoTableCommand extends Command implements NoForward, E
targetTableIf.readUnlock();
}
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSERT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
index efe63884faa..1f1409e41c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands.insert;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
@@ -258,4 +259,9 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync,
}
return false;
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSERT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
index 41568097dad..ea9b1faf74f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands.insert;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
@@ -309,4 +310,9 @@ public class InsertOverwriteTableCommand extends Command implements ForwardWithS
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
return visitor.visitInsertOverwriteTableCommand(this, context);
}
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.INSERT;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
index f23d3f8d7a4..b419c5718b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPlan.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.nereids.properties.DataTrait;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -102,4 +103,8 @@ public interface LogicalPlan extends Plan {
void computeEqualSet(DataTrait.Builder builder);
void computeFd(DataTrait.Builder builder);
+
+ default StmtType stmtType() {
+ return StmtType.OTHER;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditEvent.java
index ac68b38e258..0b64a748a10 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditEvent.java
@@ -84,6 +84,8 @@ public class AuditEvent {
public boolean isNereids = false;
@AuditField(value = "feIp")
public String feIp = "";
+ @AuditField(value = "StmtType")
+ public String stmtType = "";
@AuditField(value = "Stmt")
public String stmt = "";
@AuditField(value = "CpuTimeMS")
@@ -230,6 +232,11 @@ public class AuditEvent {
return this;
}
+ public AuditEventBuilder setStmtType(String stmtType) {
+ auditEvent.stmtType = stmtType;
+ return this;
+ }
+
public AuditEventBuilder setStmt(String stmt) {
auditEvent.stmt = stmt;
return this;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
index fb4b6153431..4503b5b8802 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
@@ -156,6 +156,7 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
logBuffer.append(event.scanRows).append("\t");
logBuffer.append(event.returnRows).append("\t");
logBuffer.append(event.stmtId).append("\t");
+ logBuffer.append(event.stmtType).append("\t");
logBuffer.append(event.isQuery ? 1 : 0).append("\t");
logBuffer.append(event.feIp).append("\t");
logBuffer.append(event.cpuTimeMs).append("\t");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
index bcca751f91c..7d14586bbea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.analysis.ValueList;
import org.apache.doris.catalog.Env;
import org.apache.doris.cluster.ClusterNamespace;
@@ -242,6 +243,9 @@ public class AuditLogHelper {
} else {
auditEventBuilder.setStmt(origStmt);
}
+
+ auditEventBuilder.setStmtType(getStmtType(parsedStmt));
+
if (!Env.getCurrentEnv().isMaster()) {
if (ctx.executor.isForwardToMaster()) {
auditEventBuilder.setState(ctx.executor.getProxyStatus());
@@ -254,4 +258,18 @@ public class AuditLogHelper {
}
Env.getCurrentEnv().getWorkloadRuntimeStatusMgr().submitFinishQueryToAudit(auditEventBuilder.build());
}
+
+ private static String getStmtType(StatementBase stmt) {
+ if (stmt == null) {
+ return StmtType.OTHER.name();
+ }
+ if (stmt.isExplain()) {
+ return StmtType.EXPLAIN.name();
+ }
+ if (stmt instanceof LogicalPlanAdapter) {
+ return ((LogicalPlanAdapter) stmt).getLogicalPlan().stmtType().name();
+ } else {
+ return stmt.stmtType().name();
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
index 5e7f5387f57..cf9d31b1cca 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
@@ -17,8 +17,10 @@
package org.apache.doris.alter;
+import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.InternalSchema;
import org.apache.doris.catalog.InternalSchemaInitializer;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
@@ -69,4 +71,16 @@ public class InternalSchemaAlterTest extends TestWithFeService {
partitionInfo.getReplicaAllocation(partition.getId()).getTotalReplicaNum());
}
}
+
+ @Test
+ public void testCheckAuditLogTable() throws AnalysisException {
+ Database db = Env.getCurrentEnv().getCatalogMgr()
+ .getInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME);
+ Assertions.assertNotNull(db);
+ OlapTable table = db.getOlapTableOrAnalysisException(AuditLoaderPlugin.AUDIT_LOG_TABLE);
+ Assertions.assertNotNull(table);
+ for (ColumnDef def : InternalSchema.AUDIT_SCHEMA) {
+ Assertions.assertNotNull(table.getColumn(def.getName()));
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
index b42a29b31bd..495063fe48d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
@@ -33,6 +33,7 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
import java.util.HashMap;
import java.util.List;
@@ -370,4 +371,32 @@ public class QueryStmtTest {
}
return resultMap;
}
+
+ @Test
+ public void testParseStmtType() throws Exception {
+ ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+ String sql = "select a from c.b";
+ StatementBase stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assert.assertEquals(stmt.stmtType(), StmtType.SELECT);
+
+ sql = "drop table a";
+ stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assertions.assertEquals(stmt.stmtType(), StmtType.DROP);
+
+ sql = "use a";
+ stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assertions.assertEquals(stmt.stmtType(), StmtType.USE);
+
+ sql = "CREATE TABLE tbl (`id` INT NOT NULL) DISTRIBUTED BY HASH(`id`) BUCKETS 1";
+ stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assertions.assertEquals(stmt.stmtType(), StmtType.CREATE);
+
+ sql = "update a set b =1";
+ stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assertions.assertEquals(stmt.stmtType(), StmtType.UPDATE);
+
+ sql = "insert into a values(1)";
+ stmt = UtFrameUtils.onlyParse(sql, ctx);
+ Assertions.assertEquals(stmt.stmtType(), StmtType.INSERT);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 5773a9f29b5..d39e5cf22a6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.analysis.StmtType;
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.StatementContext;
@@ -420,4 +421,24 @@ public class NereidsParserTest extends ParserTestBase {
NereidsParser nereidsParser = new NereidsParser();
nereidsParser.parseSingle(sql);
}
+
+ @Test
+ public void testParseStmtType() {
+ NereidsParser nereidsParser = new NereidsParser();
+ String sql = "select a from b";
+ LogicalPlan plan = nereidsParser.parseSingle(sql);
+ Assertions.assertEquals(plan.stmtType(), StmtType.SELECT);
+
+ sql = "use a";
+ plan = nereidsParser.parseSingle(sql);
+ Assertions.assertEquals(plan.stmtType(), StmtType.OTHER);
+
+ sql = "CREATE TABLE tbl (`id` INT NOT NULL) DISTRIBUTED BY HASH(`id`) BUCKETS 1";
+ plan = nereidsParser.parseSingle(sql);
+ Assertions.assertEquals(plan.stmtType(), StmtType.CREATE);
+
+ sql = "update a set b =1";
+ plan = nereidsParser.parseSingle(sql);
+ Assertions.assertEquals(plan.stmtType(), StmtType.UPDATE);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java
index 26c51fb66db..6c9f54080a0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java
@@ -62,10 +62,12 @@ public class AuditEventProcessorTest {
.setScanRows(200000)
.setReturnRows(1)
.setStmtId(1234)
+ .setStmtType("SELECT")
.setStmt("select * from tbl1").build();
Assert.assertEquals("127.0.0.1", event.clientIp);
Assert.assertEquals(200000, event.scanRows);
+ Assert.assertEquals("SELECT", event.stmtType);
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index 2f1848f4aa3..d09860351bf 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -145,6 +145,26 @@ public class UtFrameUtils {
return statementBases;
}
+ public static StatementBase onlyParse(String originStmt, ConnectContext ctx) throws Exception {
+ System.out.println("begin to parse stmt: " + originStmt);
+ SqlScanner input = new SqlScanner(new StringReader(originStmt), ctx.getSessionVariable().getSqlMode());
+ SqlParser parser = new SqlParser(input);
+ StatementBase statementBase = null;
+ try {
+ statementBase = SqlParserUtils.getFirstStmt(parser);
+ } catch (AnalysisException e) {
+ String errorMessage = parser.getErrorMsg(originStmt);
+ System.err.println("parse failed: " + errorMessage);
+ if (errorMessage == null) {
+ throw e;
+ } else {
+ throw new AnalysisException(errorMessage, e);
+ }
+ }
+ statementBase.setOrigStmt(new OriginStatement(originStmt, 0));
+ return statementBase;
+ }
+
public static String generateRandomFeRunningDir(Class testSuiteClass) {
return generateRandomFeRunningDir(testSuiteClass.getSimpleName());
}
diff --git a/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java b/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java
index 5cbb4f028bb..26e13c8c043 100755
--- a/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java
+++ b/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java
@@ -17,7 +17,6 @@
package org.apache.doris.plugin.audit;
-import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginContext;
diff --git a/fe_plugins/auditloader/src/main/assembly/plugin.properties b/fe_plugins/auditloader/src/main/assembly/plugin.properties
index 65b25f87067..eb7ba9c29b0 100755
--- a/fe_plugins/auditloader/src/main/assembly/plugin.properties
+++ b/fe_plugins/auditloader/src/main/assembly/plugin.properties
@@ -20,4 +20,4 @@ type=AUDIT
description=load audit log to olap load, and user can view the statistic of queries
version=1.0.0
java.version=1.8.0
-classname=org.apache.doris.plugin.audit.AuditLoaderPlugin
+classname=org.apache.doris.plugin.audit.custom.AuditLoaderPlugin
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/AuditLoaderPlugin.java
similarity index 97%
rename from fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
rename to fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/AuditLoaderPlugin.java
index ed4b7efc651..4d873b57fe8 100755
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/AuditLoaderPlugin.java
@@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.plugin.audit;
+package org.apache.doris.plugin.audit.custom;
import org.apache.doris.common.Config;
import org.apache.doris.catalog.Env;
-import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginContext;
@@ -27,6 +26,7 @@ import org.apache.doris.plugin.PluginException;
import org.apache.doris.plugin.PluginInfo;
import com.google.common.collect.Queues;
+import org.apache.doris.plugin.audit.AuditEvent;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -163,6 +163,7 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
logBuffer.append(longToTimeString(event.timestamp)).append("\t");
logBuffer.append(event.clientIp).append("\t");
logBuffer.append(event.user).append("\t");
+ logBuffer.append(event.ctl).append("\t");
logBuffer.append(event.db).append("\t");
logBuffer.append(event.state).append("\t");
logBuffer.append(event.errorCode).append("\t");
@@ -172,18 +173,22 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
logBuffer.append(event.scanRows).append("\t");
logBuffer.append(event.returnRows).append("\t");
logBuffer.append(event.stmtId).append("\t");
+ logBuffer.append(event.stmtType).append("\t");
logBuffer.append(event.isQuery ? 1 : 0).append("\t");
logBuffer.append(event.feIp).append("\t");
logBuffer.append(event.cpuTimeMs).append("\t");
logBuffer.append(event.sqlHash).append("\t");
logBuffer.append(event.sqlDigest).append("\t");
logBuffer.append(event.peakMemoryBytes).append("\t");
+ logBuffer.append(event.workloadGroup).append("\t");
// trim the query to avoid too long
// use `getBytes().length` to get real byte length
String stmt = truncateByBytes(event.stmt).replace("\n", " ")
- .replace("\t", " ")
- .replace("\r", " ");
- LOG.debug("receive audit event with stmt: {}", stmt);
+ .replace("\t", " ")
+ .replace("\r", " ");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("receive audit event with stmt: {}", stmt);
+ }
logBuffer.append(stmt).append("\n");
}
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/DorisStreamLoader.java
similarity index 92%
rename from fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
rename to fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/DorisStreamLoader.java
index d389f0dfa81..b5ea0277986 100644
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/custom/DorisStreamLoader.java
@@ -15,7 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.plugin.audit;
+package org.apache.doris.plugin.audit.custom;
+
+import org.apache.doris.catalog.InternalSchema;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -29,6 +31,7 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Calendar;
+import java.util.stream.Collectors;
public class DorisStreamLoader {
private final static Logger LOG = LogManager.getLogger(DorisStreamLoader.class);
@@ -71,10 +74,8 @@ public class DorisStreamLoader {
conn.addRequestProperty("label", label);
conn.addRequestProperty("max_filter_ratio", "1.0");
- conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, error_code, error_message, " +
- "query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, " +
- "sql_digest, peak_memory_bytes, stmt");
-
+ conn.addRequestProperty("columns",
+ InternalSchema.AUDIT_SCHEMA.stream().map(c -> c.getName()).collect(Collectors.joining(",")));
conn.setDoOutput(true);
conn.setDoInput(true);
@@ -88,9 +89,9 @@ public class DorisStreamLoader {
sb.append("-H \"").append("Expect\":").append("\"100-continue\" \\\n ");
sb.append("-H \"").append("Content-Type\":").append("\"text/plain; charset=UTF-8\" \\\n ");
sb.append("-H \"").append("max_filter_ratio\":").append("\"1.0\" \\\n ");
- sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, error_code, " +
- "error_message, query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, " +
- "cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt\" \\\n ");
+ sb.append("-H \"").append("columns\":")
+ .append("\"" + InternalSchema.AUDIT_SCHEMA.stream().map(c -> c.getName()).collect(
+ Collectors.joining(",")) + "\" \\\n ");
sb.append("\"").append(conn.getURL()).append("\"");
return sb.toString();
}
diff --git a/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3LogicalPlanBuilder.java b/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3LogicalPlanBuilder.java
index e64fc736dcb..d2357fc6578 100644
--- a/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3LogicalPlanBuilder.java
+++ b/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3LogicalPlanBuilder.java
@@ -42,7 +42,7 @@ public class SparkSql3LogicalPlanBuilder extends LogicalPlanBuilder {
private final ParserContext parserContext;
public SparkSql3LogicalPlanBuilder() {
- this.parserContext = new ParserContext(Dialect.SPARK_SQL);
+ this.parserContext = new ParserContext(Dialect.SPARK);
}
@Override
diff --git a/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSqlDialectConverterPlugin.java b/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSqlDialectConverterPlugin.java
index 686c692faf8..a565b64aef0 100644
--- a/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSqlDialectConverterPlugin.java
+++ b/fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSqlDialectConverterPlugin.java
@@ -37,7 +37,7 @@ public class SparkSqlDialectConverterPlugin extends Plugin implements DialectCon
@Override
public ImmutableSet<Dialect> acceptDialects() {
- return ImmutableSet.of(Dialect.SPARK_SQL);
+ return ImmutableSet.of(Dialect.SPARK);
}
@Override
diff --git a/fe_plugins/trino-converter/src/main/java/org/apache/doris/plugin/dialect/trino/TrinoLogicalPlanBuilder.java b/fe_plugins/trino-converter/src/main/java/org/apache/doris/plugin/dialect/trino/TrinoLogicalPlanBuilder.java
index 75da2236ba5..2e6ace105b1 100644
--- a/fe_plugins/trino-converter/src/main/java/org/apache/doris/plugin/dialect/trino/TrinoLogicalPlanBuilder.java
+++ b/fe_plugins/trino-converter/src/main/java/org/apache/doris/plugin/dialect/trino/TrinoLogicalPlanBuilder.java
@@ -320,7 +320,7 @@ public class TrinoLogicalPlanBuilder extends io.trino.sql.tree.AstVisitor<Object
throw new DialectTransformException("transform not standard decimal data type ");
}
// Trino only support signed, safe unsigned is false here
- return DataType.convertPrimitiveFromStrings(types, false);
+ return DataType.convertPrimitiveFromStrings(types);
} else if (dataType instanceof io.trino.sql.tree.DateTimeDataType) {
// TODO: support date data type mapping
throw new DialectTransformException("transform date data type");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/03: [Feature][external catalog/lakesoul] LakeSoul Catalog support Filter Pushdown & Cdc data handling & S3 data access (#37979)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a9e4ffdbd96eb5eadb775b30a2fe7252549bfa27
Author: Ceng <44...@qq.com>
AuthorDate: Tue Jul 30 10:21:28 2024 +0800
[Feature][external catalog/lakesoul] LakeSoul Catalog support Filter Pushdown & Cdc data handling & S3 data access (#37979)
## Proposed changes
Issue Number: close #37978
---------
Signed-off-by: zenghua <44...@qq.com>
---
fe/be-java-extensions/lakesoul-scanner/pom.xml | 22 +-
.../apache/doris/lakesoul/LakeSoulJniScanner.java | 82 +++-
.../org/apache/doris/lakesoul/LakeSoulUtils.java | 30 +-
.../apache/doris/lakesoul/arrow/ArrowUtils.java | 8 +-
.../lakesoul/arrow/LakeSoulArrowJniScanner.java | 44 +-
fe/fe-core/pom.xml | 44 +-
.../lakesoul/LakeSoulExternalCatalog.java | 22 +-
.../datasource/lakesoul/LakeSoulExternalTable.java | 22 +-
.../doris/datasource/lakesoul/LakeSoulUtils.java | 535 +++++++++++++++++++++
.../lakesoul/source/LakeSoulScanNode.java | 158 +++++-
.../doris/nereids/rules/analysis/BindRelation.java | 1 +
.../datasource/lakesoul/LakeSoulPredicateTest.java | 280 +++++++++++
fe/pom.xml | 4 +-
regression-test/conf/regression-conf.groovy | 11 +
.../lakesoul/test_lakesoul_filter.out | 8 +
.../pipeline/external/conf/regression-conf.groovy | 4 +
.../lakesoul/test_lakesoul_catalog.groovy | 23 +-
.../lakesoul/test_lakesoul_filter.groovy | 58 +++
.../lakesoul/test_external_table_lakesoul.groovy | 24 +-
19 files changed, 1269 insertions(+), 111 deletions(-)
diff --git a/fe/be-java-extensions/lakesoul-scanner/pom.xml b/fe/be-java-extensions/lakesoul-scanner/pom.xml
index cbbb473483e..24d7efc7614 100644
--- a/fe/be-java-extensions/lakesoul-scanner/pom.xml
+++ b/fe/be-java-extensions/lakesoul-scanner/pom.xml
@@ -47,21 +47,6 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-vector</artifactId>
- <version>${arrow.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-memory-unsafe</artifactId>
- <version>${arrow.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.arrow</groupId>
- <artifactId>arrow-c-data</artifactId>
- <version>${arrow.version}</version>
- </dependency>
<!-- scala deps -->
<dependency>
@@ -85,7 +70,8 @@ under the License.
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-io-java</artifactId>
- <version>2.5.4</version>
+ <version>${lakesoul.version}</version>
+ <classifier>shaded</classifier>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -99,10 +85,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.arrow</groupId>
- <artifactId>*</artifactId>
- </exclusion>
<exclusion>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
diff --git a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulJniScanner.java b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulJniScanner.java
index 3dfbff756db..a7ac785d1fb 100644
--- a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulJniScanner.java
+++ b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulJniScanner.java
@@ -17,25 +17,30 @@
package org.apache.doris.lakesoul;
-import org.apache.doris.common.jni.vec.ScanPredicate;
import org.apache.doris.lakesoul.arrow.LakeSoulArrowJniScanner;
-import org.apache.doris.lakesoul.parquet.ParquetFilter;
import com.dmetasoul.lakesoul.LakeSoulArrowReader;
import com.dmetasoul.lakesoul.lakesoul.io.NativeIOReader;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.Schema;
+import com.dmetasoul.lakesoul.lakesoul.io.substrait.SubstraitUtil;
+import com.lakesoul.shaded.com.fasterxml.jackson.core.type.TypeReference;
+import com.lakesoul.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+import com.lakesoul.shaded.org.apache.arrow.vector.VectorSchemaRoot;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Schema;
+import io.substrait.proto.Plan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class LakeSoulJniScanner extends LakeSoulArrowJniScanner {
+ private static final Logger LOG = LoggerFactory.getLogger(LakeSoulJniScanner.class);
private final Map<String, String> params;
@@ -60,6 +65,9 @@ public class LakeSoulJniScanner extends LakeSoulArrowJniScanner {
withAllocator(nativeIOReader.getAllocator());
nativeIOReader.setBatchSize(batchSize);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("opening LakeSoulJniScanner with params={}", params);
+ }
// add files
for (String file : params.get(LakeSoulUtils.FILE_NAMES).split(LakeSoulUtils.LIST_DELIM)) {
nativeIOReader.addFile(file);
@@ -72,19 +80,39 @@ public class LakeSoulJniScanner extends LakeSoulArrowJniScanner {
Arrays.stream(primaryKeys.split(LakeSoulUtils.LIST_DELIM)).collect(Collectors.toList()));
}
- Schema schema = Schema.fromJSON(params.get(LakeSoulUtils.SCHEMA_JSON));
+ String options = params.getOrDefault(LakeSoulUtils.OPTIONS, "{}");
+ Map<String, String> optionsMap = new ObjectMapper().readValue(
+ options, new TypeReference<Map<String, String>>() {}
+ );
+ String base64Predicate = optionsMap.get(LakeSoulUtils.SUBSTRAIT_PREDICATE);
+ if (base64Predicate != null) {
+ Plan predicate = SubstraitUtil.decodeBase64String(base64Predicate);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("push predicate={}", predicate);
+ }
+ nativeIOReader.addFilterProto(predicate);
+ }
+
+ for (String key : LakeSoulUtils.OBJECT_STORE_OPTIONS) {
+ String value = optionsMap.get(key);
+ if (key != null) {
+ nativeIOReader.setObjectStoreOption(key, value);
+ }
+ }
+
+ Schema tableSchema = Schema.fromJSON(params.get(LakeSoulUtils.SCHEMA_JSON));
String[] requiredFieldNames = params.get(LakeSoulUtils.REQUIRED_FIELDS).split(LakeSoulUtils.LIST_DELIM);
List<Field> requiredFields = new ArrayList<>();
for (String fieldName : requiredFieldNames) {
- requiredFields.add(schema.findField(fieldName));
+ requiredFields.add(tableSchema.findField(fieldName));
}
requiredSchema = new Schema(requiredFields);
nativeIOReader.setSchema(requiredSchema);
- HashSet<String> partitionColumn = new HashSet<>();
+ List<Field> partitionFields = new ArrayList<>();
for (String partitionKV : params.getOrDefault(LakeSoulUtils.PARTITION_DESC, "")
.split(LakeSoulUtils.LIST_DELIM)) {
if (partitionKV.isEmpty()) {
@@ -94,17 +122,15 @@ public class LakeSoulJniScanner extends LakeSoulArrowJniScanner {
if (kv.length != 2) {
throw new IllegalArgumentException("Invalid partition column = " + partitionKV);
}
- partitionColumn.add(kv[0]);
+ nativeIOReader.setDefaultColumnValue(kv[0], kv[1]);
+ partitionFields.add(tableSchema.findField(kv[0]));
+ }
+ if (!partitionFields.isEmpty()) {
+ nativeIOReader.setPartitionSchema(new Schema(partitionFields));
}
initTableInfo(params);
- for (ScanPredicate predicate : predicates) {
- if (!partitionColumn.contains(predicate.columName)) {
- nativeIOReader.addFilter(ParquetFilter.toParquetFilter(predicate).toString());
- }
- }
-
nativeIOReader.initializeReader();
lakesoulArrowReader = new LakeSoulArrowReader(nativeIOReader, awaitTimeout);
}
@@ -156,4 +182,28 @@ public class LakeSoulJniScanner extends LakeSoulArrowJniScanner {
currentBatch.close();
}
}
+
+ public static void main(String[] args) throws IOException {
+ HashMap<String, String> params = new HashMap<>();
+ params.put("required_fields", "r_regionkey;r_name;r_comment");
+ params.put("primary_keys", "r_regionkey;r_name");
+ params.put("query_id", "e9d075a6500a4cac-b94630fd4b30c171");
+ params.put("file_paths",
+ "file:/Users/ceng/Documents/GitHub/LakeSoul/rust/lakesoul-datafusion/"
+ + "default/region/part-RzmUvDFtYV8ceb3J_0000.parquet"
+ );
+ params.put("options", "{}");
+ params.put("table_schema",
+ "{\"fields\":["
+ + "{\"name\":\"r_regionkey\",\"type\":{\"name\":\"int\",\"isSigned\":true,\"bitWidth\":64},"
+ + "\"nullable\":false,\"children\":[]},"
+ + "{\"name\":\"r_name\",\"type\":{\"name\":\"utf8\"},\"nullable\":false,\"children\":[]},"
+ + "{\"name\":\"r_comment\",\"type\":{\"name\":\"utf8\"},\"nullable\":false,\"children\":[]}"
+ + "],"
+ + "\"metadata\":null}");
+ params.put("partition_descs", "");
+ LakeSoulJniScanner scanner = new LakeSoulJniScanner(1024, params);
+ scanner.open();
+ System.out.println(scanner.getNext());
+ }
}
diff --git a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulUtils.java b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulUtils.java
index 6c7f88f3ab3..ca07a81d0da 100644
--- a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulUtils.java
+++ b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/LakeSoulUtils.java
@@ -17,13 +17,29 @@
package org.apache.doris.lakesoul;
+import java.util.Arrays;
+import java.util.List;
+
public class LakeSoulUtils {
- public static String FILE_NAMES = "file_paths";
- public static String PRIMARY_KEYS = "primary_keys";
- public static String SCHEMA_JSON = "table_schema";
- public static String PARTITION_DESC = "partition_descs";
- public static String REQUIRED_FIELDS = "required_fields";
+ public static final String FILE_NAMES = "file_paths";
+ public static final String PRIMARY_KEYS = "primary_keys";
+ public static final String SCHEMA_JSON = "table_schema";
+ public static final String PARTITION_DESC = "partition_descs";
+ public static final String REQUIRED_FIELDS = "required_fields";
+ public static final String OPTIONS = "options";
+ public static final String SUBSTRAIT_PREDICATE = "substrait_predicate";
+ public static final String LIST_DELIM = ";";
+ public static final String PARTITIONS_KV_DELIM = "=";
+
+ public static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key";
+ public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
+ public static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint";
+ public static final String FS_S3A_PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
- public static String LIST_DELIM = ";";
- public static String PARTITIONS_KV_DELIM = "=";
+ public static final List<String> OBJECT_STORE_OPTIONS = Arrays.asList(
+ FS_S3A_ACCESS_KEY,
+ FS_S3A_SECRET_KEY,
+ FS_S3A_ENDPOINT,
+ FS_S3A_PATH_STYLE_ACCESS
+ );
}
diff --git a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/ArrowUtils.java b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/ArrowUtils.java
index 3ad28ba783a..94ac32935e8 100644
--- a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/ArrowUtils.java
+++ b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/ArrowUtils.java
@@ -20,10 +20,10 @@ package org.apache.doris.lakesoul.arrow;
import org.apache.doris.common.jni.utils.OffHeap;
import org.apache.doris.common.jni.utils.TypeNativeBytes;
-import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.util.Preconditions;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.memory.ArrowBuf;
+import com.lakesoul.shaded.org.apache.arrow.util.Preconditions;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.ArrowType;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
import java.time.LocalDate;
import java.time.LocalDateTime;
diff --git a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/LakeSoulArrowJniScanner.java b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/LakeSoulArrowJniScanner.java
index 320d653a20a..3c73c2f1ab4 100644
--- a/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/LakeSoulArrowJniScanner.java
+++ b/fe/be-java-extensions/lakesoul-scanner/src/main/java/org/apache/doris/lakesoul/arrow/LakeSoulArrowJniScanner.java
@@ -23,28 +23,28 @@ import org.apache.doris.common.jni.vec.ColumnType;
import org.apache.doris.common.jni.vec.ScanPredicate;
import org.apache.doris.common.jni.vec.VectorTable;
-import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.DateDayVector;
-import org.apache.arrow.vector.DecimalVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TimeStampMicroTZVector;
-import org.apache.arrow.vector.TimeStampMicroVector;
-import org.apache.arrow.vector.TimeStampMilliTZVector;
-import org.apache.arrow.vector.TimeStampMilliVector;
-import org.apache.arrow.vector.TimeStampNanoTZVector;
-import org.apache.arrow.vector.TimeStampNanoVector;
-import org.apache.arrow.vector.TimeStampSecTZVector;
-import org.apache.arrow.vector.TimeStampSecVector;
-import org.apache.arrow.vector.TimeStampVector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VarCharVector;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.StructVector;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.Schema;
+import com.lakesoul.shaded.org.apache.arrow.memory.ArrowBuf;
+import com.lakesoul.shaded.org.apache.arrow.memory.BufferAllocator;
+import com.lakesoul.shaded.org.apache.arrow.vector.BitVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.DateDayVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.DecimalVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.FieldVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampMicroTZVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampMicroVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampMilliTZVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampMilliVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampNanoTZVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampNanoVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampSecTZVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampSecVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.TimeStampVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.ValueVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.VarCharVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.VectorSchemaRoot;
+import com.lakesoul.shaded.org.apache.arrow.vector.complex.ListVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.complex.StructVector;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Schema;
import org.apache.log4j.Logger;
import java.io.IOException;
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index bac2346185d..8021f2a3b18 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -568,7 +568,7 @@ under the License.
<dependency>
<groupId>com.dmetasoul</groupId>
<artifactId>lakesoul-common</artifactId>
- <version>2.5.4</version>
+ <version>${lakesoul.version}</version>
<classifier>shaded</classifier>
<exclusions>
<exclusion>
@@ -577,6 +577,46 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.dmetasoul</groupId>
+ <artifactId>lakesoul-io-java</artifactId>
+ <version>${lakesoul.version}</version>
+ <classifier>shaded</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
@@ -1220,4 +1260,4 @@ under the License.
</extension>
</extensions>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
index dd8342ad660..e813ac2fc97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.lakesoul;
+
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
@@ -25,6 +26,7 @@ import org.apache.doris.datasource.property.PropertyConverter;
import com.dmetasoul.lakesoul.meta.DBManager;
import com.dmetasoul.lakesoul.meta.DBUtil;
+import com.dmetasoul.lakesoul.meta.entity.PartitionInfo;
import com.dmetasoul.lakesoul.meta.entity.TableInfo;
import com.google.common.collect.Lists;
@@ -33,7 +35,7 @@ import java.util.Map;
public class LakeSoulExternalCatalog extends ExternalCatalog {
- private DBManager dbManager;
+ private DBManager lakesoulMetadataManager;
private final Map<String, String> props;
@@ -48,13 +50,13 @@ public class LakeSoulExternalCatalog extends ExternalCatalog {
@Override
protected List<String> listDatabaseNames() {
initLocalObjectsImpl();
- return dbManager.listNamespaces();
+ return lakesoulMetadataManager.listNamespaces();
}
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- List<TableInfo> tifs = dbManager.getTableInfosByNamespace(dbName);
+ List<TableInfo> tifs = lakesoulMetadataManager.getTableInfosByNamespace(dbName);
List<String> tableNames = Lists.newArrayList();
for (TableInfo item : tifs) {
tableNames.add(item.getTableName());
@@ -65,14 +67,13 @@ public class LakeSoulExternalCatalog extends ExternalCatalog {
@Override
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
makeSureInitialized();
- TableInfo tableInfo = dbManager.getTableInfoByNameAndNamespace(dbName, tblName);
-
+ TableInfo tableInfo = lakesoulMetadataManager.getTableInfoByNameAndNamespace(dbName, tblName);
return null != tableInfo;
}
@Override
protected void initLocalObjectsImpl() {
- if (dbManager == null) {
+ if (lakesoulMetadataManager == null) {
if (props != null) {
if (props.containsKey(DBUtil.urlKey)) {
System.setProperty(DBUtil.urlKey, props.get(DBUtil.urlKey));
@@ -84,13 +85,18 @@ public class LakeSoulExternalCatalog extends ExternalCatalog {
System.setProperty(DBUtil.passwordKey, props.get(DBUtil.passwordKey));
}
}
- dbManager = new DBManager();
+ lakesoulMetadataManager = new DBManager();
}
}
public TableInfo getLakeSoulTable(String dbName, String tblName) {
makeSureInitialized();
- return dbManager.getTableInfoByNameAndNamespace(tblName, dbName);
+ return lakesoulMetadataManager.getTableInfoByNameAndNamespace(tblName, dbName);
+ }
+
+ public List<PartitionInfo> listPartitionInfo(String tableId) {
+ makeSureInitialized();
+ return lakesoulMetadataManager.getAllPartitionInfo(tableId);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java
index 46e8d1db47c..a5cf3478ae8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java
@@ -25,17 +25,23 @@ import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.statistics.AnalysisInfo;
+import org.apache.doris.statistics.BaseAnalysisTask;
+import org.apache.doris.statistics.ExternalAnalysisTask;
import org.apache.doris.thrift.TLakeSoulTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
import com.dmetasoul.lakesoul.meta.DBUtil;
+import com.dmetasoul.lakesoul.meta.entity.PartitionInfo;
import com.dmetasoul.lakesoul.meta.entity.TableInfo;
import com.google.common.collect.Lists;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.HashMap;
@@ -45,11 +51,20 @@ import java.util.Optional;
import java.util.stream.Collectors;
public class LakeSoulExternalTable extends ExternalTable {
-
+ private static final Logger LOG = LogManager.getLogger(LakeSoulExternalTable.class);
public static final int LAKESOUL_TIMESTAMP_SCALE_MS = 6;
+ public final String tableId;
+
public LakeSoulExternalTable(long id, String name, String dbName, LakeSoulExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.LAKESOUl_EXTERNAL_TABLE);
+ tableId = getLakeSoulTableInfo().getTableId();
+ }
+
+ @Override
+ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
+ makeSureInitialized();
+ return new ExternalAnalysisTask(info);
}
private Type arrowFiledToDorisType(Field field) {
@@ -150,6 +165,7 @@ public class LakeSoulExternalTable extends ExternalTable {
String tableSchema = tableInfo.getTableSchema();
DBUtil.TablePartitionKeys partitionKeys = DBUtil.parseTableInfoPartitions(tableInfo.getPartitions());
Schema schema;
+ LOG.info("tableSchema={}", tableSchema);
try {
schema = Schema.fromJSON(tableSchema);
} catch (IOException e) {
@@ -174,6 +190,10 @@ public class LakeSoulExternalTable extends ExternalTable {
return ((LakeSoulExternalCatalog) catalog).getLakeSoulTable(dbName, name);
}
+ public List<PartitionInfo> listPartitionInfo() {
+ return ((LakeSoulExternalCatalog) catalog).listPartitionInfo(tableId);
+ }
+
public String tablePath() {
return ((LakeSoulExternalCatalog) catalog).getLakeSoulTable(dbName, name).getTablePath();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java
new file mode 100644
index 00000000000..8f7cf83dbfc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java
@@ -0,0 +1,535 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lakesoul;
+
+import org.apache.doris.analysis.BoolLiteral;
+import org.apache.doris.analysis.CastExpr;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.DateLiteral;
+import org.apache.doris.analysis.DecimalLiteral;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FloatLiteral;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.IsNullPredicate;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.NullLiteral;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.analysis.Subquery;
+import org.apache.doris.planner.ColumnBound;
+import org.apache.doris.planner.ColumnRange;
+import org.apache.doris.thrift.TExprOpcode;
+
+import com.dmetasoul.lakesoul.lakesoul.io.substrait.SubstraitUtil;
+import com.dmetasoul.lakesoul.meta.entity.PartitionInfo;
+import com.google.common.collect.BoundType;
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Schema;
+import io.substrait.expression.Expression;
+import io.substrait.extension.DefaultExtensionCatalog;
+import io.substrait.type.Type;
+import io.substrait.type.TypeCreator;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+public class LakeSoulUtils {
+
+ public static final String FILE_NAMES = "file_paths";
+ public static final String PRIMARY_KEYS = "primary_keys";
+ public static final String SCHEMA_JSON = "table_schema";
+ public static final String PARTITION_DESC = "partition_descs";
+ public static final String REQUIRED_FIELDS = "required_fields";
+ public static final String OPTIONS = "options";
+ public static final String SUBSTRAIT_PREDICATE = "substrait_predicate";
+ public static final String CDC_COLUMN = "lakesoul_cdc_change_column";
+ public static final String LIST_DELIM = ";";
+ public static final String PARTITIONS_KV_DELIM = "=";
+ public static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key";
+ public static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
+ public static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint";
+ public static final String FS_S3A_REGION = "fs.s3a.endpoint.region";
+ public static final String FS_S3A_PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
+
+ private static final OffsetDateTime EPOCH;
+ private static final LocalDate EPOCH_DAY;
+
+ static {
+ EPOCH = Instant.ofEpochSecond(0L).atOffset(ZoneOffset.UTC);
+ EPOCH_DAY = EPOCH.toLocalDate();
+ }
+
+ public static List<PartitionInfo> applyPartitionFilters(
+ List<PartitionInfo> allPartitionInfo,
+ String tableName,
+ Schema partitionArrowSchema,
+ Map<String, ColumnRange> columnNameToRange
+ ) throws IOException {
+
+ Expression conjunctionFilter = null;
+ for (Field field : partitionArrowSchema.getFields()) {
+ ColumnRange columnRange = columnNameToRange.get(field.getName());
+ if (columnRange != null) {
+ Expression expr = columnRangeToSubstraitFilter(field, columnRange);
+ if (expr != null) {
+ if (conjunctionFilter == null) {
+ conjunctionFilter = expr;
+ } else {
+ conjunctionFilter = SubstraitUtil.and(conjunctionFilter, expr);
+ }
+ }
+ }
+ }
+ return SubstraitUtil.applyPartitionFilters(
+ allPartitionInfo,
+ partitionArrowSchema,
+ SubstraitUtil.substraitExprToProto(conjunctionFilter, tableName)
+ );
+ }
+
+ public static Expression columnRangeToSubstraitFilter(
+ Field columnField,
+ ColumnRange columnRange
+ ) throws IOException {
+ Optional<RangeSet<ColumnBound>> rangeSetOpt = columnRange.getRangeSet();
+ if (columnRange.hasConjunctiveIsNull() || !rangeSetOpt.isPresent()) {
+ return SubstraitUtil.CONST_TRUE;
+ } else {
+ RangeSet<ColumnBound> rangeSet = rangeSetOpt.get();
+ if (rangeSet.isEmpty()) {
+ return SubstraitUtil.CONST_TRUE;
+ } else {
+ Expression conjunctionFilter = null;
+ for (Range range : rangeSet.asRanges()) {
+ Expression expr = rangeToSubstraitFilter(columnField, range);
+ if (expr != null) {
+ if (conjunctionFilter == null) {
+ conjunctionFilter = expr;
+ } else {
+ conjunctionFilter = SubstraitUtil.or(conjunctionFilter, expr);
+ }
+ }
+ }
+ return conjunctionFilter;
+ }
+ }
+ }
+
+ public static Expression rangeToSubstraitFilter(Field columnField, Range range) throws IOException {
+ if (!range.hasLowerBound() && !range.hasUpperBound()) {
+ // Range.all()
+ return SubstraitUtil.CONST_TRUE;
+ } else {
+ Expression upper = SubstraitUtil.CONST_TRUE;
+ if (range.hasUpperBound()) {
+ String func = range.upperBoundType() == BoundType.OPEN ? "lt:any_any" : "lte:any_any";
+ Expression left = SubstraitUtil.arrowFieldToSubstraitField(columnField);
+ Expression right = SubstraitUtil.anyToSubstraitLiteral(
+ SubstraitUtil.arrowFieldToSubstraitType(columnField),
+ ((ColumnBound) range.upperEndpoint()).getValue().getRealValue());
+ upper = SubstraitUtil.makeBinary(
+ left,
+ right,
+ DefaultExtensionCatalog.FUNCTIONS_COMPARISON,
+ func,
+ TypeCreator.NULLABLE.BOOLEAN
+ );
+ }
+ Expression lower = SubstraitUtil.CONST_TRUE;
+ if (range.hasLowerBound()) {
+ String func = range.lowerBoundType() == BoundType.OPEN ? "gt:any_any" : "gte:any_any";
+ Expression left = SubstraitUtil.arrowFieldToSubstraitField(columnField);
+ Expression right = SubstraitUtil.anyToSubstraitLiteral(
+ SubstraitUtil.arrowFieldToSubstraitType(columnField),
+ ((ColumnBound) range.lowerEndpoint()).getValue().getRealValue());
+ lower = SubstraitUtil.makeBinary(
+ left,
+ right,
+ DefaultExtensionCatalog.FUNCTIONS_COMPARISON,
+ func,
+ TypeCreator.NULLABLE.BOOLEAN
+ );
+ }
+ return SubstraitUtil.and(upper, lower);
+ }
+ }
+
+ public static io.substrait.proto.Plan getPushPredicate(
+ List<Expr> conjuncts,
+ String tableName,
+ Schema tableSchema,
+ Schema partitionArrowSchema,
+ Map<String, String> properties,
+ boolean incRead
+ ) throws IOException {
+
+ Set<String> partitionColumn =
+ partitionArrowSchema
+ .getFields()
+ .stream()
+ .map(Field::getName)
+ .collect(Collectors.toSet());
+ Expression conjunctionFilter = null;
+ String cdcColumn = properties.get(CDC_COLUMN);
+ if (cdcColumn != null && !incRead) {
+ conjunctionFilter = SubstraitUtil.cdcColumnMergeOnReadFilter(tableSchema.findField(cdcColumn));
+ }
+ for (Expr expr : conjuncts) {
+ if (!isAllPartitionPredicate(expr, partitionColumn)) {
+ Expression predicate = convertToSubstraitExpr(expr, tableSchema);
+ if (predicate != null) {
+ if (conjunctionFilter == null) {
+ conjunctionFilter = predicate;
+ } else {
+ conjunctionFilter = SubstraitUtil.and(conjunctionFilter, predicate);
+ }
+ }
+ }
+ }
+ if (conjunctionFilter == null) {
+ return null;
+ }
+ return SubstraitUtil.substraitExprToProto(conjunctionFilter, tableName);
+ }
+
+ public static boolean isAllPartitionPredicate(Expr predicate, Set<String> partitionColumns) {
+ if (predicate == null) {
+ return false;
+ }
+ if (predicate instanceof CompoundPredicate) {
+ CompoundPredicate compoundPredicate = (CompoundPredicate) predicate;
+ return isAllPartitionPredicate(compoundPredicate.getChild(0), partitionColumns)
+ && isAllPartitionPredicate(compoundPredicate.getChild(1), partitionColumns);
+ }
+ // Make sure the col slot is always first
+ SlotRef slotRef = convertDorisExprToSlotRef(predicate.getChild(0));
+ LiteralExpr literalExpr = convertDorisExprToLiteralExpr(predicate.getChild(1));
+ if (slotRef == null || literalExpr == null) {
+ return false;
+ }
+ String colName = slotRef.getColumnName();
+ return partitionColumns.contains(colName);
+
+ }
+
+ public static SlotRef convertDorisExprToSlotRef(Expr expr) {
+ SlotRef slotRef = null;
+ if (expr instanceof SlotRef) {
+ slotRef = (SlotRef) expr;
+ } else if (expr instanceof CastExpr) {
+ if (expr.getChild(0) instanceof SlotRef) {
+ slotRef = (SlotRef) expr.getChild(0);
+ }
+ }
+ return slotRef;
+ }
+
+ public static LiteralExpr convertDorisExprToLiteralExpr(Expr expr) {
+ LiteralExpr literalExpr = null;
+ if (expr instanceof LiteralExpr) {
+ literalExpr = (LiteralExpr) expr;
+ } else if (expr instanceof CastExpr) {
+ if (expr.getChild(0) instanceof LiteralExpr) {
+ literalExpr = (LiteralExpr) expr.getChild(0);
+ }
+ }
+ return literalExpr;
+ }
+
+ public static Expression convertToSubstraitExpr(Expr predicate, Schema tableSchema) throws IOException {
+ if (predicate == null) {
+ return null;
+ }
+ if (predicate instanceof BoolLiteral) {
+ BoolLiteral boolLiteral = (BoolLiteral) predicate;
+ boolean value = boolLiteral.getValue();
+ if (value) {
+ return SubstraitUtil.CONST_TRUE;
+ } else {
+ return SubstraitUtil.CONST_FALSE;
+ }
+ }
+ if (predicate instanceof CompoundPredicate) {
+ CompoundPredicate compoundPredicate = (CompoundPredicate) predicate;
+ switch (compoundPredicate.getOp()) {
+ case AND: {
+ Expression left = convertToSubstraitExpr(compoundPredicate.getChild(0), tableSchema);
+ Expression right = convertToSubstraitExpr(compoundPredicate.getChild(1), tableSchema);
+ if (left != null && right != null) {
+ return SubstraitUtil.and(left, right);
+ } else if (left != null) {
+ return left;
+ } else {
+ return right;
+ }
+ }
+ case OR: {
+ Expression left = convertToSubstraitExpr(compoundPredicate.getChild(0), tableSchema);
+ Expression right = convertToSubstraitExpr(compoundPredicate.getChild(1), tableSchema);
+ if (left != null && right != null) {
+ return SubstraitUtil.or(left, right);
+ }
+ return null;
+ }
+ case NOT: {
+ Expression child = convertToSubstraitExpr(compoundPredicate.getChild(0), tableSchema);
+ if (child != null) {
+ return SubstraitUtil.not(child);
+ }
+ return null;
+ }
+ default:
+ return null;
+ }
+ } else if (predicate instanceof InPredicate) {
+ InPredicate inExpr = (InPredicate) predicate;
+ if (inExpr.contains(Subquery.class)) {
+ return null;
+ }
+ SlotRef slotRef = convertDorisExprToSlotRef(inExpr.getChild(0));
+ if (slotRef == null) {
+ return null;
+ }
+ String colName = slotRef.getColumnName();
+ Field field = tableSchema.findField(colName);
+ Expression fieldRef = SubstraitUtil.arrowFieldToSubstraitField(field);
+
+ colName = field.getName();
+ Type type = field.getType().accept(
+ new SubstraitUtil.ArrowTypeToSubstraitTypeConverter(field.isNullable())
+ );
+ List<Expression.Literal> valueList = new ArrayList<>();
+ for (int i = 1; i < inExpr.getChildren().size(); ++i) {
+ if (!(inExpr.getChild(i) instanceof LiteralExpr)) {
+ return null;
+ }
+ LiteralExpr literalExpr = (LiteralExpr) inExpr.getChild(i);
+ Object value = extractDorisLiteral(type, literalExpr);
+ if (value == null) {
+ return null;
+ }
+ valueList.add(SubstraitUtil.anyToSubstraitLiteral(type, value));
+ }
+ if (inExpr.isNotIn()) {
+ // not in
+ return SubstraitUtil.notIn(fieldRef, valueList);
+ } else {
+ // in
+ return SubstraitUtil.in(fieldRef, valueList);
+ }
+ }
+ return convertBinaryExpr(predicate, tableSchema);
+ }
+
+ private static Expression convertBinaryExpr(Expr dorisExpr, Schema tableSchema) throws IOException {
+ TExprOpcode opcode = dorisExpr.getOpcode();
+ // Make sure the col slot is always first
+ SlotRef slotRef = convertDorisExprToSlotRef(dorisExpr.getChild(0));
+ LiteralExpr literalExpr = convertDorisExprToLiteralExpr(dorisExpr.getChild(1));
+ if (slotRef == null || literalExpr == null) {
+ return null;
+ }
+ String colName = slotRef.getColumnName();
+ Field field = tableSchema.findField(colName);
+ Expression fieldRef = SubstraitUtil.arrowFieldToSubstraitField(field);
+
+ Type type = field.getType().accept(
+ new SubstraitUtil.ArrowTypeToSubstraitTypeConverter(field.isNullable())
+ );
+ Object value = extractDorisLiteral(type, literalExpr);
+ if (value == null) {
+ if (opcode == TExprOpcode.EQ_FOR_NULL && literalExpr instanceof NullLiteral) {
+ return SubstraitUtil.makeUnary(
+ fieldRef,
+ DefaultExtensionCatalog.FUNCTIONS_COMPARISON,
+ "is_null:any",
+ TypeCreator.NULLABLE.BOOLEAN);
+ } else {
+ return null;
+ }
+ }
+ Expression literal = SubstraitUtil.anyToSubstraitLiteral(
+ type,
+ value
+ );
+
+ String namespace;
+ String func;
+ switch (opcode) {
+ case EQ:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "equal:any_any";
+ break;
+ case EQ_FOR_NULL:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "is_null:any";
+ break;
+ case NE:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "not_equal:any_any";
+ break;
+ case GE:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "gte:any_any";
+ break;
+ case GT:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "gt:any_any";
+ break;
+ case LE:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "lte:any_any";
+ break;
+ case LT:
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "lt:any_any";
+ break;
+ case INVALID_OPCODE:
+ if (dorisExpr instanceof FunctionCallExpr) {
+ String name = dorisExpr.getExprName().toLowerCase();
+ String s = literalExpr.getStringValue();
+ if (name.equals("like") && !s.startsWith("%") && s.endsWith("%")) {
+ namespace = DefaultExtensionCatalog.FUNCTIONS_STRING;
+ func = "like:bool";
+ break;
+ }
+ } else if (dorisExpr instanceof IsNullPredicate) {
+ if (((IsNullPredicate) dorisExpr).isNotNull()) {
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "is_not_null:any";
+
+ } else {
+ namespace = DefaultExtensionCatalog.FUNCTIONS_COMPARISON;
+ func = "is_null:any";
+ }
+ break;
+ }
+ return null;
+ default:
+ return null;
+ }
+ return SubstraitUtil.makeBinary(fieldRef, literal, namespace, func, TypeCreator.NULLABLE.BOOLEAN);
+ }
+
+ public static Object extractDorisLiteral(Type type, LiteralExpr expr) {
+
+ if (expr instanceof BoolLiteral) {
+ if (type instanceof Type.Bool) {
+ return ((BoolLiteral) expr).getValue();
+ }
+ if (type instanceof Type.Str) {
+ return expr.getStringValue();
+ }
+ } else if (expr instanceof DateLiteral) {
+ DateLiteral dateLiteral = (DateLiteral) expr;
+ if (type instanceof Type.Date) {
+ if (dateLiteral.getType().isDatetimeV2() || dateLiteral.getType().isDatetime()) {
+ return null;
+ }
+ return dateLiteral.getLongValue();
+ }
+ if (type instanceof Type.TimestampTZ || type instanceof Type.Timestamp) {
+ return dateLiteral.getLongValue();
+ }
+ if (type instanceof Type.Str) {
+ return expr.getStringValue();
+ }
+ } else if (expr instanceof DecimalLiteral) {
+ DecimalLiteral decimalLiteral = (DecimalLiteral) expr;
+ if (type instanceof Type.Decimal) {
+ return decimalLiteral.getValue();
+ } else if (type instanceof Type.FP64) {
+ return decimalLiteral.getDoubleValue();
+ }
+ if (type instanceof Type.Str) {
+ return expr.getStringValue();
+ }
+ } else if (expr instanceof FloatLiteral) {
+ FloatLiteral floatLiteral = (FloatLiteral) expr;
+
+ if (floatLiteral.getType() == org.apache.doris.catalog.Type.FLOAT) {
+ return type instanceof Type.FP32
+ || type instanceof Type.FP64
+ || type instanceof Type.Decimal ? ((FloatLiteral) expr).getValue() : null;
+ } else {
+ return type instanceof Type.FP64
+ || type instanceof Type.Decimal ? ((FloatLiteral) expr).getValue() : null;
+ }
+ } else if (expr instanceof IntLiteral) {
+ if (type instanceof Type.I8
+ || type instanceof Type.I16
+ || type instanceof Type.I32
+ || type instanceof Type.I64
+ || type instanceof Type.FP32
+ || type instanceof Type.FP64
+ || type instanceof Type.Decimal
+ || type instanceof Type.Date
+ ) {
+ return expr.getRealValue();
+ }
+ if (!expr.getType().isInteger32Type()) {
+ if (type instanceof Type.Time || type instanceof Type.Timestamp || type instanceof Type.TimestampTZ) {
+ return expr.getLongValue();
+ }
+ }
+
+ } else if (expr instanceof StringLiteral) {
+ String value = expr.getStringValue();
+ if (type instanceof Type.Str) {
+ return value;
+ }
+ if (type instanceof Type.Date) {
+ try {
+ return (int) ChronoUnit.DAYS.between(
+ EPOCH_DAY,
+ LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE));
+ } catch (DateTimeParseException e) {
+ return null;
+ }
+ }
+ if (type instanceof Type.Timestamp || type instanceof Type.TimestampTZ) {
+ try {
+ return ChronoUnit.MICROS.between(
+ EPOCH,
+ OffsetDateTime.parse(value, DateTimeFormatter.ISO_DATE_TIME));
+ } catch (DateTimeParseException e) {
+ return null;
+ }
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
index 1779aeaca10..fd36bfd52bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java
@@ -17,14 +17,19 @@
package org.apache.doris.datasource.lakesoul.source;
+
+
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.LocationPath;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.FileQueryScanNode;
import org.apache.doris.datasource.TableFormatType;
import org.apache.doris.datasource.lakesoul.LakeSoulExternalTable;
+import org.apache.doris.datasource.lakesoul.LakeSoulUtils;
+import org.apache.doris.datasource.property.constants.S3Properties;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
@@ -34,15 +39,26 @@ import org.apache.doris.thrift.TFileType;
import org.apache.doris.thrift.TLakeSoulFileDesc;
import org.apache.doris.thrift.TTableFormatFileDesc;
+import com.dmetasoul.lakesoul.lakesoul.io.substrait.SubstraitUtil;
import com.dmetasoul.lakesoul.meta.DBUtil;
import com.dmetasoul.lakesoul.meta.DataFileInfo;
import com.dmetasoul.lakesoul.meta.DataOperation;
import com.dmetasoul.lakesoul.meta.LakeSoulOptions;
+import com.dmetasoul.lakesoul.meta.entity.PartitionInfo;
import com.dmetasoul.lakesoul.meta.entity.TableInfo;
import com.google.common.collect.Lists;
import com.lakesoul.shaded.com.alibaba.fastjson.JSON;
import com.lakesoul.shaded.com.alibaba.fastjson.JSONObject;
+import com.lakesoul.shaded.com.fasterxml.jackson.core.type.TypeReference;
+import com.lakesoul.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Schema;
+import io.substrait.proto.Plan;
+import lombok.SneakyThrows;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@@ -52,19 +68,54 @@ import java.util.stream.Collectors;
public class LakeSoulScanNode extends FileQueryScanNode {
- protected final LakeSoulExternalTable lakeSoulExternalTable;
+ private static final Logger LOG = LogManager.getLogger(LakeSoulScanNode.class);
+ protected LakeSoulExternalTable lakeSoulExternalTable;
+
+ String tableName;
+
+ String location;
- protected final TableInfo table;
+ String partitions;
+
+ Schema tableArrowSchema;
+
+ Schema partitionArrowSchema;
+ private Map<String, String> tableProperties;
+
+ String readType;
public LakeSoulScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
super(id, desc, "planNodeName", StatisticalType.LAKESOUL_SCAN_NODE, needCheckColumnPriv);
+ }
+
+ @Override
+ protected void doInitialize() throws UserException {
+ super.doInitialize();
lakeSoulExternalTable = (LakeSoulExternalTable) desc.getTable();
- table = lakeSoulExternalTable.getLakeSoulTableInfo();
+ TableInfo tableInfo = lakeSoulExternalTable.getLakeSoulTableInfo();
+ location = tableInfo.getTablePath();
+ tableName = tableInfo.getTableName();
+ partitions = tableInfo.getPartitions();
+ readType = LakeSoulOptions.ReadType$.MODULE$.FULL_READ();
+ try {
+ tableProperties = new ObjectMapper().readValue(
+ tableInfo.getProperties(),
+ new TypeReference<Map<String, String>>() {}
+ );
+ tableArrowSchema = Schema.fromJSON(tableInfo.getTableSchema());
+ List<Field> partitionFields =
+ DBUtil.parseTableInfoPartitions(partitions)
+ .rangeKeys
+ .stream()
+ .map(tableArrowSchema::findField).collect(Collectors.toList());
+ partitionArrowSchema = new Schema(partitionFields);
+ } catch (IOException e) {
+ throw new UserException(e);
+ }
}
@Override
protected TFileType getLocationType() throws UserException {
- String location = table.getTablePath();
return getLocationType(location);
}
@@ -81,12 +132,12 @@ public class LakeSoulScanNode extends FileQueryScanNode {
@Override
protected List<String> getPathPartitionKeys() throws UserException {
- return new ArrayList<>(DBUtil.parseTableInfoPartitions(table.getPartitions()).rangeKeys);
+ return new ArrayList<>(DBUtil.parseTableInfoPartitions(partitions).rangeKeys);
}
@Override
protected TableIf getTargetTable() throws UserException {
- return lakeSoulExternalTable;
+ return desc.getTable();
}
@Override
@@ -94,13 +145,21 @@ public class LakeSoulScanNode extends FileQueryScanNode {
return lakeSoulExternalTable.getHadoopProperties();
}
+ @SneakyThrows
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}", rangeDesc);
+ }
if (split instanceof LakeSoulSplit) {
setLakeSoulParams(rangeDesc, (LakeSoulSplit) split);
}
}
+ public ExternalCatalog getCatalog() {
+ return lakeSoulExternalTable.getCatalog();
+ }
+
public static boolean isExistHashPartition(TableInfo tif) {
JSONObject tableProperties = JSON.parseObject(tif.getProperties());
if (tableProperties.containsKey(LakeSoulOptions.HASH_BUCKET_NUM())
@@ -111,13 +170,47 @@ public class LakeSoulScanNode extends FileQueryScanNode {
}
}
- public void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulSplit) {
+ public void setLakeSoulParams(TFileRangeDesc rangeDesc, LakeSoulSplit lakeSoulSplit) throws IOException {
TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
tableFormatFileDesc.setTableFormatType(lakeSoulSplit.getTableFormatType().value());
TLakeSoulFileDesc fileDesc = new TLakeSoulFileDesc();
fileDesc.setFilePaths(lakeSoulSplit.getPaths());
fileDesc.setPrimaryKeys(lakeSoulSplit.getPrimaryKeys());
fileDesc.setTableSchema(lakeSoulSplit.getTableSchema());
+
+
+ JSONObject options = new JSONObject();
+ Plan predicate = LakeSoulUtils.getPushPredicate(
+ conjuncts,
+ tableName,
+ tableArrowSchema,
+ partitionArrowSchema,
+ tableProperties,
+ readType.equals(LakeSoulOptions.ReadType$.MODULE$.INCREMENTAL_READ()));
+ if (predicate != null) {
+ options.put(LakeSoulUtils.SUBSTRAIT_PREDICATE, SubstraitUtil.encodeBase64String(predicate));
+ }
+ Map<String, String> catalogProps = getCatalog().getProperties();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}", catalogProps);
+ }
+
+ if (catalogProps.get(S3Properties.Env.ENDPOINT) != null) {
+ options.put(LakeSoulUtils.FS_S3A_ENDPOINT, catalogProps.get(S3Properties.Env.ENDPOINT));
+ options.put(LakeSoulUtils.FS_S3A_PATH_STYLE_ACCESS, "true");
+ if (catalogProps.get(S3Properties.Env.ACCESS_KEY) != null) {
+ options.put(LakeSoulUtils.FS_S3A_ACCESS_KEY, catalogProps.get(S3Properties.Env.ACCESS_KEY));
+ }
+ if (catalogProps.get(S3Properties.Env.SECRET_KEY) != null) {
+ options.put(LakeSoulUtils.FS_S3A_SECRET_KEY, catalogProps.get(S3Properties.Env.SECRET_KEY));
+ }
+ if (catalogProps.get(S3Properties.Env.REGION) != null) {
+ options.put(LakeSoulUtils.FS_S3A_REGION, catalogProps.get(S3Properties.Env.REGION));
+ }
+ }
+
+ fileDesc.setOptions(JSON.toJSONString(options));
+
fileDesc.setPartitionDescs(lakeSoulSplit.getPartitionDesc()
.entrySet().stream().map(entry ->
String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()));
@@ -126,24 +219,51 @@ public class LakeSoulScanNode extends FileQueryScanNode {
}
public List<Split> getSplits() throws UserException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getSplits with columnFilters={}", columnFilters);
+ LOG.debug("getSplits with columnNameToRange={}", columnNameToRange);
+ LOG.debug("getSplits with conjuncts={}", conjuncts);
+ }
+
+ List<PartitionInfo> allPartitionInfo = lakeSoulExternalTable.listPartitionInfo();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allPartitionInfo={}", allPartitionInfo);
+ }
+ List<PartitionInfo> filteredPartitionInfo = allPartitionInfo;
+ try {
+ filteredPartitionInfo =
+ LakeSoulUtils.applyPartitionFilters(
+ allPartitionInfo,
+ tableName,
+ partitionArrowSchema,
+ columnNameToRange
+ );
+ } catch (IOException e) {
+ throw new UserException(e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("filteredPartitionInfo={}", filteredPartitionInfo);
+ }
+ DataFileInfo[] dataFileInfos = DataOperation.getTableDataInfo(filteredPartitionInfo);
+
List<Split> splits = new ArrayList<>();
Map<String, Map<Integer, List<String>>> splitByRangeAndHashPartition = new LinkedHashMap<>();
- TableInfo tif = table;
- DataFileInfo[] dfinfos = DataOperation.getTableDataInfo(table.getTableId());
- for (DataFileInfo pif : dfinfos) {
- if (isExistHashPartition(tif) && pif.file_bucket_id() != -1) {
- splitByRangeAndHashPartition.computeIfAbsent(pif.range_partitions(), k -> new LinkedHashMap<>())
- .computeIfAbsent(pif.file_bucket_id(), v -> new ArrayList<>())
- .add(pif.path());
+ TableInfo tableInfo = lakeSoulExternalTable.getLakeSoulTableInfo();
+
+ for (DataFileInfo fileInfo : dataFileInfos) {
+ if (isExistHashPartition(tableInfo) && fileInfo.file_bucket_id() != -1) {
+ splitByRangeAndHashPartition.computeIfAbsent(fileInfo.range_partitions(), k -> new LinkedHashMap<>())
+ .computeIfAbsent(fileInfo.file_bucket_id(), v -> new ArrayList<>())
+ .add(fileInfo.path());
} else {
- splitByRangeAndHashPartition.computeIfAbsent(pif.range_partitions(), k -> new LinkedHashMap<>())
+ splitByRangeAndHashPartition.computeIfAbsent(fileInfo.range_partitions(), k -> new LinkedHashMap<>())
.computeIfAbsent(-1, v -> new ArrayList<>())
- .add(pif.path());
+ .add(fileInfo.path());
}
}
List<String> pkKeys = null;
- if (!table.getPartitions().equals(";")) {
- pkKeys = Lists.newArrayList(table.getPartitions().split(";")[1].split(","));
+ if (!tableInfo.getPartitions().equals(";")) {
+ pkKeys = Lists.newArrayList(tableInfo.getPartitions().split(";")[1].split(","));
}
for (Map.Entry<String, Map<Integer, List<String>>> entry : splitByRangeAndHashPartition.entrySet()) {
@@ -161,7 +281,7 @@ public class LakeSoulScanNode extends FileQueryScanNode {
split.getValue(),
pkKeys,
rangeDesc,
- table.getTableSchema(),
+ tableInfo.getTableSchema(),
0, 0, 0,
new String[0], null);
lakeSoulSplit.setTableFormatType(TableFormatType.LAKESOUL);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 82945fb6963..64178846abf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -283,6 +283,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
+ case LAKESOUl_EXTERNAL_TABLE:
return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table,
qualifierWithoutTableName, unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lakesoul/LakeSoulPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lakesoul/LakeSoulPredicateTest.java
new file mode 100644
index 00000000000..aebd74f5e02
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lakesoul/LakeSoulPredicateTest.java
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.lakesoul;
+
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.BoolLiteral;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.CompoundPredicate.Operator;
+import org.apache.doris.analysis.DateLiteral;
+import org.apache.doris.analysis.DecimalLiteral;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FloatLiteral;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+
+import com.dmetasoul.lakesoul.lakesoul.io.substrait.SubstraitUtil;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.DateUnit;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.FloatingPointPrecision;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.TimeUnit;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.ArrowType;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Field;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.FieldType;
+import com.lakesoul.shaded.org.apache.arrow.vector.types.pojo.Schema;
+import io.substrait.expression.Expression;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class LakeSoulPredicateTest {
+
+ public static Schema schema;
+
+ @BeforeClass
+ public static void before() throws AnalysisException, IOException {
+ schema = new Schema(
+ Arrays.asList(
+ new Field("c_int", FieldType.nullable(new ArrowType.Int(32, true)), null),
+ new Field("c_long", FieldType.nullable(new ArrowType.Int(64, true)), null),
+ new Field("c_bool", FieldType.nullable(new ArrowType.Bool()), null),
+ new Field("c_float", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), null),
+ new Field("c_double", FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), null),
+ new Field("c_dec", FieldType.nullable(new ArrowType.Decimal(20, 10)), null),
+ new Field("c_date", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null),
+ new Field("c_ts", FieldType.nullable(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")), null),
+ new Field("c_str", FieldType.nullable(new ArrowType.Utf8()), null)
+ ));
+ }
+
+ @Test
+ public void testBinaryPredicate() throws AnalysisException, IOException {
+ List<LiteralExpr> literalList = new ArrayList<LiteralExpr>() {{
+ add(new BoolLiteral(true));
+ add(new DateLiteral("2023-01-02", Type.DATEV2));
+ add(new DateLiteral("2024-01-02 12:34:56.123456", Type.DATETIMEV2));
+ add(new DecimalLiteral(new BigDecimal("1.23")));
+ add(new FloatLiteral(1.23, Type.FLOAT));
+ add(new FloatLiteral(3.456, Type.DOUBLE));
+ add(new IntLiteral(1, Type.TINYINT));
+ add(new IntLiteral(1, Type.SMALLINT));
+ add(new IntLiteral(1, Type.INT));
+ add(new IntLiteral(1, Type.BIGINT));
+ add(new StringLiteral("abc"));
+ add(new StringLiteral("2023-01-02"));
+ add(new StringLiteral("2023-01-02 01:02:03.456789"));
+ }};
+
+ List<SlotRef> slotRefs = new ArrayList<SlotRef>() {{
+ add(new SlotRef(new TableName(), "c_int"));
+ add(new SlotRef(new TableName(), "c_long"));
+ add(new SlotRef(new TableName(), "c_bool"));
+ add(new SlotRef(new TableName(), "c_float"));
+ add(new SlotRef(new TableName(), "c_double"));
+ add(new SlotRef(new TableName(), "c_dec"));
+ add(new SlotRef(new TableName(), "c_date"));
+ add(new SlotRef(new TableName(), "c_ts"));
+ add(new SlotRef(new TableName(), "c_str"));
+ }};
+
+ // true indicates support for pushdown
+ Boolean[][] expects = new Boolean[][] {
+ { // int
+ false, false, false, false, false, false, true, true, true, true, false, false, false
+ },
+ { // long
+ false, false, false, false, false, false, true, true, true, true, false, false, false
+ },
+ { // boolean
+ true, false, false, false, false, false, false, false, false, false, false, false, false
+ },
+ { // float
+ false, false, false, false, true, false, true, true, true, true, false, false, false
+ },
+ { // double
+ false, false, false, true, true, true, true, true, true, true, false, false, false
+ },
+ { // decimal
+ false, false, false, true, true, true, true, true, true, true, false, false, false
+ },
+ { // date
+ false, true, false, false, false, false, true, true, true, true, false, true, false
+ },
+ { // timestamp
+ false, true, true, false, false, false, false, false, false, true, false, false, false
+ },
+ { // string
+ true, true, true, true, false, false, false, false, false, false, true, true, true
+ }
+ };
+
+ ArrayListMultimap<Boolean, Expr> validPredicateMap = ArrayListMultimap.create();
+
+ // binary predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ BinaryPredicate expr = new BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal);
+ Expression expression = null;
+ try {
+ expression = LakeSoulUtils.convertToSubstraitExpr(expr, schema);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // in predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), false);
+ Expression expression = null;
+ try {
+ expression = LakeSoulUtils.convertToSubstraitExpr(expr, schema);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // not in predicate
+ for (int i = 0; i < slotRefs.size(); i++) {
+ final int loc = i;
+ List<Boolean> ret = literalList.stream().map(literal -> {
+ InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), true);
+ Expression expression = null;
+ try {
+ expression = LakeSoulUtils.convertToSubstraitExpr(expr, schema);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ validPredicateMap.put(expression != null, expr);
+ return expression != null;
+ }).collect(Collectors.toList());
+ Assert.assertArrayEquals(expects[i], ret.toArray());
+ }
+
+ // bool literal
+
+ Expression trueExpr = LakeSoulUtils.convertToSubstraitExpr(new BoolLiteral(true), schema);
+ Expression falseExpr = LakeSoulUtils.convertToSubstraitExpr(new BoolLiteral(false), schema);
+ Assert.assertEquals(SubstraitUtil.CONST_TRUE, trueExpr);
+ Assert.assertEquals(SubstraitUtil.CONST_FALSE, falseExpr);
+ validPredicateMap.put(true, new BoolLiteral(true));
+ validPredicateMap.put(true, new BoolLiteral(false));
+
+ List<Expr> validExprs = validPredicateMap.get(true);
+ List<Expr> invalidExprs = validPredicateMap.get(false);
+ // OR predicate
+ // both valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < validExprs.size(); j++) {
+ CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR,
+ validExprs.get(i), validExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(orPredicate, schema);
+ Assert.assertNotNull("pred: " + orPredicate.toSql(), expression);
+ }
+ }
+ // both invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR,
+ invalidExprs.get(i), invalidExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(orPredicate, schema);
+ Assert.assertNull("pred: " + orPredicate.toSql(), expression);
+ }
+ }
+ // valid or invalid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR,
+ validExprs.get(i), invalidExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(orPredicate, schema);
+ Assert.assertNull("pred: " + orPredicate.toSql(), expression);
+ }
+ }
+
+ // AND predicate
+ // both valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < validExprs.size(); j++) {
+ CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND,
+ validExprs.get(i), validExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(andPredicate, schema);
+ Assert.assertNotNull("pred: " + andPredicate.toSql(), expression);
+ }
+ }
+ // both invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND,
+ invalidExprs.get(i), invalidExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(andPredicate, schema);
+ Assert.assertNull("pred: " + andPredicate.toSql(), expression);
+ }
+ }
+ // valid and invalid
+ for (int i = 0; i < validExprs.size(); i++) {
+ for (int j = 0; j < invalidExprs.size(); j++) {
+ CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND,
+ validExprs.get(i), invalidExprs.get(j));
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(andPredicate, schema);
+ Assert.assertNotNull("pred: " + andPredicate.toSql(), expression);
+ Assert.assertEquals(SubstraitUtil.substraitExprToProto(LakeSoulUtils.convertToSubstraitExpr(validExprs.get(i), schema), "table"),
+ SubstraitUtil.substraitExprToProto(expression, "table"));
+ }
+ }
+
+ // NOT predicate
+ // valid
+ for (int i = 0; i < validExprs.size(); i++) {
+ CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT,
+ validExprs.get(i), null);
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(notPredicate, schema);
+ Assert.assertNotNull("pred: " + notPredicate.toSql(), expression);
+ }
+ // invalid
+ for (int i = 0; i < invalidExprs.size(); i++) {
+ CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT,
+ invalidExprs.get(i), null);
+ Expression expression = LakeSoulUtils.convertToSubstraitExpr(notPredicate, schema);
+ Assert.assertNull("pred: " + notPredicate.toSql(), expression);
+ }
+ }
+}
diff --git a/fe/pom.xml b/fe/pom.xml
index 0c08bb327ae..3d43297fe21 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -317,6 +317,8 @@ under the License.
<hudi.version>0.14.1</hudi.version>
<presto.hadoop.version>2.7.4-11</presto.hadoop.version>
<presto.hive.version>3.0.0-8</presto.hive.version>
+ <!-- lakesoul -->
+ <lakesoul.version>2.6.1</lakesoul.version>
<parquet.version>1.13.1</parquet.version>
<commons-collections.version>3.2.2</commons-collections.version>
@@ -1831,4 +1833,4 @@ under the License.
</snapshots>
</repository>
</repositories>
-</project>
\ No newline at end of file
+</project>
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 527b0231394..f4fb41fdf55 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -224,3 +224,14 @@ enableTrinoConnectorTest = false
enableKerberosTest=false
kerberosHmsPort=9883
kerberosHdfsPort=8820
+
+
+// LakeSoul catalog test config
+enableLakesoulTest = false
+lakesoulPGUser="*******"
+lakesoulPGPwd="*******"
+lakesoulPGUrl="*******"
+lakesoulMinioAK="*******"
+lakesoulMinioSK="*******"
+lakesoulMinioEndpoint="*******"
+
diff --git a/regression-test/data/external_table_p0/lakesoul/test_lakesoul_filter.out b/regression-test/data/external_table_p0/lakesoul/test_lakesoul_filter.out
new file mode 100644
index 00000000000..cb1899326bd
--- /dev/null
+++ b/regression-test/data/external_table_p0/lakesoul/test_lakesoul_filter.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !lakesoul --
+0 AFRICA lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to
+2 ASIA ges. thinly even pinto beans ca
+3 EUROPE ly final courts cajole furiously final excuse
+1 AMERICA hs use ironic, even requests. s
+4 MIDDLE EAST uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl
+
diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy
index 28956568e58..0299066fa02 100644
--- a/regression-test/pipeline/external/conf/regression-conf.groovy
+++ b/regression-test/pipeline/external/conf/regression-conf.groovy
@@ -163,3 +163,7 @@ enableTrinoConnectorTest = true
enableKerberosTest = true
kerberosHmsPort=9883
kerberosHdfsPort=8820
+
+
+// LakeSoul catalog test config
+enableLakesoulTest = true
diff --git a/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_catalog.groovy b/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_catalog.groovy
index e0b8a924c30..ffd95d93097 100644
--- a/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_catalog.groovy
+++ b/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_catalog.groovy
@@ -16,20 +16,33 @@
// under the License.
suite("test_lakesoul_catalog", "p0,external,doris,external_docker,external_docker_doris") {
- def enabled = false;
+ String enabled = context.config.otherConfigs.get("enableLakesoulTest")
// open it when docker image is ready to run in regression test
- if (enabled) {
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
String catalog_name = "lakesoul"
String db_name = "default"
+ String pg_user = context.config.otherConfigs.get("lakesoulPGUser")
+ String pg_pwd = context.config.otherConfigs.get("lakesoulPGPwd")
+ String pg_url = context.config.otherConfigs.get("lakesoulPGUrl")
+ String minio_ak = context.config.otherConfigs.get("lakesoulMinioAK")
+ String minio_sk = context.config.otherConfigs.get("lakesoulMinioSK")
+ String minio_endpoint = context.config.otherConfigs.get("lakesoulMinioEndpoint")
sql """drop catalog if exists ${catalog_name}"""
- sql """
- create catalog lakesoul properties ('type'='lakesoul','lakesoul.pg.username'='lakesoul_test','lakesoul.pg.password'='lakesoul_test','lakesoul.pg.url'='jdbc:postgresql://127.0.0.1:5432/lakesoul_test?stringtype=unspecified');"""
+ sql """create catalog lakesoul properties (
+ 'type'='lakesoul',
+ 'lakesoul.pg.username'='${pg_user}',
+ 'lakesoul.pg.password'='${pg_pwd}',
+ 'lakesoul.pg.url'='${pg_url}',
+ 'minio.endpoint'='${minio_endpoint}',
+ 'minio.access_key'='${minio_ak}',
+ 'minio.secret_key'='${minio_sk}'
+ );"""
// analyze
sql """use `${catalog_name}`.`${db_name}`"""
- sq """show tables;"""
+ sql """show tables;"""
// select
sql """select * from nation;"""
diff --git a/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_filter.groovy b/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_filter.groovy
new file mode 100644
index 00000000000..799e8ba61bb
--- /dev/null
+++ b/regression-test/suites/external_table_p0/lakesoul/test_lakesoul_filter.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_lakesoul_filter", "p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableLakesoulTest")
+ // open it when docker image is ready to run in regression test
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String catalog_name = "lakesoul"
+ String db_name = "default"
+ String pg_user = context.config.otherConfigs.get("lakesoulPGUser")
+ String pg_pwd = context.config.otherConfigs.get("lakesoulPGPwd")
+ String pg_url = context.config.otherConfigs.get("lakesoulPGUrl")
+ String minio_ak = context.config.otherConfigs.get("lakesoulMinioAK")
+ String minio_sk = context.config.otherConfigs.get("lakesoulMinioSK")
+ String minio_endpoint = context.config.otherConfigs.get("lakesoulMinioEndpoint")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog lakesoul properties (
+ 'type'='lakesoul',
+ 'lakesoul.pg.username'='${pg_user}',
+ 'lakesoul.pg.password'='${pg_pwd}',
+ 'lakesoul.pg.url'='${pg_url}',
+ 'minio.endpoint'='${minio_endpoint}',
+ 'minio.access_key'='${minio_ak}',
+ 'minio.secret_key'='${minio_sk}'
+ );"""
+
+ // analyze
+ sql """use `${catalog_name}`.`${db_name}`"""
+
+ sql """show tables;"""
+ // select
+ sql """select * from region;"""
+
+ sql """select * from nation;"""
+
+ sql """select * from nation where n_regionkey = 0 or n_nationkey > 14;"""
+
+ sql """select * from nation where n_regionkey = 0 and n_nationkey > 0;"""
+
+ sql """select * from nation where n_regionkey = 0;"""
+ }
+}
+
diff --git a/regression-test/suites/external_table_p2/lakesoul/test_external_table_lakesoul.groovy b/regression-test/suites/external_table_p2/lakesoul/test_external_table_lakesoul.groovy
index 9369a28e8fe..bb85dc687d7 100644
--- a/regression-test/suites/external_table_p2/lakesoul/test_external_table_lakesoul.groovy
+++ b/regression-test/suites/external_table_p2/lakesoul/test_external_table_lakesoul.groovy
@@ -34,13 +34,25 @@ suite("test_external_table_lakesoul", "p2,external,lakesoul,external_remote,exte
if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String catalog_name = "lakesoul"
- String db_name = "default"
+ String catalog_name = "lakesoul"
+ String db_name = "default"
+ String pg_user = context.config.otherConfigs.get("lakesoulPGUser")
+ String pg_pwd = context.config.otherConfigs.get("lakesoulPGPwd")
+ String pg_url = context.config.otherConfigs.get("lakesoulPGUrl")
+ String minio_ak = context.config.otherConfigs.get("lakesoulMinioAK")
+ String minio_sk = context.config.otherConfigs.get("lakesoulMinioSK")
+ String minio_endpoint = context.config.otherConfigs.get("lakesoulMinioEndpoint")
- sql """drop catalog if exists ${catalog_name}"""
- sql """
- create catalog lakesoul properties ('type'='lakesoul','lakesoul.pg.username'='lakesoul','lakesoul.pg.url'='jdbc:postgresql://127.0.0.1:5433/lakesoul_test?stringtype=unspecified');
- """
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog lakesoul properties (
+ 'type'='lakesoul',
+ 'lakesoul.pg.username'='${pg_user}',
+ 'lakesoul.pg.password'='${pg_pwd}',
+ 'lakesoul.pg.url'='${pg_url}',
+ 'minio.endpoint'='${minio_endpoint}',
+ 'minio.access_key'='${minio_ak}',
+ 'minio.secret_key'='${minio_sk}'
+ );"""
// analyze
sql """use `${catalog_name}`.`${db_name}`"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org