You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/14 10:33:29 UTC
[doris] branch branch-2.0 updated (e546d62f58 -> 976a9e0197)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from e546d62f58 change branch-2.0 version to 2.0.0-dev
new 3ee83c77b0 [fix](merge-on-write) fix be core and delete unused pending publish info for async publish when tablet dropped (#21793)
new 26497c49a4 [Fix](MoW) Fix bug about caculating all committed rowsets delete bitmaps when do comapction (#21760)
new 9eedda343b [feature](hudi) support hudi time travel in external table (#21739)
new 1e9e6afdeb [Fix](parquet-reader) Fix parquet string column min max statistics issue which caused query result incorrectly. (#21675)
new 4d595d548f [Agg](exec) support aggregation_node limit short circuit (#21767)
new 9b61fcbf22 [FIX](map) fix map key-column nullable for arrow serde #21762
new c3c87ab3db [Improvement](multi catalog)Cache file system to improve list remote files performance (#21700)
new 580ddd8c1d [feature](table-value-functions)add catalogs table-value-function (#21790)
new 0456342faa [improve](nereids)inner join estimation: assume children output at least one tuple #21792
new 46ab7587e2 [imporve](udaf) refactor java-udaf executor by using for loop (#21713)
new 466b7b854d [fix](Nereids) use groupExpr's children to make logicalPlan (#21794)
new d7960b8053 [enhance](Nereids) Pushdown Project Through OuterJoin. (#21730)
new 5170c50ca9 [enhancement](multi-table) enable mullti table routine load on pipeline engine (#21729)
new 26f5bea217 [feature](nereids) adjust min/max of column stats for cast function (#21772)
new 7e0c9f551e [Enhancement] (binlog) TBinlog and BinlogManager V2 (#21674)
new dcbdb9cb1e [Feature](Nereids) support udf for Nereids (#18257)
new a51741cba6 [regression] add order by in test case for stable output (#21815)
new 976a9e0197 [imporve](bloomfilter) refactor runtime_filter_mgr with bloomfilter and fix bug in change_to_bloom_filter (#21783)
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/runtime_filter.cpp | 5 +-
be/src/io/file_factory.cpp | 27 +-
be/src/io/file_factory.h | 2 +-
be/src/io/fs/multi_table_pipe.cpp | 85 ++++--
be/src/io/fs/multi_table_pipe.h | 3 +
be/src/io/fs/stream_load_pipe.cpp | 7 +
be/src/io/fs/stream_load_pipe.h | 3 +
be/src/olap/compaction.cpp | 46 +++-
be/src/olap/compaction.h | 3 +
be/src/olap/olap_server.cpp | 6 +-
be/src/olap/storage_engine.cpp | 27 ++
be/src/olap/storage_engine.h | 2 +
be/src/pipeline/pipeline_fragment_context.cpp | 6 +-
be/src/runtime/runtime_filter_mgr.cpp | 38 +--
be/src/runtime/runtime_filter_mgr.h | 13 +-
be/src/runtime/runtime_state.cpp | 6 +-
be/src/runtime/runtime_state.h | 9 +-
.../aggregate_function_java_udaf.h | 119 ++++----
.../vec/data_types/serde/data_type_map_serde.cpp | 15 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 3 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 3 +-
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 14 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 15 +-
be/src/vec/exec/scan/vmeta_scanner.cpp | 18 ++
be/src/vec/exec/scan/vmeta_scanner.h | 3 +-
be/src/vec/exec/vaggregation_node.cpp | 4 +
be/src/vec/exec/vaggregation_node.h | 5 +
.../serde/data_type_serde_arrow_test.cpp | 84 +++++-
docs/en/docs/lakehouse/multi-catalog/hudi.md | 22 +-
.../sql-functions/table-functions/catalogs.md | 91 +++++++
docs/sidebars.json | 3 +-
docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md | 22 +-
.../sql-functions/table-functions/catalogs.md | 92 +++++++
fe/be-java-extensions/hudi-scanner/pom.xml | 3 +-
.../org/apache/doris/hudi/BaseSplitReader.scala | 8 +-
fe/be-java-extensions/java-udf/pom.xml | 6 +
.../java/org/apache/doris/udf/BaseExecutor.java | 181 +++++++++++++
.../java/org/apache/doris/udf/UdafExecutor.java | 89 +++++-
.../main/java/org/apache/doris/udf/UdfConvert.java | 262 +++++++++---------
.../java/org/apache/doris/udf/UdfExecutor.java | 165 +-----------
.../main/java/org/apache/doris/common/Config.java | 4 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 7 +-
.../java/org/apache/doris/analysis/SlotRef.java | 5 +
.../java/org/apache/doris/analysis/TableRef.java | 32 ++-
.../org/apache/doris/analysis/TableSnapshot.java | 6 +-
.../org/apache/doris/binlog/BinlogComparator.java | 24 ++
.../java/org/apache/doris/binlog/BinlogGcer.java | 5 +-
.../org/apache/doris/binlog/BinlogManager.java | 120 +++++----
.../org/apache/doris/binlog/BinlogTombstone.java | 8 +-
.../java/org/apache/doris/binlog/BinlogUtils.java | 53 ++++
.../java/org/apache/doris/binlog/DBBinlog.java | 298 +++++++++++++++------
.../java/org/apache/doris/binlog/TableBinlog.java | 167 ++++++++----
.../java/org/apache/doris/catalog/Database.java | 4 +
.../org/apache/doris/catalog/FunctionRegistry.java | 89 +++++-
.../org/apache/doris/catalog/FunctionUtil.java | 33 +++
.../apache/doris/catalog/GlobalFunctionMgr.java | 11 +
.../doris/catalog/HiveMetaStoreClientHelper.java | 13 +-
.../java/org/apache/doris/catalog/HudiUtils.java | 34 +++
.../java/org/apache/doris/common/ErrorCode.java | 2 +-
.../org/apache/doris/datasource/CatalogMgr.java | 2 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 29 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 24 +-
.../java/org/apache/doris/fs/FileSystemCache.java | 91 +++++++
.../org/apache/doris/fs/FileSystemFactory.java | 43 ++-
.../java/org/apache/doris/fs/FileSystemType.java | 25 ++
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +
.../doris/nereids/analyzer/UnboundFunction.java | 19 +-
.../glue/translator/ExpressionTranslator.java | 18 ++
.../hypergraph/receiver/PlanReceiver.java | 6 +-
.../java/org/apache/doris/nereids/memo/Memo.java | 11 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 11 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
...a => PushdownProjectThroughInnerOuterJoin.java} | 37 ++-
.../rules/expression/rules/FunctionBinder.java | 14 +-
.../doris/nereids/stats/ExpressionEstimation.java | 31 ++-
.../apache/doris/nereids/stats/JoinEstimation.java | 2 +-
.../nereids/trees/expressions/functions/Udf.java | 49 ++++
.../trees/expressions/functions/udf/AliasUdf.java | 141 ++++++++++
.../expressions/functions/udf/AliasUdfBuilder.java | 108 ++++++++
.../trees/expressions/functions/udf/JavaUdaf.java | 201 ++++++++++++++
.../expressions/functions/udf/JavaUdafBuilder.java | 72 +++++
.../trees/expressions/functions/udf/JavaUdf.java | 167 ++++++++++++
.../expressions/functions/udf/JavaUdfBuilder.java | 82 ++++++
.../expressions/functions/udf/UdfBuilder.java | 30 +++
.../visitor/AggregateFunctionVisitor.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../trees/plans/commands/ExplainCommand.java | 1 +
.../doris/planner/external/FileQueryScanNode.java | 7 +-
.../doris/planner/external/HiveScanNode.java | 2 +-
.../planner/external/TablePartitionValues.java | 255 ++++++++++++++++++
.../hudi/HudiCachedPartitionProcessor.java | 131 +++++++++
.../planner/external/hudi/HudiPartitionMgr.java | 86 ++++++
.../external/hudi/HudiPartitionProcessor.java | 124 +++++++++
.../doris/planner/external/hudi/HudiScanNode.java | 84 +++++-
.../apache/doris/statistics/ColumnStatistic.java | 4 +-
.../doris/statistics/util/StatisticsUtil.java | 3 +-
.../tablefunction/CatalogsTableValuedFunction.java | 88 ++++++
.../doris/tablefunction/MetadataGenerator.java | 37 +++
.../tablefunction/MetadataTableValuedFunction.java | 2 +
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
.../apache/doris/catalog/CreateFunctionTest.java | 7 +-
.../rules/analysis/FunctionRegistryTest.java | 2 +-
... PushdownProjectThroughInnerOuterJoinTest.java} | 10 +-
.../doris/nereids/trees/expressions/UdfTest.java | 174 ++++++++++++
gensrc/thrift/FrontendService.thrift | 5 +-
gensrc/thrift/Types.thrift | 3 +-
.../table_valued_function/test_catalogs_tvf.out | 10 +
.../hive/test_multi_langs.out | 148 ++++++++++
.../data/index_p0/test_ngram_bloomfilter_index.out | 8 +-
.../javaudf}/collect/test_javaudf_groupcount.out | 0
.../javaudf}/collect/test_javaudf_murmurhash3.out | 0
.../javaudf}/collect/test_javaudf_sessionize.out | 0
.../javaudf}/date/test_javaudf_adddays.out | 0
.../javaudf}/date/test_javaudf_addisoperioud.out | 0
.../javaudf}/date/test_javaudf_daydiff.out | 0
.../json/test_javaudf_convertfromcamelcase.out | 0
.../json/test_javaudf_converttocamelcase.out | 0
.../javaudf}/sanity/test_javaudf_assertequal.out | 0
.../sanity/test_javaudf_assertlessthan.out | 0
.../javaudf}/sanity/test_javaudf_assertudf.out | 0
.../javaudf}/sketch/test_javaudf_md5.out | 0
.../javaudf}/test_javaudaf_my_date_datetime.out | 0
.../test_javaudaf_mygroupconcat_string.out | 0
.../javaudf/test_javaudaf_mysum_array.out | 87 ++++++
.../javaudf}/test_javaudaf_mysum_decimal.out | 0
.../javaudf}/test_javaudaf_mysum_double.out | 0
.../javaudf}/test_javaudaf_mysum_float_double.out | 0
.../nereids_p0/javaudf/test_javaudaf_mysum_int.out | 33 +++
.../javaudf}/test_javaudaf_null_test.out | 0
.../data/nereids_p0/javaudf/test_javaudf_array.out | 133 +++++++++
.../javaudf}/test_javaudf_boolean.out | 0
.../javaudf}/test_javaudf_case.out | 0
.../javaudf}/test_javaudf_date.out | 0
.../javaudf}/test_javaudf_decimal.out | 0
.../javaudf}/test_javaudf_float.out | 0
.../javaudf}/test_javaudf_int.out | 0
.../javaudf}/test_javaudf_null.out | 0
.../javaudf}/test_javaudf_string.out | 0
.../table_valued_function/test_catalogs_tvf.groovy | 71 +++++
.../hive/test_multi_langs.groovy | 61 +++++
.../index_p0/test_ngram_bloomfilter_index.groovy | 26 +-
.../javaudf/collect/test_javaudf_groupcount.groovy | 61 +++++
.../collect/test_javaudf_murmurhash3.groovy | 62 +++++
.../javaudf/collect/test_javaudf_sessionize.groovy | 65 +++++
.../javaudf/date/test_javaudf_adddays.groovy | 62 +++++
.../javaudf/date/test_javaudf_addisoperioud.groovy | 62 +++++
.../javaudf/date/test_javaudf_daydiff.groovy | 63 +++++
.../json/test_javaudf_convertfromcamelcase.groovy | 62 +++++
.../json/test_javaudf_converttocamelcase.groovy | 62 +++++
.../javaudf/sanity/test_javaudf_assertequal.groovy | 64 +++++
.../sanity/test_javaudf_assertlessthan.groovy | 64 +++++
.../javaudf/sanity/test_javaudf_assertudf.groovy | 54 ++++
.../javaudf/sketch/test_javaudf_md5.groovy | 62 +++++
.../nereids_p0/javaudf/test_alias_function.groovy | 78 ++++++
.../javaudf/test_javaudaf_my_date_datetime.groovy | 127 +++++++++
.../test_javaudaf_mygroupconcat_string.groovy | 78 ++++++
.../javaudf/test_javaudaf_mysum_array.groovy | 127 +++++++++
.../javaudf/test_javaudaf_mysum_decimal.groovy | 74 +++++
.../test_javaudaf_mysum_float_double.groovy | 94 +++++++
.../javaudf/test_javaudaf_mysum_int.groovy | 84 ++++++
.../javaudf/test_javaudaf_null_test.groovy | 81 ++++++
.../nereids_p0/javaudf/test_javaudf_array.groovy | 133 +++++++++
.../nereids_p0/javaudf/test_javaudf_boolean.groovy | 75 ++++++
.../nereids_p0/javaudf/test_javaudf_case.groovy | 73 +++++
.../nereids_p0/javaudf/test_javaudf_date.groovy | 211 +++++++++++++++
.../nereids_p0/javaudf/test_javaudf_decimal.groovy | 86 ++++++
.../nereids_p0/javaudf/test_javaudf_float.groovy | 93 +++++++
.../nereids_p0/javaudf/test_javaudf_int.groovy | 127 +++++++++
.../nereids_p0/javaudf/test_javaudf_null.groovy | 74 +++++
.../nereids_p0/javaudf/test_javaudf_string.groovy | 79 ++++++
172 files changed, 6951 insertions(+), 759 deletions(-)
create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md
create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogComparator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemType.java
rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/{PushdownProjectThroughInnerJoin.java => PushdownProjectThroughInnerOuterJoin.java} (79%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfBuilder.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/external/TablePartitionValues.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionMgr.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionProcessor.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java
rename fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/{PushdownProjectThroughInnerJoinTest.java => PushdownProjectThroughInnerOuterJoinTest.java} (94%)
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
create mode 100644 regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out
create mode 100644 regression-test/data/external_table_emr_p2/hive/test_multi_langs.out
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/collect/test_javaudf_groupcount.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/collect/test_javaudf_murmurhash3.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/collect/test_javaudf_sessionize.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/date/test_javaudf_adddays.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/date/test_javaudf_addisoperioud.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/date/test_javaudf_daydiff.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/json/test_javaudf_convertfromcamelcase.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/json/test_javaudf_converttocamelcase.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/sanity/test_javaudf_assertequal.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/sanity/test_javaudf_assertlessthan.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/sanity/test_javaudf_assertudf.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/sketch/test_javaudf_md5.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_my_date_datetime.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_mygroupconcat_string.out (100%)
create mode 100644 regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_array.out
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_mysum_decimal.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_mysum_double.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_mysum_float_double.out (100%)
create mode 100644 regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_int.out
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudaf_null_test.out (100%)
create mode 100644 regression-test/data/nereids_p0/javaudf/test_javaudf_array.out
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_boolean.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_case.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_date.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_decimal.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_float.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_int.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_null.out (100%)
copy regression-test/data/{javaudf_p0 => nereids_p0/javaudf}/test_javaudf_string.out (100%)
create mode 100644 regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy
create mode 100644 regression-test/suites/external_table_emr_p2/hive/test_multi_langs.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_groupcount.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_sessionize.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/date/test_javaudf_adddays.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/date/test_javaudf_addisoperioud.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/date/test_javaudf_daydiff.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertequal.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertudf.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/sketch/test_javaudf_md5.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_alias_function.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_my_date_datetime.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_array.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_decimal.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_float_double.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_int.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudaf_null_test.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_array.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_boolean.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_case.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_date.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_decimal.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_float.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_int.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_null.groovy
create mode 100644 regression-test/suites/nereids_p0/javaudf/test_javaudf_string.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 16/18: [Feature](Nereids) support udf for Nereids (#18257)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit dcbdb9cb1e093c872ac1647f0ce9be4d6caf91b7
Author: mch_ucchi <41...@users.noreply.github.com>
AuthorDate: Fri Jul 14 17:02:01 2023 +0800
[Feature](Nereids) support udf for Nereids (#18257)
Support alias function, Java UDF, Java UDAF for Nereids.
Implementation:
UDFs(alias function, Java UD(A)F) are saved in database object, we get it by FunctionDesc, which requires function name and arg types. So firstly we bind expressions of its children so that we can get the return type of args. Then we get the best selection.
Secondly:
For alias function:
The original function of the alias function is represented as original planner-style function, it's too hard to translate it to nereids-style expression hence we transfer it to the corresponding sql and parse it. Now we get the nereids-style function, and try to bind the function.
the bound function will also change the type by add cast node of its children to its expecting input types, so that if we travel a bound function more than one times, the cast node will be different. To solve the problem, we add a flag isAnalyzedFunction. it's set false by default and will be set true when return from the visitor function. If the flag is true, it will return immediately in visitor function.
Now we can ensure that the bound functions in children will be the same though we travel it more than one time. we can replace the alias function to its original function and bind the unbound functions.
For JavaUDF and JavaUDAF
JavaUDF and JavaUDAF can be recognized as a catalog function and hard to be entirely translated to Nereids-style function, we create a nereids expression object JavaUdf and JavaUdaf to wrap it.
All in all, now Nereids support UDFs and nesting them.
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 6 +-
.../java/org/apache/doris/analysis/SlotRef.java | 5 +
.../java/org/apache/doris/catalog/Database.java | 4 +
.../org/apache/doris/catalog/FunctionRegistry.java | 89 ++++++-
.../org/apache/doris/catalog/FunctionUtil.java | 33 +++
.../apache/doris/catalog/GlobalFunctionMgr.java | 11 +
.../doris/nereids/analyzer/UnboundFunction.java | 19 +-
.../glue/translator/ExpressionTranslator.java | 18 ++
.../doris/nereids/parser/LogicalPlanBuilder.java | 8 +-
.../rules/expression/rules/FunctionBinder.java | 14 +-
.../nereids/trees/expressions/functions/Udf.java | 49 ++++
.../trees/expressions/functions/udf/AliasUdf.java | 141 +++++++++++
.../expressions/functions/udf/AliasUdfBuilder.java | 108 ++++++++
.../trees/expressions/functions/udf/JavaUdaf.java | 201 +++++++++++++++
.../expressions/functions/udf/JavaUdafBuilder.java | 72 ++++++
.../trees/expressions/functions/udf/JavaUdf.java | 167 +++++++++++++
.../expressions/functions/udf/JavaUdfBuilder.java | 82 ++++++
.../expressions/functions/udf/UdfBuilder.java | 30 +++
.../visitor/AggregateFunctionVisitor.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../apache/doris/catalog/CreateFunctionTest.java | 7 +-
.../rules/analysis/FunctionRegistryTest.java | 2 +-
.../doris/nereids/trees/expressions/UdfTest.java | 174 +++++++++++++
.../javaudf/collect/test_javaudf_groupcount.out | 6 +
.../javaudf/collect/test_javaudf_murmurhash3.out | 6 +
.../javaudf/collect/test_javaudf_sessionize.out | 5 +
.../javaudf/date/test_javaudf_adddays.out | 6 +
.../javaudf/date/test_javaudf_addisoperioud.out | 6 +
.../javaudf/date/test_javaudf_daydiff.out | 5 +
.../json/test_javaudf_convertfromcamelcase.out | 5 +
.../json/test_javaudf_converttocamelcase.out | 5 +
.../javaudf/sanity/test_javaudf_assertequal.out | 4 +
.../javaudf/sanity/test_javaudf_assertlessthan.out | 5 +
.../javaudf/sanity/test_javaudf_assertudf.out | 4 +
.../nereids_p0/javaudf/sketch/test_javaudf_md5.out | 5 +
.../javaudf/test_javaudaf_my_date_datetime.out | 36 +++
.../javaudf/test_javaudaf_mygroupconcat_string.out | 18 ++
.../javaudf/test_javaudaf_mysum_array.out | 87 +++++++
.../javaudf/test_javaudaf_mysum_decimal.out | 14 ++
.../javaudf/test_javaudaf_mysum_double.out | 21 ++
.../javaudf/test_javaudaf_mysum_float_double.out | 30 +++
.../nereids_p0/javaudf/test_javaudaf_mysum_int.out | 33 +++
.../nereids_p0/javaudf/test_javaudaf_null_test.out | 24 ++
.../data/nereids_p0/javaudf/test_javaudf_array.out | 133 ++++++++++
.../nereids_p0/javaudf/test_javaudf_boolean.out | 28 +++
.../data/nereids_p0/javaudf/test_javaudf_case.out | 17 ++
.../data/nereids_p0/javaudf/test_javaudf_date.out | 276 +++++++++++++++++++++
.../nereids_p0/javaudf/test_javaudf_decimal.out | 26 ++
.../data/nereids_p0/javaudf/test_javaudf_float.out | 40 +++
.../data/nereids_p0/javaudf/test_javaudf_int.out | 88 +++++++
.../data/nereids_p0/javaudf/test_javaudf_null.out | 26 ++
.../nereids_p0/javaudf/test_javaudf_string.out | 45 ++++
.../javaudf/collect/test_javaudf_groupcount.groovy | 61 +++++
.../collect/test_javaudf_murmurhash3.groovy | 62 +++++
.../javaudf/collect/test_javaudf_sessionize.groovy | 65 +++++
.../javaudf/date/test_javaudf_adddays.groovy | 62 +++++
.../javaudf/date/test_javaudf_addisoperioud.groovy | 62 +++++
.../javaudf/date/test_javaudf_daydiff.groovy | 63 +++++
.../json/test_javaudf_convertfromcamelcase.groovy | 62 +++++
.../json/test_javaudf_converttocamelcase.groovy | 62 +++++
.../javaudf/sanity/test_javaudf_assertequal.groovy | 64 +++++
.../sanity/test_javaudf_assertlessthan.groovy | 64 +++++
.../javaudf/sanity/test_javaudf_assertudf.groovy | 54 ++++
.../javaudf/sketch/test_javaudf_md5.groovy | 62 +++++
.../nereids_p0/javaudf/test_alias_function.groovy | 78 ++++++
.../javaudf/test_javaudaf_my_date_datetime.groovy | 127 ++++++++++
.../test_javaudaf_mygroupconcat_string.groovy | 78 ++++++
.../javaudf/test_javaudaf_mysum_array.groovy | 127 ++++++++++
.../javaudf/test_javaudaf_mysum_decimal.groovy | 74 ++++++
.../test_javaudaf_mysum_float_double.groovy | 94 +++++++
.../javaudf/test_javaudaf_mysum_int.groovy | 84 +++++++
.../javaudf/test_javaudaf_null_test.groovy | 81 ++++++
.../nereids_p0/javaudf/test_javaudf_array.groovy | 133 ++++++++++
.../nereids_p0/javaudf/test_javaudf_boolean.groovy | 75 ++++++
.../nereids_p0/javaudf/test_javaudf_case.groovy | 73 ++++++
.../nereids_p0/javaudf/test_javaudf_date.groovy | 211 ++++++++++++++++
.../nereids_p0/javaudf/test_javaudf_decimal.groovy | 86 +++++++
.../nereids_p0/javaudf/test_javaudf_float.groovy | 93 +++++++
.../nereids_p0/javaudf/test_javaudf_int.groovy | 127 ++++++++++
.../nereids_p0/javaudf/test_javaudf_null.groovy | 74 ++++++
.../nereids_p0/javaudf/test_javaudf_string.groovy | 79 ++++++
81 files changed, 4674 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c32a09f3f6..b688955a10 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -377,7 +377,11 @@ primaryExpression
source=valueExpression RIGHT_PAREN #extract
;
-functionIdentifier
+functionIdentifier
+ : (dbName=identifier DOT)? functionNameIdentifier
+ ;
+
+functionNameIdentifier
: identifier
| LEFT | RIGHT
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
index 8040510840..de0cb93b60 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
@@ -251,6 +251,11 @@ public class SlotRef extends Expr {
} else {
return label;
}
+ } else if (desc == null) {
+ // virtual slot of an alias function
+ // when we try to translate an alias function to Nereids style, the desc in the place holding slotRef
+ // is null, and we just need the name of col.
+ return col;
} else if (desc.getSourceExprs() != null) {
if (!disableTableName && (ToSqlContext.get() == null || ToSqlContext.get().isNeedSlotRefId())) {
if (desc.getId().asInt() != 1) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 009df4f382..90f0184287 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -717,12 +717,14 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
function.checkWritable();
if (FunctionUtil.addFunctionImpl(function, ifNotExists, false, name2Function)) {
Env.getCurrentEnv().getEditLog().logAddFunction(function);
+ FunctionUtil.translateToNereids(this.getFullName(), function);
}
}
public synchronized void replayAddFunction(Function function) {
try {
FunctionUtil.addFunctionImpl(function, false, true, name2Function);
+ FunctionUtil.translateToNereids(this.getFullName(), function);
} catch (UserException e) {
throw new RuntimeException(e);
}
@@ -731,12 +733,14 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
public synchronized void dropFunction(FunctionSearchDesc function, boolean ifExists) throws UserException {
if (FunctionUtil.dropFunctionImpl(function, ifExists, name2Function)) {
Env.getCurrentEnv().getEditLog().logDropFunction(function);
+ FunctionUtil.dropFromNereids(this.getFullName(), function);
}
}
public synchronized void replayDropFunction(FunctionSearchDesc functionSearchDesc) {
try {
FunctionUtil.dropFunctionImpl(functionSearchDesc, false, name2Function);
+ FunctionUtil.dropFromNereids(this.getFullName(), functionSearchDesc);
} catch (UserException e) {
throw new RuntimeException(e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
index 3fdcdcde92..53c729fd62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
@@ -17,13 +17,20 @@
package org.apache.doris.catalog;
+import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.nereids.annotation.Developing;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.functions.AggStateFunctionBuilder;
import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
+import org.apache.doris.nereids.trees.expressions.functions.udf.UdfBuilder;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.commons.collections.CollectionUtils;
import java.util.List;
@@ -40,12 +47,18 @@ import javax.annotation.concurrent.ThreadSafe;
@Developing
@ThreadSafe
public class FunctionRegistry {
- private final Map<String, List<FunctionBuilder>> name2Builders;
+
+ // to record the global alias function and other udf.
+ private static final String GLOBAL_FUNCTION = "__GLOBAL_FUNCTION__";
+
+ private final Map<String, List<FunctionBuilder>> name2InternalBuiltinBuilders;
+ private final Map<String, Map<String, List<FunctionBuilder>>> name2UdfBuilders;
public FunctionRegistry() {
- name2Builders = new ConcurrentHashMap<>();
- registerBuiltinFunctions(name2Builders);
- afterRegisterBuiltinFunctions(name2Builders);
+ name2InternalBuiltinBuilders = new ConcurrentHashMap<>();
+ name2UdfBuilders = new ConcurrentHashMap<>();
+ registerBuiltinFunctions(name2InternalBuiltinBuilders);
+ afterRegisterBuiltinFunctions(name2InternalBuiltinBuilders);
}
// this function is used to test.
@@ -54,20 +67,23 @@ public class FunctionRegistry {
@VisibleForTesting
protected void afterRegisterBuiltinFunctions(Map<String, List<FunctionBuilder>> name2Builders) {}
+ public FunctionBuilder findFunctionBuilder(String name, List<?> arguments) {
+ return findFunctionBuilder(null, name, arguments);
+ }
public FunctionBuilder findFunctionBuilder(String name, Object argument) {
- return findFunctionBuilder(name, ImmutableList.of(argument));
+ return findFunctionBuilder(null, name, ImmutableList.of(argument));
}
- // currently we only find function by name and arity
- public FunctionBuilder findFunctionBuilder(String name, List<?> arguments) {
+ // currently we only find function by name and arity and args' types.
+ public FunctionBuilder findFunctionBuilder(String dbName, String name, List<?> arguments) {
int arity = arguments.size();
- List<FunctionBuilder> functionBuilders = name2Builders.get(name.toLowerCase());
+ List<FunctionBuilder> functionBuilders = name2InternalBuiltinBuilders.get(name.toLowerCase());
if (CollectionUtils.isEmpty(functionBuilders) && AggStateFunctionBuilder.isAggStateCombinator(name)) {
String nestedName = AggStateFunctionBuilder.getNestedName(name);
String combinatorSuffix = AggStateFunctionBuilder.getCombinatorSuffix(name);
- functionBuilders = name2Builders.get(nestedName.toLowerCase());
+ functionBuilders = name2InternalBuiltinBuilders.get(nestedName.toLowerCase());
if (functionBuilders != null) {
functionBuilders = functionBuilders.stream().map(builder -> {
@@ -76,7 +92,10 @@ public class FunctionRegistry {
}
}
if (functionBuilders == null || functionBuilders.isEmpty()) {
- throw new AnalysisException("Can not found function '" + name + "'");
+ functionBuilders = findUdfBuilder(dbName, name);
+ if (functionBuilders == null || functionBuilders.isEmpty()) {
+ throw new AnalysisException("Can not found function '" + name + "'");
+ }
}
// check the arity and type
@@ -97,6 +116,33 @@ public class FunctionRegistry {
return candidateBuilders.get(0);
}
+ /**
+ * public for test.
+ */
+ public List<FunctionBuilder> findUdfBuilder(String dbName, String name) {
+ List<String> scopes = ImmutableList.of(GLOBAL_FUNCTION);
+ if (ConnectContext.get() != null) {
+ dbName = ClusterNamespace.getFullName(ConnectContext.get().getClusterName(),
+ dbName == null ? ConnectContext.get().getDatabase() : dbName);
+ if (dbName == null) {
+ scopes = ImmutableList.of(GLOBAL_FUNCTION);
+ } else {
+ scopes = ImmutableList.of(dbName, GLOBAL_FUNCTION);
+ }
+ }
+
+ synchronized (name2UdfBuilders) {
+ for (String scope : scopes) {
+ List<FunctionBuilder> candidate = name2UdfBuilders.getOrDefault(scope, ImmutableMap.of())
+ .get(name.toLowerCase());
+ if (candidate != null && !candidate.isEmpty()) {
+ return candidate;
+ }
+ }
+ }
+ return ImmutableList.of();
+ }
+
private void registerBuiltinFunctions(Map<String, List<FunctionBuilder>> name2Builders) {
FunctionHelper.addFunctions(name2Builders, BuiltinScalarFunctions.INSTANCE.scalarFunctions);
FunctionHelper.addFunctions(name2Builders, BuiltinAggregateFunctions.INSTANCE.aggregateFunctions);
@@ -110,4 +156,27 @@ public class FunctionRegistry {
.map(builder -> name + builder.toString())
.collect(Collectors.joining(", ", "[", "]"));
}
+
+
+ public void addUdf(String dbName, String name, UdfBuilder builder) {
+ if (dbName == null) {
+ dbName = GLOBAL_FUNCTION;
+ }
+ synchronized (name2UdfBuilders) {
+ Map<String, List<FunctionBuilder>> builders = name2UdfBuilders
+ .computeIfAbsent(dbName, k -> Maps.newHashMap());
+ builders.computeIfAbsent(name, k -> Lists.newArrayList()).add(builder);
+ }
+ }
+
+ public void dropUdf(String dbName, String name, List<DataType> argTypes) {
+ if (dbName == null) {
+ dbName = GLOBAL_FUNCTION;
+ }
+ synchronized (name2UdfBuilders) {
+ Map<String, List<FunctionBuilder>> builders = name2UdfBuilders.getOrDefault(dbName, ImmutableMap.of());
+ builders.getOrDefault(name, Lists.newArrayList()).removeIf(builder -> ((UdfBuilder) builder).getArgTypes()
+ .equals(argTypes));
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
index 641d30c1a5..403ae42b51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
@@ -25,6 +25,10 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
+import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
+import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
@@ -35,10 +39,12 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
/**
* function util.
@@ -217,4 +223,31 @@ public class FunctionUtil {
}
return dbName;
}
+
+ public static boolean translateToNereids(String dbName, Function function) {
+ try {
+ if (function instanceof AliasFunction) {
+ AliasUdf.translateToNereidsFunction(dbName, ((AliasFunction) function));
+ } else if (function instanceof ScalarFunction) {
+ JavaUdf.translateToNereidsFunction(dbName, ((ScalarFunction) function));
+ } else if (function instanceof AggregateFunction) {
+ JavaUdaf.translateToNereidsFunction(dbName, ((AggregateFunction) function));
+ }
+ } catch (Exception e) {
+ LOG.warn("Nereids create function {}:{} failed", dbName, function.getFunctionName().getFunction(), e);
+ }
+ return true;
+ }
+
+ public static boolean dropFromNereids(String dbName, FunctionSearchDesc function) {
+ try {
+ String fnName = function.getName().getFunction();
+ List<DataType> argTypes = Arrays.stream(function.getArgTypes()).map(DataType::fromCatalogType)
+ .collect(Collectors.toList());
+ Env.getCurrentEnv().getFunctionRegistry().dropUdf(dbName, fnName, argTypes);
+ } catch (Exception e) {
+ LOG.warn("Nereids drop function {}:{} failed", dbName, function.getName(), e);
+ }
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
index a23306d93d..b62d0b2fe5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
@@ -22,6 +22,8 @@ import org.apache.doris.common.UserException;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
@@ -34,6 +36,7 @@ import java.util.concurrent.ConcurrentMap;
* Provides management of global functions such as add, drop and other operations
*/
public class GlobalFunctionMgr extends MetaObject {
+ private static final Logger LOG = LogManager.getLogger(GlobalFunctionMgr.class);
// user define function
private ConcurrentMap<String, ImmutableList<Function>> name2Function = Maps.newConcurrentMap();
@@ -62,6 +65,11 @@ public class GlobalFunctionMgr extends MetaObject {
function.checkWritable();
if (FunctionUtil.addFunctionImpl(function, ifNotExists, false, name2Function)) {
Env.getCurrentEnv().getEditLog().logAddGlobalFunction(function);
+ try {
+ FunctionUtil.translateToNereids(null, function);
+ } catch (Exception e) {
+ LOG.warn("Nereids add function failed", e);
+ }
}
}
@@ -70,6 +78,7 @@ public class GlobalFunctionMgr extends MetaObject {
try {
function.setGlobal(true);
FunctionUtil.addFunctionImpl(function, false, true, name2Function);
+ FunctionUtil.translateToNereids(null, function);
} catch (UserException e) {
throw new RuntimeException(e);
}
@@ -78,12 +87,14 @@ public class GlobalFunctionMgr extends MetaObject {
public synchronized void dropFunction(FunctionSearchDesc function, boolean ifExists) throws UserException {
if (FunctionUtil.dropFunctionImpl(function, ifExists, name2Function)) {
Env.getCurrentEnv().getEditLog().logDropGlobalFunction(function);
+ FunctionUtil.dropFromNereids(null, function);
}
}
public synchronized void replayDropFunction(FunctionSearchDesc functionSearchDesc) {
try {
FunctionUtil.dropFunctionImpl(functionSearchDesc, false, name2Function);
+ FunctionUtil.dropFromNereids(null, functionSearchDesc);
} catch (UserException e) {
throw new RuntimeException(e);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
index fa6e24722e..fa4b8a8ba4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFunction.java
@@ -33,16 +33,25 @@ import java.util.stream.Collectors;
* Expression for unbound function.
*/
public class UnboundFunction extends Function implements Unbound, PropagateNullable {
-
+ private final String dbName;
private final String name;
private final boolean isDistinct;
public UnboundFunction(String name, List<Expression> arguments) {
- this(name, false, arguments);
+ this(null, name, false, arguments);
+ }
+
+ public UnboundFunction(String dbName, String name, List<Expression> arguments) {
+ this(dbName, name, false, arguments);
}
public UnboundFunction(String name, boolean isDistinct, List<Expression> arguments) {
+ this(null, name, isDistinct, arguments);
+ }
+
+ public UnboundFunction(String dbName, String name, boolean isDistinct, List<Expression> arguments) {
super(arguments);
+ this.dbName = dbName;
this.name = Objects.requireNonNull(name, "name cannot be null");
this.isDistinct = isDistinct;
}
@@ -51,6 +60,10 @@ public class UnboundFunction extends Function implements Unbound, PropagateNulla
return name;
}
+ public String getDbName() {
+ return dbName;
+ }
+
public boolean isDistinct() {
return isDistinct;
}
@@ -80,7 +93,7 @@ public class UnboundFunction extends Function implements Unbound, PropagateNulla
@Override
public UnboundFunction withChildren(List<Expression> children) {
- return new UnboundFunction(name, isDistinct, children);
+ return new UnboundFunction(dbName, name, isDistinct, children);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index bdf34f43b7..bb2a044e55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -83,6 +83,8 @@ import org.apache.doris.nereids.trees.expressions.functions.combinator.StateComb
import org.apache.doris.nereids.trees.expressions.functions.combinator.UnionCombinator;
import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
import org.apache.doris.nereids.trees.expressions.functions.window.WindowFunction;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
@@ -517,6 +519,22 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
return new FunctionCallExpr(catalogFunction, new FunctionParams(function.isDistinct(), arguments));
}
+ @Override
+ public Expr visitJavaUdf(JavaUdf udf, PlanTranslatorContext context) {
+ FunctionParams exprs = new FunctionParams(udf.children().stream()
+ .map(expression -> expression.accept(this, context))
+ .collect(Collectors.toList()));
+ return new FunctionCallExpr(udf.getCatalogFunction(), exprs);
+ }
+
+ @Override
+ public Expr visitJavaUdaf(JavaUdaf udaf, PlanTranslatorContext context) {
+ FunctionParams exprs = new FunctionParams(udaf.isDistinct(), udaf.children().stream()
+ .map(expression -> expression.accept(this, context))
+ .collect(Collectors.toList()));
+ return new FunctionCallExpr(udaf.getCatalogFunction(), exprs);
+ }
+
// TODO: Supports for `distinct`
private Expr translateAggregateFunction(AggregateFunction function,
List<Expression> currentPhaseArguments, List<Expr> aggFnArguments,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index d4c044ad73..bb1181e008 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -1057,7 +1057,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public Expression visitFunctionCall(DorisParser.FunctionCallContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
- String functionName = ctx.functionIdentifier().getText();
+ String functionName = ctx.functionIdentifier().functionNameIdentifier().getText();
boolean isDistinct = ctx.DISTINCT() != null;
List<Expression> params = visit(ctx.expression(), Expression.class);
@@ -1085,7 +1085,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
throw new ParseException("'*' can only be used in conjunction with COUNT: " + functionName, ctx);
} else {
- UnboundFunction function = new UnboundFunction(functionName, isDistinct, params);
+ String dbName = null;
+ if (ctx.functionIdentifier().dbName != null) {
+ dbName = ctx.functionIdentifier().dbName.getText();
+ }
+ UnboundFunction function = new UnboundFunction(dbName, functionName, isDistinct, params);
if (ctx.windowSpec() != null) {
if (isDistinct) {
throw new ParseException("DISTINCT not allowed in analytic function: " + functionName, ctx);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
index 2ab66cc1b6..f3821e3cad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.FunctionRegistry;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.nereids.analyzer.UnboundFunction;
@@ -44,6 +45,7 @@ import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
+import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder;
import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.coercion.AbstractDataType;
@@ -86,7 +88,7 @@ public class FunctionBinder extends AbstractExpressionRewriteRule {
.map(e -> e.accept(this, context)).collect(Collectors.toList()));
// bind function
- FunctionRegistry functionRegistry = context.cascadesContext.getConnectContext().getFunctionRegistry();
+ FunctionRegistry functionRegistry = Env.getCurrentEnv().getFunctionRegistry();
String functionName = unboundFunction.getName();
List<Object> arguments = unboundFunction.isDistinct()
? ImmutableList.builder()
@@ -103,9 +105,15 @@ public class FunctionBinder extends AbstractExpressionRewriteRule {
.accept(this, context);
}
- FunctionBuilder builder = functionRegistry.findFunctionBuilder(functionName, arguments);
+ FunctionBuilder builder = functionRegistry.findFunctionBuilder(unboundFunction.getDbName(),
+ functionName, arguments);
BoundFunction boundFunction = builder.build(functionName, arguments);
- return TypeCoercionUtils.processBoundFunction(boundFunction);
+ if (builder instanceof AliasUdfBuilder) {
+ // we do type coercion in build function in alias function, so it's ok to return directly.
+ return boundFunction;
+ } else {
+ return TypeCoercionUtils.processBoundFunction(boundFunction);
+ }
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
new file mode 100644
index 0000000000..e1bdbf5add
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions;
+
+import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+import java.util.List;
+
+/**
+ * interface for udf
+ */
+public interface Udf extends ComputeNullable {
+ @Override
+ default boolean nullable() {
+ NullableMode mode = getNullableMode();
+ if (mode == NullableMode.ALWAYS_NOT_NULLABLE) {
+ return false;
+ } else if (mode == NullableMode.ALWAYS_NULLABLE) {
+ return true;
+ } else if (mode == NullableMode.DEPEND_ON_ARGUMENT) {
+ return children().stream().anyMatch(ExpressionTrait::nullable);
+ }
+ throw new AnalysisException("unsupported nullable mode for udf in Nereids");
+ }
+
+ Function getCatalogFunction() throws org.apache.doris.common.AnalysisException;
+
+ NullableMode getNullableMode();
+
+ List<Expression> children();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java
new file mode 100644
index 0000000000..0aa0bfdf85
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java
@@ -0,0 +1,141 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.catalog.AliasFunction;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.analyzer.UnboundFunction;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NullType;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * alias function
+ */
+public class AliasUdf extends ScalarFunction implements ExplicitlyCastableSignature {
+ private final UnboundFunction unboundFunction;
+ private final List<String> parameters;
+ private final List<DataType> argTypes;
+
+ /**
+ * constructor
+ */
+ public AliasUdf(String name, List<DataType> argTypes, UnboundFunction unboundFunction,
+ List<String> parameters, Expression... arguments) {
+ super(name, arguments);
+ this.argTypes = argTypes;
+ this.unboundFunction = unboundFunction;
+ this.parameters = parameters;
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return ImmutableList.of(Suppliers.memoize(() -> FunctionSignature
+ .of(NullType.INSTANCE, argTypes)).get());
+ }
+
+ public List<String> getParameters() {
+ return parameters;
+ }
+
+ public UnboundFunction getUnboundFunction() {
+ return unboundFunction;
+ }
+
+ public List<DataType> getArgTypes() {
+ return argTypes;
+ }
+
+ @Override
+ public boolean nullable() {
+ return false;
+ }
+
+ /**
+ * translate catalog alias function to nereids alias function
+ */
+ public static void translateToNereidsFunction(String dbName, AliasFunction function) {
+ String functionSql = function.getOriginFunction().toSql();
+ Expression parsedFunction = new NereidsParser().parseExpression(functionSql);
+
+ Map<String, SlotReference> replaceMap = Maps.newHashMap();
+ for (int i = 0; i < function.getNumArgs(); ++i) {
+ replaceMap.put(function.getParameters().get(i),
+ new SlotReference(
+ function.getParameters().get(i),
+ DataType.fromCatalogType(function.getArgs()[i])));
+ }
+
+ Expression slotBoundFunction = VirtualSlotReplacer.INSTANCE.replace(parsedFunction, replaceMap);
+
+ AliasUdf aliasUdf = new AliasUdf(
+ function.functionName(),
+ Arrays.stream(function.getArgs()).map(DataType::fromCatalogType).collect(Collectors.toList()),
+ ((UnboundFunction) slotBoundFunction),
+ function.getParameters());
+
+ AliasUdfBuilder builder = new AliasUdfBuilder(aliasUdf);
+ Env.getCurrentEnv().getFunctionRegistry().addUdf(dbName, aliasUdf.getName(), builder);
+ }
+
+ @Override
+ public int arity() {
+ return argTypes.size();
+ }
+
+ @Override
+ public Expression withChildren(List<Expression> children) {
+ return new AliasUdf(getName(), argTypes, unboundFunction, parameters,
+ children.toArray(new Expression[0]));
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitAliasUdf(this, context);
+ }
+
+ private static class VirtualSlotReplacer extends DefaultExpressionRewriter<Map<String, SlotReference>> {
+ public static final VirtualSlotReplacer INSTANCE = new VirtualSlotReplacer();
+
+ public Expression replace(Expression expression, Map<String, SlotReference> context) {
+ return expression.accept(this, context);
+ }
+
+ @Override
+ public Expression visitUnboundSlot(UnboundSlot slot, Map<String, SlotReference> context) {
+ return context.get(slot.getName());
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java
new file mode 100644
index 0000000000..18268aea97
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdfBuilder.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.common.util.ReflectionUtils;
+import org.apache.doris.nereids.rules.expression.rules.FunctionBinder;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * alias function builder
+ */
+public class AliasUdfBuilder extends UdfBuilder {
+ private final AliasUdf aliasUdf;
+
+ public AliasUdfBuilder(AliasUdf aliasUdf) {
+ this.aliasUdf = aliasUdf;
+ }
+
+ @Override
+ public List<DataType> getArgTypes() {
+ return aliasUdf.getArgTypes();
+ }
+
+ @Override
+ public boolean canApply(List<?> arguments) {
+ if (arguments.size() != aliasUdf.arity()) {
+ return false;
+ }
+ for (Object argument : arguments) {
+ if (!(argument instanceof Expression)) {
+ Optional<Class> primitiveType = ReflectionUtils.getPrimitiveType(argument.getClass());
+ if (!primitiveType.isPresent() || !Expression.class.isAssignableFrom(primitiveType.get())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BoundFunction build(String name, List<?> arguments) {
+ // use AliasFunction to process TypeCoercion
+ BoundFunction boundAliasFunction = ((BoundFunction) aliasUdf.withChildren(arguments.stream()
+ .map(Expression.class::cast).collect(Collectors.toList())));
+
+ Expression processedExpression = TypeCoercionUtils.processBoundFunction(boundAliasFunction);
+ List<Expression> inputs = processedExpression.getArguments();
+
+ Expression boundFunction = FunctionBinder.INSTANCE.rewrite(aliasUdf.getUnboundFunction(), null);
+
+ // replace the placeholder slot to the input expressions.
+ // adjust input, parameter and replaceMap to be corresponding.
+ Map<String, SlotReference> slots = ((Set<SlotReference>) boundFunction
+ .collect(SlotReference.class::isInstance))
+ .stream().collect(Collectors.toMap(SlotReference::getName, k -> k, (v1, v2) -> v2));
+
+ Map<SlotReference, Expression> replaceMap = Maps.newHashMap();
+ for (int i = 0; i < inputs.size(); ++i) {
+ String parameter = aliasUdf.getParameters().get(i);
+ Preconditions.checkArgument(slots.containsKey(parameter));
+ replaceMap.put(slots.get(parameter), inputs.get(i));
+ }
+
+ return ((BoundFunction) SlotReplacer.INSTANCE.replace(boundFunction, replaceMap));
+ }
+
+ private static class SlotReplacer extends DefaultExpressionRewriter<Map<SlotReference, Expression>> {
+ public static final SlotReplacer INSTANCE = new SlotReplacer();
+
+ public Expression replace(Expression expression, Map<SlotReference, Expression> context) {
+ return expression.accept(this, context);
+ }
+
+ @Override
+ public Expression visitSlotReference(SlotReference slot, Map<SlotReference, Expression> context) {
+ return context.get(slot);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java
new file mode 100644
index 0000000000..96be9dbcb0
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java
@@ -0,0 +1,201 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.analysis.FunctionName;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.util.URI;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Udf;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.AbstractDataType;
+import org.apache.doris.thrift.TFunctionBinaryType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Java UDAF for Nereids
+ */
+public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSignature, Udf {
+ private final String dbName;
+ private final TFunctionBinaryType binaryType;
+ private final FunctionSignature signature;
+ private final AbstractDataType intermediateType;
+ private final NullableMode nullableMode;
+ private final String objectFile;
+ private final String symbol;
+ private final String initFn;
+ private final String updateFn;
+ private final String mergeFn;
+ private final String serializeFn;
+ private final String finalizeFn;
+ private final String getValueFn;
+ private final String removeFn;
+ private final String checkSum;
+
+ /**
+ * Constructor of UDAF
+ */
+ public JavaUdaf(String name, String dbName, TFunctionBinaryType binaryType, FunctionSignature signature,
+ AbstractDataType intermediateType, NullableMode nullableMode,
+ String objectFile, String symbol,
+ String initFn, String updateFn, String mergeFn,
+ String serializeFn, String finalizeFn, String getValueFn, String removeFn,
+ boolean isDistinct, String checkSum, Expression... args) {
+ super(name, isDistinct, args);
+ this.dbName = dbName;
+ this.binaryType = binaryType;
+ this.signature = signature;
+ this.intermediateType = intermediateType == null ? signature.returnType : intermediateType;
+ this.nullableMode = nullableMode;
+ this.objectFile = objectFile;
+ this.symbol = symbol;
+ this.initFn = initFn;
+ this.updateFn = updateFn;
+ this.mergeFn = mergeFn;
+ this.serializeFn = serializeFn;
+ this.finalizeFn = finalizeFn;
+ this.getValueFn = getValueFn;
+ this.removeFn = removeFn;
+ this.checkSum = checkSum;
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return ImmutableList.of(signature);
+ }
+
+ @Override
+ public boolean hasVarArguments() {
+ return signature.hasVarArgs;
+ }
+
+ @Override
+ public int arity() {
+ return signature.argumentsTypes.size();
+ }
+
+ @Override
+ public NullableMode getNullableMode() {
+ return nullableMode;
+ }
+
+ /**
+ * withChildren.
+ */
+ @Override
+ public JavaUdaf withDistinctAndChildren(boolean isDistinct, List<Expression> children) {
+ Preconditions.checkArgument(children.size() == this.children.size());
+ return new JavaUdaf(getName(), dbName, binaryType, signature, intermediateType, nullableMode,
+ objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn,
+ isDistinct, checkSum, children.toArray(new Expression[0]));
+ }
+
+ /**
+ * translate catalog java udf to nereids java udf
+ */
+ public static void translateToNereidsFunction(String dbName, org.apache.doris.catalog.AggregateFunction aggregate) {
+ String fnName = aggregate.functionName();
+ DataType retType = DataType.fromCatalogType(aggregate.getReturnType());
+ List<DataType> argTypes = Arrays.stream(aggregate.getArgs())
+ .map(DataType::fromCatalogType)
+ .collect(Collectors.toList());
+
+ FunctionSignature.FuncSigBuilder sigBuilder = FunctionSignature.ret(retType);
+ FunctionSignature sig = aggregate.hasVarArgs()
+ ? sigBuilder.varArgs(argTypes.toArray(new DataType[0]))
+ : sigBuilder.args(argTypes.toArray(new DataType[0]));
+
+ VirtualSlotReference[] virtualSlots = argTypes.stream()
+ .map(type -> new VirtualSlotReference(type.toString(), type, Optional.empty(),
+ (shape) -> ImmutableList.of()))
+ .toArray(VirtualSlotReference[]::new);
+
+ DataType intermediateType = null;
+ if (aggregate.getIntermediateType() != null) {
+ intermediateType = DataType.fromCatalogType(aggregate.getIntermediateType());
+ }
+
+ JavaUdaf udaf = new JavaUdaf(fnName, dbName, aggregate.getBinaryType(), sig,
+ intermediateType,
+ aggregate.getNullableMode(),
+ aggregate.getLocation().getLocation(),
+ aggregate.getSymbolName(),
+ aggregate.getInitFnSymbol(),
+ aggregate.getUpdateFnSymbol(),
+ aggregate.getMergeFnSymbol(),
+ aggregate.getSerializeFnSymbol(),
+ aggregate.getFinalizeFnSymbol(),
+ aggregate.getGetValueFnSymbol(),
+ aggregate.getRemoveFnSymbol(),
+ false,
+ aggregate.getChecksum(),
+ virtualSlots);
+
+ JavaUdafBuilder builder = new JavaUdafBuilder(udaf);
+ Env.getCurrentEnv().getFunctionRegistry().addUdf(dbName, fnName, builder);
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitJavaUdaf(this, context);
+ }
+
+ @Override
+ public Function getCatalogFunction() {
+ try {
+ org.apache.doris.catalog.AggregateFunction expr = new org.apache.doris.catalog.AggregateFunction(
+ new FunctionName(dbName, getName()),
+ signature.argumentsTypes.stream().map(AbstractDataType::toCatalogDataType).toArray(Type[]::new),
+ signature.returnType.toCatalogDataType(),
+ signature.hasVarArgs,
+ intermediateType.toCatalogDataType(),
+ URI.create(objectFile),
+ initFn,
+ updateFn,
+ mergeFn,
+ serializeFn,
+ finalizeFn,
+ getValueFn,
+ removeFn
+ );
+ expr.setSymbolName(symbol);
+ expr.setBinaryType(binaryType);
+ expr.setNullableMode(nullableMode);
+ expr.setChecksum(checkSum);
+ return expr;
+ } catch (Exception e) {
+ throw new AnalysisException(e.getMessage(), e.getCause());
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java
new file mode 100644
index 0000000000..d0c0b067e9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdafBuilder.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.common.util.ReflectionUtils;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
+import org.apache.doris.nereids.types.DataType;
+
+import com.google.common.base.Suppliers;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * function builder for java udaf
+ */
+public class JavaUdafBuilder extends UdfBuilder {
+ private final JavaUdaf udf;
+ private final int arity;
+ private final boolean isVarArgs;
+
+ public JavaUdafBuilder(JavaUdaf udf) {
+ this.udf = udf;
+ this.isVarArgs = udf.hasVarArguments();
+ this.arity = udf.arity();
+ }
+
+ @Override
+ public List<DataType> getArgTypes() {
+ return Suppliers.memoize(() -> udf.getSignatures().get(0).argumentsTypes.stream()
+ .map(DataType.class::cast)
+ .collect(Collectors.toList())).get();
+ }
+
+ @Override
+ public boolean canApply(List<?> arguments) {
+ if ((isVarArgs && arity > arguments.size() + 1) || (!isVarArgs && arguments.size() != arity)) {
+ return false;
+ }
+ for (Object argument : arguments) {
+ if (!(argument instanceof Expression)) {
+ Optional<Class> primitiveType = ReflectionUtils.getPrimitiveType(argument.getClass());
+ if (!primitiveType.isPresent() || !Expression.class.isAssignableFrom(primitiveType.get())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BoundFunction build(String name, List<?> arguments) {
+ return udf.withChildren(arguments.stream().map(Expression.class::cast).collect(Collectors.toList()));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java
new file mode 100644
index 0000000000..431937af10
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.analysis.FunctionName;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.util.URI;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Udf;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.AbstractDataType;
+import org.apache.doris.thrift.TFunctionBinaryType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Java UDF for Nereids
+ */
+public class JavaUdf extends ScalarFunction implements ExplicitlyCastableSignature, Udf {
+ private final String dbName;
+ private final TFunctionBinaryType binaryType;
+ private final FunctionSignature signature;
+ private final NullableMode nullableMode;
+ private final String objectFile;
+ private final String symbol;
+ private final String prepareFn;
+ private final String closeFn;
+ private final String checkSum;
+
+ /**
+ * Constructor of UDF
+ */
+ public JavaUdf(String name, String dbName, TFunctionBinaryType binaryType, FunctionSignature signature,
+ NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn,
+ String checkSum, Expression... args) {
+ super(name, args);
+ this.dbName = dbName;
+ this.binaryType = binaryType;
+ this.signature = signature;
+ this.nullableMode = nullableMode;
+ this.objectFile = objectFile;
+ this.symbol = symbol;
+ this.prepareFn = prepareFn;
+ this.closeFn = closeFn;
+ this.checkSum = checkSum;
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return ImmutableList.of(signature);
+ }
+
+ @Override
+ public boolean hasVarArguments() {
+ return signature.hasVarArgs;
+ }
+
+ @Override
+ public int arity() {
+ return signature.argumentsTypes.size();
+ }
+
+ @Override
+ public NullableMode getNullableMode() {
+ return nullableMode;
+ }
+
+ /**
+ * withChildren.
+ */
+ @Override
+ public JavaUdf withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == this.children.size());
+ return new JavaUdf(getName(), dbName, binaryType, signature, nullableMode,
+ objectFile, symbol, prepareFn, closeFn, checkSum, children.toArray(new Expression[0]));
+ }
+
+ /**
+ * translate catalog java udf to nereids java udf
+ */
+ public static void translateToNereidsFunction(String dbName, org.apache.doris.catalog.ScalarFunction scalar) {
+ String fnName = scalar.functionName();
+ DataType retType = DataType.fromCatalogType(scalar.getReturnType());
+ List<DataType> argTypes = Arrays.stream(scalar.getArgs())
+ .map(DataType::fromCatalogType)
+ .collect(Collectors.toList());
+
+ FunctionSignature.FuncSigBuilder sigBuilder = FunctionSignature.ret(retType);
+ FunctionSignature sig = scalar.hasVarArgs()
+ ? sigBuilder.varArgs(argTypes.toArray(new DataType[0]))
+ : sigBuilder.args(argTypes.toArray(new DataType[0]));
+
+ VirtualSlotReference[] virtualSlots = argTypes.stream()
+ .map(type -> new VirtualSlotReference(type.toString(), type, Optional.empty(),
+ (shape) -> ImmutableList.of()))
+ .toArray(VirtualSlotReference[]::new);
+
+ JavaUdf udf = new JavaUdf(fnName, dbName, scalar.getBinaryType(), sig,
+ scalar.getNullableMode(),
+ scalar.getLocation().getLocation(),
+ scalar.getSymbolName(),
+ scalar.getPrepareFnSymbol(),
+ scalar.getCloseFnSymbol(),
+ scalar.getChecksum(),
+ virtualSlots);
+
+ JavaUdfBuilder builder = new JavaUdfBuilder(udf);
+ Env.getCurrentEnv().getFunctionRegistry().addUdf(dbName, fnName, builder);
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitJavaUdf(this, context);
+ }
+
+ @Override
+ public Function getCatalogFunction() {
+ try {
+ org.apache.doris.catalog.ScalarFunction expr = org.apache.doris.catalog.ScalarFunction.createUdf(
+ binaryType,
+ new FunctionName(dbName, getName()),
+ signature.argumentsTypes.stream().map(AbstractDataType::toCatalogDataType).toArray(Type[]::new),
+ signature.returnType.toCatalogDataType(),
+ signature.hasVarArgs,
+ URI.create(objectFile),
+ symbol,
+ prepareFn,
+ closeFn
+ );
+ expr.setNullableMode(nullableMode);
+ expr.setChecksum(checkSum);
+ return expr;
+ } catch (Exception e) {
+ throw new AnalysisException(e.getMessage(), e.getCause());
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java
new file mode 100644
index 0000000000..c11b07665a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdfBuilder.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.common.util.ReflectionUtils;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.AbstractDataType;
+import org.apache.doris.nereids.util.TypeCoercionUtils;
+
+import com.google.common.base.Suppliers;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * function builder for java udf
+ */
+public class JavaUdfBuilder extends UdfBuilder {
+ private final JavaUdf udf;
+ private final int arity;
+ private final boolean isVarArgs;
+
+ public JavaUdfBuilder(JavaUdf udf) {
+ this.udf = udf;
+ this.isVarArgs = udf.hasVarArguments();
+ this.arity = udf.arity();
+ }
+
+ @Override
+ public List<DataType> getArgTypes() {
+ return Suppliers.memoize(() -> udf.getSignatures().get(0).argumentsTypes.stream()
+ .map(DataType.class::cast)
+ .collect(Collectors.toList())).get();
+ }
+
+ @Override
+ public boolean canApply(List<?> arguments) {
+ if ((isVarArgs && arity > arguments.size() + 1) || (!isVarArgs && arguments.size() != arity)) {
+ return false;
+ }
+ for (Object argument : arguments) {
+ if (!(argument instanceof Expression)) {
+ Optional<Class> primitiveType = ReflectionUtils.getPrimitiveType(argument.getClass());
+ if (!primitiveType.isPresent() || !Expression.class.isAssignableFrom(primitiveType.get())) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BoundFunction build(String name, List<?> arguments) {
+ List<Expression> exprs = arguments.stream().map(Expression.class::cast).collect(Collectors.toList());
+ List<AbstractDataType> argTypes = udf.getSignatures().get(0).argumentsTypes;
+
+ List<Expression> processedExprs = Lists.newArrayList();
+ for (int i = 0; i < exprs.size(); ++i) {
+ processedExprs.add(TypeCoercionUtils.castIfNotSameType(exprs.get(i), ((DataType) argTypes.get(i))));
+ }
+ return udf.withChildren(processedExprs);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfBuilder.java
new file mode 100644
index 0000000000..892d9a4312
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/UdfBuilder.java
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.udf;
+
+import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
+import org.apache.doris.nereids.types.DataType;
+
+import java.util.List;
+
+/**
+ * abstract class for udf function builder
+ */
+public abstract class UdfBuilder extends FunctionBuilder {
+ public abstract List<DataType> getArgTypes();
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
index 139cda5f7d..f828a486ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/AggregateFunctionVisitor.java
@@ -67,6 +67,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.VarianceSamp;
import org.apache.doris.nereids.trees.expressions.functions.agg.WindowFunnel;
import org.apache.doris.nereids.trees.expressions.functions.combinator.MergeCombinator;
import org.apache.doris.nereids.trees.expressions.functions.combinator.UnionCombinator;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
/** AggregateFunctionVisitor. */
public interface AggregateFunctionVisitor<R, C> {
@@ -268,4 +269,8 @@ public interface AggregateFunctionVisitor<R, C> {
default R visitUnionCombinator(UnionCombinator combinator, C context) {
return visitAggregateFunction(combinator, context);
}
+
+ default R visitJavaUdaf(JavaUdaf javaUdaf, C context) {
+ return visitAggregateFunction(javaUdaf, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index 51e29fc3a9..5a09cda069 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -341,6 +341,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.YearWeek;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub;
+import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdf;
+import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf;
/** ScalarFunctionVisitor. */
public interface ScalarFunctionVisitor<R, C> {
@@ -1638,4 +1640,12 @@ public interface ScalarFunctionVisitor<R, C> {
default R visitStateCombinator(StateCombinator combinator, C context) {
return visitScalarFunction(combinator, context);
}
+
+ default R visitJavaUdf(JavaUdf javaUdf, C context) {
+ return visitScalarFunction(javaUdf, context);
+ }
+
+ default R visitAliasUdf(AliasUdf aliasUdf, C context) {
+ return visitScalarFunction(aliasUdf, context);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
index 77f56003cc..a3d6ac9152 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateFunctionTest.java
@@ -61,6 +61,7 @@ public class CreateFunctionTest {
FeConstants.runningUnitTest = true;
// create connect context
connectContext = UtFrameUtils.createDefaultCtx();
+ connectContext.getSessionVariable().setEnableNereidsPlanner(false);
}
@AfterClass
@@ -72,6 +73,7 @@ public class CreateFunctionTest {
@Test
public void test() throws Exception {
ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+ ctx.getSessionVariable().setEnableNereidsPlanner(false);
// create database db1
createDatabase(ctx, "create database db1;");
@@ -112,7 +114,7 @@ public class CreateFunctionTest {
queryStr = "select db1.id_masking(k1) from db1.tbl1";
Assert.assertTrue(
- dorisAssert.query(queryStr).explainQuery().contains("concat(left(`k1`, 3), '****', right(`k1`, 4))"));
+ dorisAssert.query(queryStr).explainQuery().contains("concat(left(k1, 3), '****', right(k1, 4))"));
// create alias function with cast
// cast any type to decimal with specific precision and scale
@@ -203,6 +205,7 @@ public class CreateFunctionTest {
@Test
public void testCreateGlobalFunction() throws Exception {
ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+ ctx.getSessionVariable().setEnableNereidsPlanner(false);
// 1. create database db2
createDatabase(ctx, "create database db2;");
@@ -232,7 +235,7 @@ public class CreateFunctionTest {
queryStr = "select id_masking(k1) from db2.tbl1";
Assert.assertTrue(
- dorisAssert.query(queryStr).explainQuery().contains("concat(left(`k1`, 3), '****', right(`k1`, 4))"));
+ dorisAssert.query(queryStr).explainQuery().contains("concat(left(k1, 3), '****', right(k1, 4))"));
// 4. create alias function with cast
// cast any type to decimal with specific precision and scale
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
index 02ddb20f64..c7e32bed81 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
@@ -111,7 +111,7 @@ public class FunctionRegistryTest implements MemoPatternMatchSupported {
}
@Test
- public void testAddFunction() {
+ public void testAddFunction() throws Exception {
FunctionRegistry functionRegistry = new FunctionRegistry() {
@Override
protected void afterRegisterBuiltinFunctions(Map<String, List<FunctionBuilder>> name2builders) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
new file mode 100644
index 0000000000..35cb8ba14d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateFormat;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DayOfMonth;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DayOfWeek;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Hour;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursSub;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Now;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+import org.apache.doris.nereids.types.DateTimeV2Type;
+import org.apache.doris.nereids.types.DateV2Type;
+import org.apache.doris.nereids.types.DoubleType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanPatternMatchSupported;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class UdfTest extends TestWithFeService implements PlanPatternMatchSupported {
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("test");
+ createDatabase("test_1");
+ }
+
+ @Override
+ protected void runBeforeEach() throws Exception {
+ connectContext.setDatabase("default_cluster:test");
+ }
+
+ @Test
+ public void testSimpleAliasFunction() throws Exception {
+ createFunction("create global alias function f(int) with parameter(n) as hours_add(now(3), n)");
+ createFunction("create alias function f(int) with parameter(n) as hours_sub(now(3), n)");
+
+ String sql = "select f(3)";
+ Expression expected = new HoursSub(new Now(new IntegerLiteral(3)), new IntegerLiteral(3));
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalOneRowRelation()
+ .when(relation -> relation.getProjects().get(0).child(0).equals(expected))
+ );
+
+ connectContext.setDatabase("default_cluster:test_1");
+ Expression expected1 = new HoursAdd(new Now(new IntegerLiteral(3)), new IntegerLiteral(3));
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalOneRowRelation()
+ .when(relation -> relation.getProjects().get(0).child(0).equals(expected1))
+ );
+
+ sql = "select test.f(3)";
+ Expression expected2 = new HoursSub(new Now(new IntegerLiteral(3)), new IntegerLiteral(3));
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalOneRowRelation()
+ .when(relation -> relation.getProjects().get(0).child(0).equals(expected2))
+ );
+ }
+
+ @Test
+ public void testNestedAliasFunction() throws Exception {
+ createFunction("create global alias function f1(int) with parameter(n) as hours_add(now(3), n)");
+ createFunction("create global alias function f2(int) with parameter(n) as dayofweek(days_add(f1(3), n))");
+ createFunction("create global alias function f3(date) with parameter(dt) as hours_sub(days_sub(dt, f2(3)), dayofmonth(f1(f2(4))))");
+
+ Assertions.assertEquals(1, Env.getCurrentEnv().getFunctionRegistry()
+ .findUdfBuilder(connectContext.getDatabase(), "f3").size());
+
+ String sql = "select f3(now(3))";
+ Expression expected = new HoursSub(
+ new DaysSub(
+ new Cast(new Now(new IntegerLiteral(3)), DateV2Type.INSTANCE),
+ new Cast(new DayOfWeek(new DaysAdd(
+ new HoursAdd(
+ new Now(new IntegerLiteral(3)),
+ new IntegerLiteral(3)
+ ),
+ new IntegerLiteral(3))
+ ), IntegerType.INSTANCE)),
+ new Cast(new DayOfMonth(new HoursAdd(
+ new Now(new IntegerLiteral(3)),
+ new Cast(new DayOfWeek(new DaysAdd(
+ new HoursAdd(
+ new Now(new IntegerLiteral(3)),
+ new IntegerLiteral(3)
+ ),
+ new IntegerLiteral(4))
+ ), IntegerType.INSTANCE)
+ )), IntegerType.INSTANCE)
+ );
+
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalOneRowRelation()
+ .when(relation -> relation.getProjects().size() == 1
+ && relation.getProjects().get(0).child(0).equals(expected))
+ );
+ }
+
+ @Test
+ public void testParameterUseMoreThanOneTime() throws Exception {
+ createFunction("CREATE ALIAS FUNCTION f7(DATETIMEV2(3), INT) with PARAMETER (datetime1, int1) as\n"
+ + " DATE_FORMAT(HOURS_ADD(\n"
+ + " date_trunc(datetime1, 'day'),\n"
+ + " add(multiply(floor(divide(HOUR(datetime1), divide(24, int1))), 1), 1)), '%Y%m%d:%H')");
+
+ String sql = "select f7('2023-05-20 12:23:45', 3)";
+
+ Expression expected = new DateFormat(
+ new HoursAdd(
+ new DateTrunc(
+ new Cast(new VarcharLiteral("2023-05-20 12:23:45"), DateTimeV2Type.SYSTEM_DEFAULT),
+ new VarcharLiteral("day")),
+ new Cast(new Add(
+ new Multiply(
+ new Floor(new Divide(
+ new Cast(
+ new Hour(new Cast(new VarcharLiteral("2023-05-20 12:23:45"), DateTimeV2Type.SYSTEM_DEFAULT)),
+ DoubleType.INSTANCE
+ ),
+ new Divide(
+ new Cast(new TinyIntLiteral(((byte) 24)), DoubleType.INSTANCE),
+ new Cast(new IntegerLiteral(((byte) 3)), DoubleType.INSTANCE)
+ ))
+ ),
+ new Cast(new TinyIntLiteral(((byte) 1)), DoubleType.INSTANCE)
+ ),
+ new Cast(new TinyIntLiteral(((byte) 1)), DoubleType.INSTANCE)
+ ), IntegerType.INSTANCE)
+ ),
+ new VarcharLiteral("%Y%m%d:%H")
+ );
+
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .matches(
+ logicalOneRowRelation()
+ .when(relation -> relation.getProjects().size() == 1
+ && relation.getProjects().get(0).child(0).equals(expected))
+ );
+ }
+}
diff --git a/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_groupcount.out b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_groupcount.out
new file mode 100644
index 0000000000..ae75543b64
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_groupcount.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+0
+0
+1
+
diff --git a/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.out b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.out
new file mode 100644
index 0000000000..21a89b50ef
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+-1435112961
+301691705
+301691705
+
diff --git a/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_sessionize.out b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_sessionize.out
new file mode 100644
index 0000000000..8ec0794fa1
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/collect/test_javaudf_sessionize.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+098f6bcd-4621-3373-8ade-4e832627b4f6
+098f6bcd-4621-3373-8ade-4e832627b4f6
+
diff --git a/regression-test/data/nereids_p0/javaudf/date/test_javaudf_adddays.out b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_adddays.out
new file mode 100644
index 0000000000..3a08a1fad2
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_adddays.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+20220101
+20220102
+20220301
+
diff --git a/regression-test/data/nereids_p0/javaudf/date/test_javaudf_addisoperioud.out b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_addisoperioud.out
new file mode 100644
index 0000000000..79dc1c6f89
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_addisoperioud.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+20211231
+20220101
+20220228
+
diff --git a/regression-test/data/nereids_p0/javaudf/date/test_javaudf_daydiff.out b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_daydiff.out
new file mode 100644
index 0000000000..86c88b834b
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/date/test_javaudf_daydiff.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+1
+1
+
diff --git a/regression-test/data/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.out b/regression-test/data/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.out
new file mode 100644
index 0000000000..0ec37b2092
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+get_id_from_table
+test_java_udf
+
diff --git a/regression-test/data/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.out b/regression-test/data/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.out
new file mode 100644
index 0000000000..10d167d2f7
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+getIdFromTable
+testJavaUdf
+
diff --git a/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertequal.out b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertequal.out
new file mode 100644
index 0000000000..3376296a3c
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertequal.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+23.34 == 23.34
+
diff --git a/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.out b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.out
new file mode 100644
index 0000000000..41cb52194e
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+0.123 < 0.124
+23.34 < 23.35
+
diff --git a/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertudf.out b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertudf.out
new file mode 100644
index 0000000000..7787cd21cd
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/sanity/test_javaudf_assertudf.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+OK
+
diff --git a/regression-test/data/nereids_p0/javaudf/sketch/test_javaudf_md5.out b/regression-test/data/nereids_p0/javaudf/sketch/test_javaudf_md5.out
new file mode 100644
index 0000000000..b326f3cbb9
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/sketch/test_javaudf_md5.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+202cb962ac59075b964b07152d234b70
+900150983cd24fb0d6963f7d28e17f72
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_my_date_datetime.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_my_date_datetime.out
new file mode 100644
index 0000000000..a888dd87e2
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_my_date_datetime.out
@@ -0,0 +1,36 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+
+-- !select1 --
+2022-01-09
+
+-- !select2 --
+1 2022-01-09
+
+-- !select3 --
+2022-01-09
+
+-- !select4 --
+1 2022-01-09
+
+-- !select5 --
+2022-01-05T03:11:11
+
+-- !select6 --
+1 2022-01-05T03:11:11
+
+-- !select5 --
+2022-01-05T03:11:11
+
+-- !select6 --
+1 2022-01-05T03:11:11
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.out
new file mode 100644
index 0000000000..d15cddf0de
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.out
@@ -0,0 +1,18 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 1 abcdefg1 poiuytre1abcdefg
+2 2 abcdefg2 poiuytre2abcdefg
+0 3 abcdefg3 poiuytre3abcdefg
+1 4 abcdefg4 poiuytre4abcdefg
+2 5 abcdefg5 poiuytre5abcdefg
+0 6 abcdefg6 poiuytre6abcdefg
+1 7 abcdefg7 poiuytre7abcdefg
+2 8 abcdefg8 poiuytre8abcdefg
+9 9 abcdefg9 poiuytre9abcdefg
+
+-- !select1 --
+0 abcdefg3-abcdefg6
+1 abcdefg1-abcdefg4-abcdefg7
+2 abcdefg2-abcdefg5-abcdefg8
+9 abcdefg9
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_array.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_array.out
new file mode 100644
index 0000000000..e33f21c6a6
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_array.out
@@ -0,0 +1,87 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 1 abcdefg1 poiuytre1abcdefg
+2 2 abcdefg2 poiuytre2abcdefg
+0 3 abcdefg3 poiuytre3abcdefg
+1 4 abcdefg4 poiuytre4abcdefg
+2 5 abcdefg5 poiuytre5abcdefg
+0 6 abcdefg6 poiuytre6abcdefg
+1 7 abcdefg7 poiuytre7abcdefg
+2 8 abcdefg8 poiuytre8abcdefg
+9 9 abcdefg9 poiuytre9abcdefg
+
+-- !select1 --
+18
+
+-- !select2 --
+0 0
+1 3
+2 6
+9 9
+
+-- !select3 --
+0 0 0
+1 3 3
+2 6 6
+9 9 9
+
+-- !select4 --
+0 0 0
+1 3 3
+2 6 6
+9 9 9
+
+-- !select5 --
+36
+
+-- !select6 --
+[0, 0, 1, 1, 1, 2, 2, 2, 9]
+
+-- !select6_1 --
+[0, 0]
+[1, 1, 1]
+[2, 2, 2]
+[9]
+
+-- !select7 --
+[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 9, 9]
+
+-- !select7_1 --
+[0, 0, 0, 0]
+[1, 1, 1, 1, 1, 1]
+[2, 2, 2, 2, 2, 2]
+[9, 9]
+
+-- !select_8 --
+['poiuytre1abcdefg', 'poiuytre2abcdefg', 'poiuytre3abcdefg', 'poiuytre4abcdefg', 'poiuytre5abcdefg', 'poiuytre6abcdefg', 'poiuytre7abcdefg', 'poiuytre8abcdefg', 'poiuytre9abcdefg']
+
+-- !select_9 --
+['poiuytre3abcdefg', 'poiuytre6abcdefg'] 0
+['poiuytre1abcdefg', 'poiuytre4abcdefg', 'poiuytre7abcdefg'] 1
+['poiuytre2abcdefg', 'poiuytre5abcdefg', 'poiuytre8abcdefg'] 2
+['poiuytre9abcdefg'] 9
+
+-- !select_10 --
+['0', '0', 'poiuytre3abcdefg', 'poiuytre6abcdefg'] 0
+['1', '1', '1', 'poiuytre1abcdefg', 'poiuytre4abcdefg', 'poiuytre7abcdefg'] 1
+['2', '2', '2', 'poiuytre2abcdefg', 'poiuytre5abcdefg', 'poiuytre8abcdefg'] 2
+['9', 'poiuytre9abcdefg'] 9
+
+-- !select_11 --
+[]
+
+-- !select_13 --
+poiuytre3abcdefg--poiuytre6abcdefg- 0
+poiuytre1abcdefg--poiuytre4abcdefg--poiuytre7abcdefg- 1
+poiuytre2abcdefg--poiuytre5abcdefg--poiuytre8abcdefg- 2
+poiuytre9abcdefg- 9
+
+-- !select_14 --
+poiuytre3abcdefg-0--poiuytre6abcdefg-0- 0
+poiuytre1abcdefg-1--poiuytre4abcdefg-1--poiuytre7abcdefg-1- 1
+poiuytre2abcdefg-2--poiuytre5abcdefg-2--poiuytre8abcdefg-2- 2
+poiuytre9abcdefg-9- 9
+
+-- !select_15 --
+
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_decimal.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_decimal.out
new file mode 100644
index 0000000000..da9b252664
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_decimal.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+111 11111.111110000 222222.333333300
+112 1234556.111110000 222222.333333300
+113 87654321.111110000 \N
+
+-- !select1 --
+88899988.333330000
+
+-- !select2 --
+111 11111.111110000
+112 1234556.111110000
+113 87654321.111110000
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_double.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_double.out
new file mode 100644
index 0000000000..cf5d17af0f
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_double.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 11.1 1.111 poiuytre1abcdefg
+2 22.2 2.222 poiuytre2abcdefg
+0 33.3 3.333 poiuytre3abcdefg
+1 44.4 4.444 poiuytre4abcdefg
+2 55.5 5.555 poiuytre5abcdefg
+0 66.6 6.666 poiuytre6abcdefg
+1 77.7 7.777 poiuytre7abcdefg
+2 88.8 8.888 poiuytre8abcdefg
+9 99.9 10.89 poiuytre9abcdefg
+
+-- !select --
+101.77200000000002
+
+-- !select --
+0 19.997999999999998
+1 26.664
+2 33.33
+9 21.78
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_float_double.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_float_double.out
new file mode 100644
index 0000000000..372334c7e5
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_float_double.out
@@ -0,0 +1,30 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 11.1 1.111 poiuytre1abcdefg
+2 22.2 2.222 poiuytre2abcdefg
+0 33.3 3.333 poiuytre3abcdefg
+1 44.4 4.444 poiuytre4abcdefg
+2 55.5 5.555 poiuytre5abcdefg
+0 66.6 6.666 poiuytre6abcdefg
+1 77.7 7.777 poiuytre7abcdefg
+2 88.8 8.888 poiuytre8abcdefg
+9 99.9 10.89 poiuytre9abcdefg
+
+-- !select1 --
+101.77199999999999
+
+-- !select2 --
+0 19.997999999999998
+1 26.664
+2 33.33
+9 21.78
+
+-- !select3 --
+499.5
+
+-- !select4 --
+0 99.899994
+1 133.2
+2 166.5
+9 99.9
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_int.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_int.out
new file mode 100644
index 0000000000..47c14ac114
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_mysum_int.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 1 abcdefg1 poiuytre1abcdefg
+2 2 abcdefg2 poiuytre2abcdefg
+0 3 abcdefg3 poiuytre3abcdefg
+1 4 abcdefg4 poiuytre4abcdefg
+2 5 abcdefg5 poiuytre5abcdefg
+0 6 abcdefg6 poiuytre6abcdefg
+1 7 abcdefg7 poiuytre7abcdefg
+2 8 abcdefg8 poiuytre8abcdefg
+9 9 abcdefg9 poiuytre9abcdefg
+
+-- !select1 --
+18
+
+-- !select2 --
+0 0
+1 3
+2 6
+9 9
+
+-- !select3 --
+0 0 0
+1 3 3
+2 6 6
+9 9 9
+
+-- !select4 --
+0 0 0
+1 3 3
+2 6 6
+9 9 9
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudaf_null_test.out b/regression-test/data/nereids_p0/javaudf/test_javaudaf_null_test.out
new file mode 100644
index 0000000000..a297c2761c
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudaf_null_test.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 1 abcdefg1 poiuytre1abcdefg
+2 2 abcdefg2 poiuytre2abcdefg
+0 3 abcdefg3 poiuytre3abcdefg
+1 4 abcdefg4 poiuytre4abcdefg
+2 5 abcdefg5 poiuytre5abcdefg
+0 6 abcdefg6 poiuytre6abcdefg
+1 7 abcdefg7 poiuytre7abcdefg
+2 8 abcdefg8 poiuytre8abcdefg
+9 9 abcdefg9 poiuytre9abcdefg
+
+-- !select1 --
+\N
+
+-- !select2 --
+0 \N
+1 \N
+2 \N
+9 \N
+
+-- !select3 --
+\N
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_array.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_array.out
new file mode 100644
index 0000000000..ad5bda0b92
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_array.out
@@ -0,0 +1,133 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 2 2022-01-01 2022-01-01T11:11:11 a1b
+2 4 2022-01-01 2022-01-01T11:11:11 a2b
+3 6 2022-01-01 2022-01-01T11:11:11 a3b
+4 8 2022-01-01 2022-01-01T11:11:11 a4b
+5 10 2022-01-01 2022-01-01T11:11:11 a5b
+6 12 2022-01-01 2022-01-01T11:11:11 a6b
+7 14 2022-01-01 2022-01-01T11:11:11 a7b
+8 16 2022-01-01 2022-01-01T11:11:11 a8b
+9 18 2022-01-01 2022-01-01T11:11:11 a9b
+10 20 2022-06-06 2022-01-01T12:12:12 a10b
+
+-- !select_1 --
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+
+-- !select_2 --
+\N
+
+-- !select_3 --
+[1] 2
+[2] 4
+[3] 6
+[4] 8
+[5] 10
+[6] 12
+[7] 14
+[8] 16
+[9] 18
+[10] 20
+
+-- !select_4 --
+[2] 2
+[4] 4
+[6] 6
+[8] 8
+[10] 10
+[12] 12
+[14] 14
+[16] 16
+[18] 18
+[20] 20
+
+-- !select_5 --
+\N
+
+-- !select_6 --
+['a1b'] 2
+['a2b'] 4
+['a3b'] 6
+['a4b'] 8
+['a5b'] 10
+['a6b'] 12
+['a7b'] 14
+['a8b'] 16
+['a9b'] 18
+['a10b'] 20
+
+-- !select_7 --
+['a1b1'] 2
+['a2b2'] 4
+['a3b3'] 6
+['a4b4'] 8
+['a5b5'] 10
+['a6b6'] 12
+['a7b7'] 14
+['a8b8'] 16
+['a9b9'] 18
+['a10b10'] 20
+
+-- !select_8 --
+\N
+
+-- !select_9 --
+a1b 2
+a2b 4
+a3b 6
+a4b 8
+a5b 10
+a6b 12
+a7b 14
+a8b 16
+a9b 18
+a10b 20
+
+-- !select_10 --
+a1b1 2
+a2b2 4
+a3b3 6
+a4b4 8
+a5b5 10
+a6b6 12
+a7b7 14
+a8b8 16
+a9b9 18
+a10b10 20
+
+-- !select_11 --
+\N
+
+-- !select_12 --
+[2022-01-01 11:11:11] 2
+[2022-01-01 11:11:11] 4
+[2022-01-01 11:11:11] 6
+[2022-01-01 11:11:11] 8
+[2022-01-01 11:11:11] 10
+[2022-01-01 11:11:11] 12
+[2022-01-01 11:11:11] 14
+[2022-01-01 11:11:11] 16
+[2022-01-01 11:11:11] 18
+[2022-01-01 12:12:12] 20
+
+-- !select_13 --
+[2022-01-01] 2
+[2022-01-01] 4
+[2022-01-01] 6
+[2022-01-01] 8
+[2022-01-01] 10
+[2022-01-01] 12
+[2022-01-01] 14
+[2022-01-01] 16
+[2022-01-01] 18
+[2022-06-06] 20
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_boolean.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_boolean.out
new file mode 100644
index 0000000000..7a293780a0
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_boolean.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+111 true
+112 false
+113 false
+114 true
+
+-- !select --
+false
+
+-- !select --
+true
+
+-- !select --
+false
+
+-- !select --
+true
+
+-- !select --
+\N
+
+-- !select --
+111 false
+112 true
+113 true
+114 false
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_case.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_case.out
new file mode 100644
index 0000000000..3db56045a2
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_case.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+2022-10-21 -3 0
+2022-10-25 -7 -3
+2022-10-26 -7 \N
+
+-- !select --
+2022-10-17,2022-10-18,2022-10-19,2022-10-20,2022-10-21,2022-10-22,2022-10-23
+
+-- !select --
+\N
+
+-- !select --
+2022-10-21 2022-10-18,2022-10-19,2022-10-20
+2022-10-25 2022-10-18,2022-10-19,2022-10-20,2022-10-21
+2022-10-26 \N
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_date.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_date.out
new file mode 100644
index 0000000000..4fb1082ba4
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_date.out
@@ -0,0 +1,276 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+1 2022-01-01 2022-01-01T11:11:11 2022-01-01 2022-01-01T11:11:11
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+false
+false
+false
+false
+false
+false
+false
+false
+false
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
+-- !select --
+true
+true
+true
+true
+true
+true
+true
+true
+true
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_decimal.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_decimal.out
new file mode 100644
index 0000000000..b1ad6275aa
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_decimal.out
@@ -0,0 +1,26 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+111 11111.111110000 222222.333333300
+112 1234556.111110000 222222.333333300
+113 87654321.111110000 \N
+
+-- !select --
+113.947561100
+
+-- !select --
+113.947561100
+
+-- !select --
+\N
+
+-- !select --
+\N
+
+-- !select --
+111 233333.444443300
+112 1456778.444443300
+113 \N
+
+-- !select_decimal_string --
+0E-9
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_float.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_float.out
new file mode 100644
index 0000000000..d806cbd546
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_float.out
@@ -0,0 +1,40 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+111 11111.111 222222.33 1.234567834455677E7 1111112.0
+112 1234556.1 222222.33 2.2222222233333334E8 4.444444444444556E12
+113 8.765432E7 \N 6.666666666666667E9 \N
+
+-- !select --
+-108.27467
+
+-- !select --
+-108.27467
+
+-- !select --
+\N
+
+-- !select --
+\N
+
+-- !select --
+111 -211111.22
+112 1012333.8
+113 \N
+
+-- !select --
+113.9475611
+
+-- !select --
+113.9475611
+
+-- !select --
+\N
+
+-- !select --
+\N
+
+-- !select --
+111 2.469135668911354E7
+112 4.444444446666667E8
+113 1.3333333333333334E10
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_int.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_int.out
new file mode 100644
index 0000000000..561ba219da
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_int.out
@@ -0,0 +1,88 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 2 3 4 5
+2 4 6 8 10
+3 6 9 12 15
+4 8 12 16 20
+5 10 15 20 25
+6 12 18 24 30
+7 14 21 28 35
+8 16 24 32 40
+9 18 27 36 45
+10 20 30 40 50
+
+-- !select --
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+
+-- !select --
+\N
+
+-- !select --
+2
+4
+6
+8
+10
+12
+14
+16
+18
+20
+
+-- !select --
+\N
+
+-- !select --
+3
+6
+9
+12
+15
+18
+21
+24
+27
+30
+
+-- !select --
+\N
+
+-- !select --
+5
+9
+13
+17
+21
+25
+29
+33
+37
+41
+
+-- !select --
+\N
+
+-- !select --
+6
+11
+16
+21
+26
+31
+36
+41
+46
+51
+
+-- !select --
+\N
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_null.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_null.out
new file mode 100644
index 0000000000..dc59ed24af
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_null.out
@@ -0,0 +1,26 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+-- !select --
+\N
+
+-- !select --
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+\N
+
diff --git a/regression-test/data/nereids_p0/javaudf/test_javaudf_string.out b/regression-test/data/nereids_p0/javaudf/test_javaudf_string.out
new file mode 100644
index 0000000000..892c524582
--- /dev/null
+++ b/regression-test/data/nereids_p0/javaudf/test_javaudf_string.out
@@ -0,0 +1,45 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 1 abcdefg1 poiuytre1abcdefg
+2 2 abcdefg2 poiuytre2abcdefg
+3 3 abcdefg3 poiuytre3abcdefg
+4 4 abcdefg4 poiuytre4abcdefg
+5 5 abcdefg5 poiuytre5abcdefg
+6 6 abcdefg6 poiuytre6abcdefg
+7 7 abcdefg7 poiuytre7abcdefg
+8 8 abcdefg8 poiuytre8abcdefg
+9 9 abcdefg9 poiuytre9abcdefg
+
+-- !select --
+ab***fg1
+ab***fg2
+ab***fg3
+ab***fg4
+ab***fg5
+ab***fg6
+ab***fg7
+ab***fg8
+ab***fg9
+
+-- !select --
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+po***********efg
+
+-- !select --
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+ab*def ab**efg
+
diff --git a/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_groupcount.groovy b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_groupcount.groovy
new file mode 100644
index 0000000000..25efda736c
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_groupcount.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_groupcount") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_groupcount"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("abc"), ("123"), ("123"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION group_count(string) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.collect.GroupCountUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT group_count(col_1) as a FROM ${tableName} ORDER BY a; """
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS group_count(string);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.groovy b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.groovy
new file mode 100644
index 0000000000..d1d3e6de6f
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_murmurhash3.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_murmurhash3") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_murmurhash3"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("abc"), ("123"), ("123"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION murmurhash3(string) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.collect.MurmurHash3UDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT murmurhash3(col_1) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS murmurhash3(string);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_sessionize.groovy b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_sessionize.groovy
new file mode 100644
index 0000000000..5717137ef3
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/collect/test_javaudf_sessionize.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_sessionize") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_sessionize"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL,
+ `col_2` bigint NOT NULL,
+ `col_3` int NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("abc", 1234500000, 20), ("bcd", 1234500000, 10); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION sessionize(string, bigint, int) RETURNS String PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.collect.SessionizeUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT sessionize(col_1, col_2, col_3) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS sessionize(string, bigint, int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
+
diff --git a/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_adddays.groovy b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_adddays.groovy
new file mode 100644
index 0000000000..18ff9210fb
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_adddays.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_adddays") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_adddays"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("20220101"), ("20211231"), ("20220228"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION add_days(string, int) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.date.AddDaysUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT add_days(col_1, 1) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS add_days(string, int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_addisoperioud.groovy b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_addisoperioud.groovy
new file mode 100644
index 0000000000..346f1f5a5d
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_addisoperioud.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_addisoperioud") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_addisoperioud"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("20220101"), ("20211231"), ("20220228"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION add_days(string, string, string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.date.AddISOPeriodUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT add_days(col_1, 'YYYYMMdd', 'PT02H00M') as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS add_days(string, string, string);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_daydiff.groovy b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_daydiff.groovy
new file mode 100644
index 0000000000..34d75521ae
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/date/test_javaudf_daydiff.groovy
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_daydiff") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_daydiff"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL,
+ `col_2` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("20220101", "20220102"), ("20221231", "20230101"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION day_diff(string, string) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.date.DayDiffUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT day_diff(col_1, col_2) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS day_diff(string, string);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.groovy b/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.groovy
new file mode 100644
index 0000000000..0a62ca2247
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_convertfromcamelcase.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_convertfromcamelcase") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_convertfromcamelcase"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(20) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("testJavaUdf"), ("getIdFromTable"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION convert_from_camel(string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.json.ConvertFromCamelCaseUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT convert_from_camel(col_1) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS convert_from_camel(string); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.groovy b/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.groovy
new file mode 100644
index 0000000000..a2a879b7b3
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/json/test_javaudf_converttocamelcase.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_converttocamelcase") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_converttocamelcase"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(20) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("test_java_udf"), ("get_id_from_table"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION convert_to_camel(string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.json.ConvertToCamelCaseUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT convert_to_camel(col_1) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS convert_to_camel(string); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertequal.groovy b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertequal.groovy
new file mode 100644
index 0000000000..8b81756638
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertequal.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_assertequal") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_assertequal"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col` varchar(10) NOT NULL,
+ `col_1` double NOT NULL,
+ `col_2` double NOT NULL
+ )
+ DISTRIBUTED BY HASH(col) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ('abc', 23.34, 23.34); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION asser_equal(double, double) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.sanity.AssertEqualsUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT asser_equal(col_1, col_2) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS asser_equal(double, double); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.groovy b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.groovy
new file mode 100644
index 0000000000..9675d40709
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertlessthan.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_assertlessthan") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_assertlessthan"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col` varchar(10) NOT NULL,
+ `col_1` double NOT NULL,
+ `col_2` double NOT NULL
+ )
+ DISTRIBUTED BY HASH(col) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ('abc', 23.34, 23.35), ('bcd', 0.123, 0.124); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION asser_lessthan(double, double) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.sanity.AssertLessThanUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT asser_lessthan(col_1, col_2) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS asser_lessthan(double, double); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertudf.groovy b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertudf.groovy
new file mode 100644
index 0000000000..7b624a5d29
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/sanity/test_javaudf_assertudf.groovy
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_assertudf") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_assertudf"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION assert(boolean, string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.sanity.AssertUDF",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT assert(true, "not OK"); """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS assert(boolean, string); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/sketch/test_javaudf_md5.groovy b/regression-test/suites/nereids_p0/javaudf/sketch/test_javaudf_md5.groovy
new file mode 100644
index 0000000000..833f139f58
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/sketch/test_javaudf_md5.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_md5") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_md5"
+ File path = new File("${context.file.parent}")
+ def jarPath = """${path.getParent()}/../../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `col_1` varchar(10) NOT NULL
+ )
+ DISTRIBUTED BY HASH(col_1) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES ("abc"), ("123"); """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION md5(string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.sketch.Md5",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT md5(col_1) as a FROM ${tableName} ORDER BY a; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS md5(string); ")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_alias_function.groovy b/regression-test/suites/nereids_p0/javaudf/test_alias_function.groovy
new file mode 100644
index 0000000000..6b3d240fa8
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_alias_function.groovy
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nereids_test_alias_function") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+ sql 'use nereids_test_query_db'
+
+ sql 'drop function if exists f1(datetimev2(3), int)'
+ sql 'drop function if exists f2(datetimev2(3), int)'
+ sql 'drop function if exists f3(int)'
+
+ sql '''
+ create alias function f1(datetimev2(3), int) with parameter (datetime1, int1) as
+ date_trunc(days_sub(datetime1, int1), 'day')
+ '''
+ sql '''
+ CREATE ALIAS FUNCTION f2(DATETIMEV2(3), INT) with PARAMETER (datetime1, int1) as
+ DATE_FORMAT(HOURS_ADD(
+ date_trunc(datetime1, 'day'),
+ add(multiply(floor(divide(HOUR(datetime1), divide(24, int1))), 1), 1)), '%Y%m%d:%H')
+ '''
+ sql '''
+ CREATE ALIAS FUNCTION f3(INT) with PARAMETER (int1) as
+ f2(f1('2023-05-20', 2), int1)
+ '''
+
+ test {
+ sql 'select cast(f1(\'2023-06-01\', 3) as string);'
+ result([['2023-05-29 00:00:00']])
+ }
+ test {
+ sql 'select f2(f1(\'2023-05-20\', 2), 3)'
+ result([['20230518:01']])
+ }
+ test {
+ sql 'select f3(4)'
+ result([['20230518:01']])
+ }
+ test {
+ sql 'select cast(f1(\'2023-06-01\', k1) as string) from test order by k1'
+ result([
+ ['2023-05-31 00:00:00'],
+ ['2023-05-30 00:00:00'],
+ ['2023-05-29 00:00:00']
+ ])
+ }
+ test {
+ sql 'select f2(f1(\'2023-05-20\', k1), 4) from test order by k1'
+ result([
+ ['20230519:01'],
+ ['20230518:01'],
+ ['20230517:01']
+ ])
+ }
+ test {
+ sql 'select f3(k1) from test order by k1'
+ result([
+ ['20230518:01'],
+ ['20230518:01'],
+ ['20230518:01']
+ ])
+ }
+}
\ No newline at end of file
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_my_date_datetime.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_my_date_datetime.groovy
new file mode 100644
index 0000000000..0ca66ad13a
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_my_date_datetime.groovy
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_my_date_datetime") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_my_date_datetime"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `date_col` date NOT NULL,
+ `datetime_col` datetime NOT NULL,
+ `datev2_col` datev2 NOT NULL,
+ `datetimev2_col` datetimev2 NOT NULL
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (1, '2022-01-01', '2022-01-01 11:11:11', '2022-01-01', '2022-01-01 11:11:11'),
+ """)
+ }
+ sb.append("""
+ (1, '2022-01-01', '2022-01-01 11:11:11', '2022-01-01', '2022-01-01 11:11:11')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+ // test datev2
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_day_datev2(datev2) RETURNS datev2 PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyDayDate",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT udaf_my_day_datev2(datev2_col) result FROM ${tableName}; """
+
+ qt_select2 """ select user_id, udaf_my_day_datev2(datev2_col) from ${tableName} group by user_id order by user_id; """
+
+
+
+ // test date
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_day_date(date) RETURNS date PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyDayDate",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select3 """ SELECT udaf_my_day_date(date_col) result FROM ${tableName}; """
+
+ qt_select4 """ select user_id, udaf_my_day_date(date_col) from ${tableName} group by user_id order by user_id; """
+
+
+
+ // test datetimev2
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_hour_datetimev2(datetimev2) RETURNS datetimev2 PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyHourDateTime",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select5 """ SELECT udaf_my_hour_datetimev2(datetimev2_col) result FROM ${tableName}; """
+
+ qt_select6 """ select user_id, udaf_my_hour_datetimev2(datetimev2_col) from ${tableName} group by user_id order by user_id; """
+
+
+
+ // test datetime
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_hour_datetime(datetime) RETURNS datetime PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyHourDateTime",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select5 """ SELECT udaf_my_hour_datetime(datetime_col) result FROM ${tableName}; """
+
+ qt_select6 """ select user_id, udaf_my_hour_datetime(datetime_col) from ${tableName} group by user_id order by user_id; """
+
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_hour_datetime(datetime)")
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_day_datev2(datev2);")
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_day_date(date);")
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_hour_datetimev2(datetimev2);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.groovy
new file mode 100644
index 0000000000..02c865de18
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mygroupconcat_string.groovy
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_mygroupconcat_string") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_mygroupconcat_string"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `char_col` CHAR NOT NULL COMMENT "",
+ `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+ `string_col` STRING NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i % 3}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),
+ """)
+ }
+ sb.append("""
+ (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_mygroupconcat_string(string) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MyGroupConcatString",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT user_id, udaf_mygroupconcat_string(varchar_col) result FROM ${tableName} GROUP BY user_id order by user_id; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_mygroupconcat_string(string);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_array.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_array.groovy
new file mode 100644
index 0000000000..3787cdc396
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_array.groovy
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_mysum_array") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ // nereids doesn't support array now, we will enable the suite after supporting.
+ // def tableName = "test_javaudaf_mysum_array"
+ // File path = new File("${context.file.parent}")
+ // def jarPath = """${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"""
+
+ // log.info("Jar path: ${jarPath}".toString())
+ // try {
+ // sql """ DROP TABLE IF EXISTS ${tableName} """
+ // sql """
+ // CREATE TABLE IF NOT EXISTS ${tableName} (
+ // `user_id` INT NOT NULL COMMENT "用户id",
+ // `char_col` CHAR NOT NULL COMMENT "",
+ // `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+ // `string_col` STRING NOT NULL COMMENT ""
+ // )
+ // DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ // """
+ // StringBuilder sb = new StringBuilder()
+ // int i = 1
+ // for (; i < 9; i ++) {
+ // sb.append("""
+ // (${i % 3}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),
+ // """)
+ // }
+ // sb.append("""
+ // (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')
+ // """)
+ // sql """ INSERT INTO ${tableName} VALUES
+ // ${sb.toString()}
+ // """
+ // qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """
+
+ // File path1 = new File(jarPath)
+ // if (!path1.exists()) {
+ // throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ // }
+
+ // sql """ DROP FUNCTION IF EXISTS udaf_my_sum_arrayint(array<int>); """
+ // sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_arrayint(array<int>) RETURNS BigInt PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.MySumArrayInt",
+ // "type"="JAVA_UDF"
+ // ); """
+
+ // qt_select1 """ SELECT udaf_my_sum_arrayint(array(user_id)) result FROM ${tableName}; """
+
+ // qt_select2 """ select user_id, udaf_my_sum_arrayint(array(user_id)) from ${tableName} group by user_id order by user_id; """
+
+ // qt_select3 """ select user_id, sum(user_id), udaf_my_sum_arrayint(array(user_id)) from ${tableName} group by user_id order by user_id; """
+
+ // qt_select4 """ select user_id, udaf_my_sum_arrayint(array(user_id)), sum(user_id) from ${tableName} group by user_id order by user_id; """
+
+ // qt_select5 """ SELECT udaf_my_sum_arrayint(array(user_id, user_id)) result FROM ${tableName}; """
+
+
+ // sql """ DROP FUNCTION IF EXISTS udaf_my_sum_return_arrayint(array<int>); """
+ // sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_return_arrayint(array<int>) RETURNS array<int> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.MySumReturnArrayInt",
+ // "type"="JAVA_UDF"
+ // ); """
+
+ // qt_select6 """ SELECT udaf_my_sum_return_arrayint(array(user_id)) result FROM ${tableName}; """
+ // qt_select6_1 """ SELECT udaf_my_sum_return_arrayint(array(user_id)) result FROM ${tableName} group by user_id order by user_id; """
+ // qt_select7 """ SELECT udaf_my_sum_return_arrayint(array(user_id, user_id)) result FROM ${tableName}; """
+ // qt_select7_1 """ SELECT udaf_my_sum_return_arrayint(array(user_id, user_id)) result FROM ${tableName} group by user_id order by user_id; """
+
+
+ // sql """ DROP FUNCTION IF EXISTS udaf_my_sum_return_arraystring(array<string>); """
+ // sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_return_arraystring(array<string>) RETURNS array<string> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.MyReturnArrayString",
+ // "type"="JAVA_UDF"
+ // ); """
+
+ // qt_select_8 """ SELECT udaf_my_sum_return_arraystring(array(string_col)) FROM ${tableName}; """
+ // qt_select_9 """ SELECT udaf_my_sum_return_arraystring(array(string_col)), user_id as result FROM ${tableName} group by result ORDER BY result; """
+ // qt_select_10 """ SELECT udaf_my_sum_return_arraystring(array(string_col, cast(user_id as string))), user_id as result FROM ${tableName} group by result ORDER BY result; """
+ // qt_select_11 """ SELECT udaf_my_sum_return_arraystring(null) result FROM ${tableName}; """
+
+
+ // sql """ DROP FUNCTION IF EXISTS udaf_my_sum_arraystring(array<string>); """
+ // sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_arraystring(array<string>) RETURNS string PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.MyArrayString",
+ // "type"="JAVA_UDF"
+ // ); """
+
+ // qt_select_13 """ SELECT udaf_my_sum_arraystring(array(string_col)), user_id as result FROM ${tableName} group by result ORDER BY result; """
+ // qt_select_14 """ SELECT udaf_my_sum_arraystring(array(string_col, cast(user_id as string))), user_id as result FROM ${tableName} group by result ORDER BY result; """
+ // qt_select_15 """ SELECT udaf_my_sum_arraystring(null) result FROM ${tableName}; """
+
+ // } finally {
+ // try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_arraystring(array<string>);")
+ // try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_return_arraystring(array<string>);")
+ // try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_arrayint(array<int>);")
+ // try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_return_arrayint(array<int>);")
+ // try_sql("DROP TABLE IF EXISTS ${tableName}")
+ // }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_decimal.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_decimal.groovy
new file mode 100644
index 0000000000..e39ab4d3dd
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_decimal.groovy
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_mysum_decimal") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_mysum_decimal"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "",
+ `cost_1` decimal(27,9) NOT NULL COMMENT "",
+ `cost_2` decimal(27,9) COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+
+
+ sql """ INSERT INTO ${tableName} (`user_id`,`cost_1`,`cost_2`) VALUES
+ (111,11111.11111,222222.3333333),
+ (112,1234556.11111,222222.3333333),
+ (113,87654321.11111,null)
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_decimal(decimal(27,9)) RETURNS decimal(27,9) PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MySumDecimal",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT udaf_my_sum_decimal(cost_1) result FROM ${tableName}; """
+
+ qt_select2 """ select user_id, udaf_my_sum_decimal(cost_1) from ${tableName} group by user_id order by user_id; """
+
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_decimal(decimal(27,9));")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_float_double.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_float_double.groovy
new file mode 100644
index 0000000000..8ac94de762
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_float_double.groovy
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_mysum_float_double") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_mysum_float_double"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `float_col` float NOT NULL COMMENT "",
+ `double_col` double NOT NULL COMMENT "",
+ `string_col` STRING NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i % 3}, '${i*111/10}','${i*111/100+i/1000}','poiuytre${i}abcdefg'),
+ """)
+ }
+ sb.append("""
+ (${i}, '${i*111/10}','${i*111/100+i/10}','poiuytre${i}abcdefg')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY float_col; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_double(double,double) RETURNS double PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MySumDouble",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT udaf_my_sum_double(double_col,double_col) result FROM ${tableName}; """
+
+ qt_select2 """ select user_id, udaf_my_sum_double(double_col,double_col) from ${tableName} group by user_id order by user_id; """
+
+
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_float(float) RETURNS float PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MySumFloat",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select3 """ SELECT udaf_my_sum_float(float_col) result FROM ${tableName}; """
+
+ qt_select4 """ select user_id, udaf_my_sum_float(float_col) from ${tableName} group by user_id order by user_id; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_float(float);")
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_double(double,double);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_int.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_int.groovy
new file mode 100644
index 0000000000..2d70f5ed77
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_mysum_int.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_mysum_int") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_mysum_int"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `char_col` CHAR NOT NULL COMMENT "",
+ `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+ `string_col` STRING NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i % 3}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),
+ """)
+ }
+ sb.append("""
+ (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_my_sum_int(int) RETURNS BigInt PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.MySumInt",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT udaf_my_sum_int(user_id) result FROM ${tableName}; """
+
+ qt_select2 """ select user_id, udaf_my_sum_int(user_id) from ${tableName} group by user_id order by user_id; """
+
+ qt_select3 """ select user_id, sum(user_id), udaf_my_sum_int(user_id) from ${tableName} group by user_id order by user_id; """
+
+ qt_select4 """ select user_id, udaf_my_sum_int(user_id), sum(user_id) from ${tableName} group by user_id order by user_id; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_my_sum_int(int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudaf_null_test.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_null_test.groovy
new file mode 100644
index 0000000000..9ec3b5965c
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudaf_null_test.groovy
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudaf_null_test") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudaf_null_test"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `char_col` CHAR NOT NULL COMMENT "",
+ `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+ `string_col` STRING NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i % 3}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),
+ """)
+ }
+ sb.append("""
+ (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE AGGREGATE FUNCTION udaf_null_test_int(int) RETURNS BigInt PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.UdafNullTest",
+ "always_nullable"="true",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select1 """ SELECT udaf_null_test_int(user_id) result FROM ${tableName}; """
+
+ qt_select2 """ select user_id, udaf_null_test_int(user_id) from ${tableName} group by user_id order by user_id; """
+
+ qt_select3 """ SELECT udaf_null_test_int(1) result FROM ${tableName}; """
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS udaf_null_test_int(int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_array.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_array.groovy
new file mode 100644
index 0000000000..fdc5c770ca
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_array.groovy
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_array") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ // nereids doesn't support array now, we will enable the suite after supporting.
+ // def tableName = "test_javaudf_array"
+ // File path = new File("${context.file.parent}")
+ // def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ // log.info("Jar path: ${jarPath}".toString())
+ // try {
+ // sql """ DROP TABLE IF EXISTS ${tableName} """
+ // sql """
+ // CREATE TABLE IF NOT EXISTS ${tableName} (
+ // `user_id` INT NOT NULL COMMENT "",
+ // `tinyint_col` TINYINT NOT NULL COMMENT "",
+ // `datev2_col` datev2 NOT NULL COMMENT "",
+ // `datetimev2_col` datetimev2 NOT NULL COMMENT "",
+ // `string_col` STRING NOT NULL COMMENT ""
+ // )
+ // DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ // """
+ // StringBuilder sb = new StringBuilder()
+ // int i = 1
+ // for (; i < 10; i ++) {
+ // sb.append("""
+ // (${i},${i}*2,'2022-01-01','2022-01-01 11:11:11','a${i}b'),
+ // """)
+ // }
+ // sb.append("""
+ // (${i},${i}*2,'2022-06-06','2022-01-01 12:12:12','a${i}b')
+ // """)
+ // sql """ INSERT INTO ${tableName} VALUES
+ // ${sb.toString()}
+ // """
+ // qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ // File path1 = new File(jarPath)
+ // if (!path1.exists()) {
+ // throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ // }
+
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>); """
+ // sql """ CREATE FUNCTION java_udf_array_int_test(array<int>) RETURNS int PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayIntTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_1 """ SELECT java_udf_array_int_test(array(user_id)) result FROM ${tableName} ORDER BY result; """
+ // qt_select_2 """ SELECT java_udf_array_int_test(null) result ; """
+
+
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_return_int_test(array<int>); """
+ // sql """ CREATE FUNCTION java_udf_array_return_int_test(array<int>) RETURNS array<int> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayReturnArrayIntTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_3 """ SELECT java_udf_array_return_int_test(array(user_id)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_4 """ SELECT java_udf_array_return_int_test(array(user_id,user_id)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_5 """ SELECT java_udf_array_return_int_test(null) result ; """
+
+
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_return_string_test(array<string>); """
+ // sql """ CREATE FUNCTION java_udf_array_return_string_test(array<string>) RETURNS array<string> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayReturnArrayStringTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_6 """ SELECT java_udf_array_return_string_test(array(string_col)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_7 """ SELECT java_udf_array_return_string_test(array(string_col, cast(user_id as string))), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_8 """ SELECT java_udf_array_return_string_test(null) result ; """
+
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_string_test(array<string>); """
+ // sql """ CREATE FUNCTION java_udf_array_string_test(array<string>) RETURNS string PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayStringTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_9 """ SELECT java_udf_array_string_test(array(string_col)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_10 """ SELECT java_udf_array_string_test(array(string_col, cast(user_id as string))), tinyint_col as result FROM ${tableName} ORDER BY result; """
+ // qt_select_11 """ SELECT java_udf_array_string_test(null) result ; """
+
+ // //ArrayDateTimeTest
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_datatime_test(array<datetime>); """
+ // sql """ CREATE FUNCTION java_udf_array_datatime_test(array<datetime>) RETURNS array<datetime> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayDateTimeTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_12 """ SELECT java_udf_array_datatime_test(array(datetimev2_col)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+
+ // sql """ DROP FUNCTION IF EXISTS java_udf_array_date_test(array<date>); """
+ // sql """ CREATE FUNCTION java_udf_array_date_test(array<date>) RETURNS array<date> PROPERTIES (
+ // "file"="file://${jarPath}",
+ // "symbol"="org.apache.doris.udf.ArrayDateTest",
+ // "type"="JAVA_UDF"
+ // ); """
+ // qt_select_13 """ SELECT java_udf_array_date_test(array(datev2_col)), tinyint_col as result FROM ${tableName} ORDER BY result; """
+
+ // } finally {
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>);")
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_int_test(array<int>);")
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_string_test(array<string>);")
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_string_test(array<string>);")
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_datatime_test(array<datetime>);")
+ // try_sql("DROP FUNCTION IF EXISTS java_udf_array_date_test(array<date>);")
+ // try_sql("DROP TABLE IF EXISTS ${tableName}")
+ // }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_boolean.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_boolean.groovy
new file mode 100644
index 0000000000..3471e1d3ea
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_boolean.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_boolean") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_boolean"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "",
+ `boo_1` BOOLEAN NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${tableName} (`user_id`,`boo_1`) VALUES
+ (111,true),
+ (112,false),
+ (113,0),
+ (114,1)
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_boolean_test(BOOLEAN) RETURNS BOOLEAN PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.BooleanTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_boolean_test(1) as result; """
+ qt_select """ SELECT java_udf_boolean_test(0) as result ; """
+ qt_select """ SELECT java_udf_boolean_test(true) as result ; """
+ qt_select """ SELECT java_udf_boolean_test(false) as result ; """
+ qt_select """ SELECT java_udf_boolean_test(null) as result ; """
+ qt_select """ SELECT user_id,java_udf_boolean_test(boo_1) as result FROM ${tableName} order by user_id; """
+
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_boolean_test(BOOLEAN);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_case.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_case.groovy
new file mode 100644
index 0000000000..617bb74e3a
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_case.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_case") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_datecasetest"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `starttime` Date NOT NULL COMMENT "",
+ `start` int NOT NULL COMMENT "",
+ `end` int COMMENT ""
+ )
+ DISTRIBUTED BY HASH(starttime) PROPERTIES("replication_num" = "1");
+ """
+
+
+ sql """ INSERT INTO ${tableName} (`starttime`,`start`,`end`) VALUES
+ ("2022-10-21",-3,0),
+ ("2022-10-25",-7,-3),
+ ("2022-10-26",-7,null)
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY starttime; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_dateCase_test(date,int,int) RETURNS String PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateCaseTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_dateCase_test("2022-10-24",-7,0) as result; """
+ qt_select """ SELECT java_udf_dateCase_test(null,-7,0) as result ; """
+ qt_select """ SELECT starttime,java_udf_dateCase_test(starttime,start,end) as sum FROM ${tableName} order by starttime; """
+
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_dateCase_test(date,int,int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_date.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_date.groovy
new file mode 100644
index 0000000000..4a2438c465
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_date.groovy
@@ -0,0 +1,211 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_date") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_date"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `date_col` date NOT NULL,
+ `datetime_col` datetime NOT NULL,
+ `datev2_col` datev2 NOT NULL,
+ `datetimev2_col` datetimev2 NOT NULL
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (1, '2022-01-01', '2022-01-01 11:11:11', '2022-01-01', '2022-01-01 11:11:11'),
+ """)
+ }
+ sb.append("""
+ (1, '2022-01-01', '2022-01-01 11:11:11', '2022-01-01', '2022-01-01 11:11:11')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_date_test1(date, date) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest1",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_date_test1(date_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test1('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test1(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_date_test2(date, date) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest2",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_date_test2(date_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test2('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test2(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_date_test3(date, date) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest3",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_date_test3(date_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test3('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_date_test3(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetime_test1(datetime, datetime) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest1",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetime_test1(datetime_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test1('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test1(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetime_test2(datetime, datetime) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest2",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetime_test2(datetime_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test2('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test2(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetime_test3(datetime, datetime) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest3",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetime_test3(datetime_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test3('2011-01-01', '2011-01-01') result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_datetime_test3(null, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+
+ sql """ CREATE FUNCTION java_udf_datev2_test1(datev2, datev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest1",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datev2_test1(datev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datev2_test2(datev2, datev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest2",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datev2_test2(datev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datev2_test3(datev2, datev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTest3",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datev2_test3(datev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetimev2_test1(datetimev2, datetimev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest1",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetimev2_test1(datetimev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetimev2_test2(datetimev2, datetimev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest2",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetimev2_test2(datetimev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_datetimev2_test3(datetimev2, datetimev2) RETURNS boolean PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DateTimeTest3",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_datetimev2_test3(datetimev2_col, '2011-01-01') result FROM ${tableName} ORDER BY result; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_date_test3(date, date);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetime_test1(datetime, datetime);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetime_test2(datetime, datetime);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetime_test3(datetime, datetime);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datev2_test1(datev2, datev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datev2_test2(datev2, datev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datev2_test3(datev2, datev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetimev2_test1(datetimev2, datetimev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetimev2_test2(datetimev2, datetimev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_datetimev2_test3(datetimev2, datetimev2);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_date_test2(date, date);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_date_test1(date, date);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_decimal.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_decimal.groovy
new file mode 100644
index 0000000000..e738135688
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_decimal.groovy
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_decimal") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_decimal"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "",
+ `cost_1` decimal(27,9) NOT NULL COMMENT "",
+ `cost_2` decimal(27,9) COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+
+
+ sql """ INSERT INTO ${tableName} (`user_id`,`cost_1`,`cost_2`) VALUES
+ (111,11111.11111,222222.3333333),
+ (112,1234556.11111,222222.3333333),
+ (113,87654321.11111,null)
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_decimal_test(decimal(27,9),decimal(27,9)) RETURNS decimal(27,9) PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DecimalTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_decimal_test(cast(2.83645 as decimal(27,9)),cast(111.1111111 as decimal(27,9))) as result; """
+ qt_select """ SELECT java_udf_decimal_test(2.83645,111.1111111) as result ; """
+ qt_select """ SELECT java_udf_decimal_test(2.83645,null) as result ; """
+ qt_select """ SELECT java_udf_decimal_test(cast(2.83645 as decimal(27,9)),null) as result ; """
+ qt_select """ SELECT user_id,java_udf_decimal_test(cost_1, cost_2) as sum FROM ${tableName} order by user_id; """
+
+
+
+
+
+ sql """ DROP FUNCTION if exists java_udf_decimal_string_test(decimal(27,9),String,String); """
+ sql """ CREATE FUNCTION java_udf_decimal_string_test(decimal(27,9),String,String) RETURNS decimal(27,9) PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DecimalStringTest",
+ "type"="JAVA_UDF"
+ ); """
+ qt_select_decimal_string """ SELECT java_udf_decimal_string_test(2.83645,'asd','a') as result; """
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_decimal_test(decimal(27,9),decimal(27,9));")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_decimal_string_test(decimal(27,9),String,String);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_float.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_float.groovy
new file mode 100644
index 0000000000..daea66fc42
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_float.groovy
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_float") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_float"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "",
+ `float_1` FLOAT NOT NULL COMMENT "",
+ `float_2` FLOAT COMMENT "",
+ `double_1` DOUBLE NOT NULL COMMENT "",
+ `double_2` DOUBLE COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+
+
+ sql """ INSERT INTO ${tableName} (`user_id`,`float_1`,`float_2`,double_1,double_2) VALUES
+ (111,11111.11111,222222.3333333,12345678.34455677,1111111.999999999999),
+ (112,1234556.11111,222222.3333333,222222222.3333333333333,4444444444444.555555555555),
+ (113,87654321.11111,null,6666666666.6666666666,null)
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_float_test(FLOAT,FLOAT) RETURNS FLOAT PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.FloatTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_float_test(cast(2.83645 as float),cast(111.1111111 as float)) as result; """
+ qt_select """ SELECT java_udf_float_test(2.83645,111.1111111) as result ; """
+ qt_select """ SELECT java_udf_float_test(2.83645,null) as result ; """
+ qt_select """ SELECT java_udf_float_test(cast(2.83645 as float),null) as result ; """
+ qt_select """ SELECT user_id,java_udf_float_test(float_1, float_2) as sum FROM ${tableName} order by user_id; """
+
+
+
+
+ sql """ CREATE FUNCTION java_udf_double_test(DOUBLE,DOUBLE) RETURNS DOUBLE PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.DoubleTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_double_test(cast(2.83645 as DOUBLE),cast(111.1111111 as DOUBLE)) as result; """
+ qt_select """ SELECT java_udf_double_test(2.83645,111.1111111) as result ; """
+ qt_select """ SELECT java_udf_double_test(2.83645,null) as result ; """
+ qt_select """ SELECT java_udf_double_test(cast(2.83645 as DOUBLE),null) as result ; """
+ qt_select """ SELECT user_id,java_udf_double_test(double_1, double_1) as sum FROM ${tableName} order by user_id; """
+
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_double_test(DOUBLE,DOUBLE);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_float_test(FLOAT,FLOAT);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_int.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_int.groovy
new file mode 100644
index 0000000000..fdc2eca485
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_int.groovy
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_int") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_int"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "",
+ `tinyint_col` TINYINT NOT NULL COMMENT "",
+ `smallint_col` SMALLINT NOT NULL COMMENT "",
+ `bigint_col` BIGINT NOT NULL COMMENT "",
+ `largeint_col` LARGEINT NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 10; i ++) {
+ sb.append("""
+ (${i},${i}*2,${i}*3,${i}*4,${i}*5),
+ """)
+ }
+ sb.append("""
+ (${i},${i}*2,${i}*3,${i}*4,${i}*5)
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_int_test(int) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.IntTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_int_test(user_id) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_int_test(null) result ; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_tinyint_test(tinyint) RETURNS tinyint PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.TinyintTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_tinyint_test(tinyint_col) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_tinyint_test(null) result ; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_smallint_test(smallint) RETURNS smallint PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.SmallintTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_smallint_test(smallint_col) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_smallint_test(null) result ; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_bigint_test(bigint) RETURNS bigint PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.BigintTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_bigint_test(bigint_col) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_bigint_test(null) result ; """
+
+
+
+ sql """ CREATE FUNCTION java_udf_largeint_test(largeint) RETURNS largeint PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.LargeintTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_largeint_test(largeint_col) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_largeint_test(null) result ; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_tinyint_test(tinyint);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_smallint_test(smallint);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_bigint_test(bigint);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test(largeint);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_int_test(int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_null.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_null.groovy
new file mode 100644
index 0000000000..e7879221e2
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_null.groovy
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_null") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_null"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i}),
+ """)
+ }
+ sb.append("""
+ (${i})
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_null_test(int) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.NullTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_null_test(1) result; """
+ qt_select """ SELECT java_udf_null_test(user_id) result FROM ${tableName} ORDER BY result; """
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_null_test(int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/javaudf/test_javaudf_string.groovy b/regression-test/suites/nereids_p0/javaudf/test_javaudf_string.groovy
new file mode 100644
index 0000000000..7dbecab0cd
--- /dev/null
+++ b/regression-test/suites/nereids_p0/javaudf/test_javaudf_string.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("nereids_test_javaudf_string") {
+ sql 'set enable_nereids_planner=true'
+ sql 'set enable_fallback_to_original_planner=false'
+
+ def tableName = "test_javaudf_string"
+ File path = new File("${context.file.parent}")
+ def jarPath = "${path.getParent()}/../../java-udf-src/target/java-udf-case-jar-with-dependencies.jar"
+
+ log.info("Jar path: ${jarPath}".toString())
+ try {
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `user_id` INT NOT NULL COMMENT "用户id",
+ `char_col` CHAR NOT NULL COMMENT "",
+ `varchar_col` VARCHAR(10) NOT NULL COMMENT "",
+ `string_col` STRING NOT NULL COMMENT ""
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ StringBuilder sb = new StringBuilder()
+ int i = 1
+ for (; i < 9; i ++) {
+ sb.append("""
+ (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg'),
+ """)
+ }
+ sb.append("""
+ (${i}, '${i}','abcdefg${i}','poiuytre${i}abcdefg')
+ """)
+ sql """ INSERT INTO ${tableName} VALUES
+ ${sb.toString()}
+ """
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+
+ File path1 = new File(jarPath)
+ if (!path1.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ sql """ CREATE FUNCTION java_udf_string_test(string, int, int) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.StringTest",
+ "type"="JAVA_UDF"
+ ); """
+
+ qt_select """ SELECT java_udf_string_test(varchar_col, 2, 3) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_string_test(string_col, 2, 3) result FROM ${tableName} ORDER BY result; """
+ qt_select """ SELECT java_udf_string_test('abcdef', 2, 3), java_udf_string_test('abcdefg', 2, 3) result FROM ${tableName} ORDER BY result; """
+
+
+ } finally {
+ try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/18: [feature](table-value-functions)add catalogs table-value-function (#21790)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 580ddd8c1dde3db3d88c9ec93eb3b5f15b3dff49
Author: daidai <20...@qq.com>
AuthorDate: Fri Jul 14 10:25:16 2023 +0800
[feature](table-value-functions)add catalogs table-value-function (#21790)
mysql> select * from catalogs() order by CatalogId;
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 18 +++++
be/src/vec/exec/scan/vmeta_scanner.h | 3 +-
.../sql-functions/table-functions/catalogs.md | 91 +++++++++++++++++++++
docs/sidebars.json | 3 +-
.../sql-functions/table-functions/catalogs.md | 92 ++++++++++++++++++++++
.../tablefunction/CatalogsTableValuedFunction.java | 88 +++++++++++++++++++++
.../doris/tablefunction/MetadataGenerator.java | 37 +++++++++
.../tablefunction/MetadataTableValuedFunction.java | 2 +
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
gensrc/thrift/Types.thrift | 3 +-
.../table_valued_function/test_catalogs_tvf.out | 10 +++
.../table_valued_function/test_catalogs_tvf.groovy | 71 +++++++++++++++++
12 files changed, 417 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp
index 49cd0f1043..f0adefef90 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -216,6 +216,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
case TMetadataType::WORKLOAD_GROUPS:
RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range, &request));
break;
+ case TMetadataType::CATALOGS:
+ RETURN_IF_ERROR(_build_catalogs_metadata_request(meta_scan_range, &request));
+ break;
default:
_meta_eos = true;
return Status::OK();
@@ -323,6 +326,21 @@ Status VMetaScanner::_build_workload_groups_metadata_request(
return Status::OK();
}
+Status VMetaScanner::_build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range,
+ TFetchSchemaTableDataRequest* request) {
+ VLOG_CRITICAL << "VMetaScanner::_build_catalogs_metadata_request";
+
+ // create request
+ request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
+
+ // create TMetadataTableRequestParams
+ TMetadataTableRequestParams metadata_table_params;
+ metadata_table_params.__set_metadata_type(TMetadataType::CATALOGS);
+
+ request->__set_metada_table_params(metadata_table_params);
+ return Status::OK();
+}
+
Status VMetaScanner::close(RuntimeState* state) {
VLOG_CRITICAL << "VMetaScanner::close";
RETURN_IF_ERROR(VScanner::close(state));
diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h
index 22f3dfe681..4c2dd00584 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.h
+++ b/be/src/vec/exec/scan/vmeta_scanner.h
@@ -73,7 +73,8 @@ private:
TFetchSchemaTableDataRequest* request);
Status _build_workload_groups_metadata_request(const TMetaScanRange& meta_scan_range,
TFetchSchemaTableDataRequest* request);
-
+ Status _build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range,
+ TFetchSchemaTableDataRequest* request);
bool _meta_eos;
TupleId _tuple_id;
TUserIdentity _user_identity;
diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md b/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md
new file mode 100644
index 0000000000..d0c821f7bb
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md
@@ -0,0 +1,91 @@
+---
+{
+ "title": "catalogs",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `catalogs`
+
+### Name
+
+
+catalogs
+
+
+### description
+
+The table function generates a temporary table of catalogs to view the information of the catalogs created in the current Doris.
+
+This function is used in the from clause.
+
+#### syntax
+
+`catalogs()`
+
+Catalogs () table structure:
+```
+mysql> desc function catalogs();
++-------------+--------+------+-------+---------+-------+
+| Field | Type | Null | Key | Default | Extra |
++-------------+--------+------+-------+---------+-------+
+| CatalogId | BIGINT | No | false | NULL | NONE |
+| CatalogName | TEXT | No | false | NULL | NONE |
+| CatalogType | TEXT | No | false | NULL | NONE |
+| Property | TEXT | No | false | NULL | NONE |
+| Value | TEXT | No | false | NULL | NONE |
++-------------+--------+------+-------+---------+-------+
+5 rows in set (0.04 sec)
+```
+
+The information presented by `catalogs()` tvf is the result of synthesizing `show catalogs` and `show catalog xxx` statements.
+
+The table generated by tvf can be used for filtering, join and other operations.
+
+
+### example
+
+```
+mysql> select * from catalogs();
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+| CatalogId | CatalogName | CatalogType | Property | Value |
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+| 16725 | hive | hms | dfs.client.failover.proxy.provider.HANN | org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider |
+| 16725 | hive | hms | dfs.ha.namenodes.HANN | nn1,nn2 |
+| 16725 | hive | hms | create_time | 2023-07-13 16:24:38.968 |
+| 16725 | hive | hms | ipc.client.fallback-to-simple-auth-allowed | true |
+| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn1 | nn1_host:rpc_port |
+| 16725 | hive | hms | hive.metastore.uris | thrift://127.0.0.1:7004 |
+| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn2 | nn2_host:rpc_port |
+| 16725 | hive | hms | type | hms |
+| 16725 | hive | hms | dfs.nameservices | HANN |
+| 0 | internal | internal | NULL | NULL |
+| 16726 | es | es | create_time | 2023-07-13 16:24:44.922 |
+| 16726 | es | es | type | es |
+| 16726 | es | es | hosts | http://127.0.0.1:9200 |
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+13 rows in set (0.01 sec)
+```
+
+### keywords
+
+ catalogs
diff --git a/docs/sidebars.json b/docs/sidebars.json
index 9af06eded0..26fc605142 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -701,7 +701,8 @@
"sql-manual/sql-functions/table-functions/iceberg_meta",
"sql-manual/sql-functions/table-functions/backends",
"sql-manual/sql-functions/table-functions/frontends",
- "sql-manual/sql-functions/table-functions/workload-group"
+ "sql-manual/sql-functions/table-functions/workload-group",
+ "sql-manual/sql-functions/table-functions/catalogs"
]
},
{
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md
new file mode 100644
index 0000000000..b95795e8e6
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md
@@ -0,0 +1,92 @@
+---
+{
+ "title": "catalogs",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `catalogs`
+
+### Name
+
+
+catalogs
+
+
+### description
+
+表函数，生成 catalogs 临时表，可以查看当前doris中的创建的 catalogs 信息。
+
+该函数用于 from 子句中。
+
+#### syntax
+
+`catalogs()`
+
+catalogs()表结构：
+```
+mysql> desc function catalogs();
++-------------+--------+------+-------+---------+-------+
+| Field | Type | Null | Key | Default | Extra |
++-------------+--------+------+-------+---------+-------+
+| CatalogId | BIGINT | No | false | NULL | NONE |
+| CatalogName | TEXT | No | false | NULL | NONE |
+| CatalogType | TEXT | No | false | NULL | NONE |
+| Property | TEXT | No | false | NULL | NONE |
+| Value | TEXT | No | false | NULL | NONE |
++-------------+--------+------+-------+---------+-------+
+5 rows in set (0.04 sec)
+```
+
+`catalogs()` tvf展示的信息是综合了 `show catalogs` 与 `show catalog xxx` 语句的结果。
+
+可以利用tvf生成的表去做过滤、join等操作。
+
+
+
+### example
+
+```
+mysql> select * from catalogs();
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+| CatalogId | CatalogName | CatalogType | Property | Value |
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+| 16725 | hive | hms | dfs.client.failover.proxy.provider.HANN | org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider |
+| 16725 | hive | hms | dfs.ha.namenodes.HANN | nn1,nn2 |
+| 16725 | hive | hms | create_time | 2023-07-13 16:24:38.968 |
+| 16725 | hive | hms | ipc.client.fallback-to-simple-auth-allowed | true |
+| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn1 | nn1_host:rpc_port |
+| 16725 | hive | hms | hive.metastore.uris | thrift://127.0.0.1:7004 |
+| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn2 | nn2_host:rpc_port |
+| 16725 | hive | hms | type | hms |
+| 16725 | hive | hms | dfs.nameservices | HANN |
+| 0 | internal | internal | NULL | NULL |
+| 16726 | es | es | create_time | 2023-07-13 16:24:44.922 |
+| 16726 | es | es | type | es |
+| 16726 | es | es | hosts | http://127.0.0.1:9200 |
++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+
+13 rows in set (0.01 sec)
+```
+
+### keywords
+
+ catalogs
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java
new file mode 100644
index 0000000000..062cb29b19
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.tablefunction;
+
+
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TMetaScanRange;
+import org.apache.doris.thrift.TMetadataType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.List;
+import java.util.Map;
+
+public class CatalogsTableValuedFunction extends MetadataTableValuedFunction {
+ public static final String NAME = "catalogs";
+
+ private static final ImmutableList<Column> SCHEMA = ImmutableList.of(
+ new Column("CatalogId", ScalarType.createType(PrimitiveType.BIGINT)),
+ new Column("CatalogName", ScalarType.createStringType()),
+ new Column("CatalogType", ScalarType.createStringType()),
+ new Column("Property", ScalarType.createStringType()),
+ new Column("Value", ScalarType.createStringType()));
+
+ public CatalogsTableValuedFunction(Map<String, String> params)
+ throws org.apache.doris.nereids.exceptions.AnalysisException {
+ if (params.size() != 0) {
+ throw new org.apache.doris.nereids.exceptions
+ .AnalysisException("catalogs table-valued-function does not support any params");
+ }
+ }
+
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
+
+ static {
+ ImmutableMap.Builder<String, Integer> builder = new ImmutableMap.Builder();
+ for (int i = 0; i < SCHEMA.size(); i++) {
+ builder.put(SCHEMA.get(i).getName().toLowerCase(), i);
+ }
+ COLUMN_TO_INDEX = builder.build();
+ }
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
+ @Override
+ public String getTableName() {
+ return "CatalogsTableValuedFunction";
+ }
+
+ @Override
+ public List<Column> getTableColumns() throws AnalysisException {
+ return SCHEMA;
+ }
+
+ @Override
+ public TMetadataType getMetadataType() {
+ return TMetadataType.CATALOGS;
+ }
+
+ @Override
+ public TMetaScanRange getMetaScanRange() {
+ TMetaScanRange metaScanRange = new TMetaScanRange();
+ metaScanRange.setMetadataType(TMetadataType.CATALOGS);
+ return metaScanRange;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index ff2d9ce55c..1dcc8418d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.proc.FrontendsProcNode;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.system.Backend;
@@ -59,6 +60,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+
+
public class MetadataGenerator {
private static final Logger LOG = LogManager.getLogger(MetadataGenerator.class);
@@ -81,6 +84,9 @@ public class MetadataGenerator {
case WORKLOAD_GROUPS:
result = workloadGroupsMetadataResult(params);
break;
+ case CATALOGS:
+ result = catalogsMetadataResult(params);
+ break;
default:
return errorResult("Metadata table params is not set.");
}
@@ -257,6 +263,37 @@ public class MetadataGenerator {
return result;
}
+ private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) {
+ TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
+ List<CatalogIf> info = Env.getCurrentEnv().getCatalogMgr().listCatalogs();
+ List<TRow> dataBatch = Lists.newArrayList();
+
+ for (CatalogIf catalog : info) {
+ TRow trow = new TRow();
+ trow.addToColumnValue(new TCell().setLongVal(catalog.getId()));
+ trow.addToColumnValue(new TCell().setStringVal(catalog.getName()));
+ trow.addToColumnValue(new TCell().setStringVal(catalog.getType()));
+
+ Map<String, String> properties = catalog.getProperties();
+
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ TRow subTrow = new TRow(trow);
+ subTrow.addToColumnValue(new TCell().setStringVal(entry.getKey()));
+ subTrow.addToColumnValue(new TCell().setStringVal(entry.getValue()));
+ dataBatch.add(subTrow);
+ }
+ if (properties.isEmpty()) {
+ trow.addToColumnValue(new TCell().setStringVal("NULL"));
+ trow.addToColumnValue(new TCell().setStringVal("NULL"));
+ dataBatch.add(trow);
+ }
+ }
+
+ result.setDataBatch(dataBatch);
+ result.setStatus(new TStatus(TStatusCode.OK));
+ return result;
+ }
+
private static TFetchSchemaTableDataResult workloadGroupsMetadataResult(TMetadataTableRequestParams params) {
if (!params.isSetCurrentUserIdent()) {
return errorResult("current user ident is not set.");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
index ff01524e76..c5740c739b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
@@ -37,6 +37,8 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf
return IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName);
case WORKLOAD_GROUPS:
return WorkloadGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ case CATALOGS:
+ return CatalogsTableValuedFunction.getColumnIndexFromColumnName(columnName);
default:
throw new AnalysisException("Unknown Metadata TableValuedFunction type");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index 0cc0a61067..ea135b8b1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -57,6 +57,8 @@ public abstract class TableValuedFunctionIf {
return new FrontendsTableValuedFunction(params);
case WorkloadGroupsTableValuedFunction.NAME:
return new WorkloadGroupsTableValuedFunction(params);
+ case CatalogsTableValuedFunction.NAME:
+ return new CatalogsTableValuedFunction(params);
default:
throw new AnalysisException("Could not find table function " + funcName);
}
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 9dec29db22..522b80c0d8 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -689,7 +689,8 @@ enum TMetadataType {
ICEBERG,
BACKENDS,
WORKLOAD_GROUPS,
- FRONTENDS
+ FRONTENDS,
+ CATALOGS
}
enum TIcebergQueryType {
diff --git a/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out b/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out
new file mode 100644
index 0000000000..310da69d76
--- /dev/null
+++ b/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !create --
+catalog_test_es00 es type es
+catalog_test_hive00 hms type hms
+
+-- !delete --
+
+-- !create --
+catalog_test_es00 es type es
+
diff --git a/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy b/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy
new file mode 100644
index 0000000000..e1a98b0037
--- /dev/null
+++ b/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_catalogs_tvf") {
+ List<List<Object>> table = sql """ select * from catalogs(); """
+ assertTrue(table.size() > 0)
+ assertEquals(5, table[0].size)
+
+
+ table = sql """ select CatalogId,CatalogName from catalogs();"""
+ assertTrue(table.size() > 0)
+ assertTrue(table[0].size == 2)
+
+
+ table = sql """ select * from catalogs() where CatalogId=0;"""
+ assertTrue(table.size() > 0)
+ assertEquals("NULL", table[0][3])
+ assertEquals("NULL", table[0][4])
+
+
+ def res = sql """ select count(*) from catalogs(); """
+ assertTrue(res[0][0] > 0)
+
+ res = sql """ select * from catalogs() order by CatalogId; """
+ assertTrue(res[0][0] == 0)
+ assertEquals(res[0][1],"internal")
+ assertEquals(res[0][2],"internal")
+
+ sql """ drop catalog if exists catalog_test_hive00 """
+ sql """ drop catalog if exists catalog_test_es00 """
+
+ sql """ CREATE CATALOG catalog_test_hive00 PROPERTIES (
+ 'type'='hms',
+ 'hive.metastore.uris' = 'thrift://127.0.0.1:7004',
+ 'dfs.nameservices'='HANN',
+ 'dfs.ha.namenodes.HANN'='nn1,nn2',
+ 'dfs.namenode.rpc-address.HANN.nn1'='nn1_host:rpc_port',
+ 'dfs.namenode.rpc-address.HANN.nn2'='nn2_host:rpc_port',
+ 'dfs.client.failover.proxy.provider.HANN'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
+ ) """
+
+ sql """ CREATE CATALOG catalog_test_es00 PROPERTIES (
+ "type"="es",
+ "hosts"="http://127.0.0.1:9200"
+ ) """
+
+
+ qt_create """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName in ("catalog_test_es00","catalog_test_hive00") and Property="type" order by Value"""
+
+ sql """ drop catalog catalog_test_hive00 """
+
+ qt_delete """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName="catalog_test_hive00" """
+
+ qt_create """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName in ("catalog_test_es00","catalog_test_hive00") and Property="type" order by Value"""
+
+ sql """ drop catalog catalog_test_es00 """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/18: [Fix](parquet-reader) Fix parquet string column min max statistics issue which caused query result incorrectly. (#21675)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1e9e6afdeb4fe4f2c982263ae63fb123871a5566
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Fri Jul 14 00:09:41 2023 +0800
[Fix](parquet-reader) Fix parquet string column min max statistics issue which caused query result incorrectly. (#21675)
In parquet, min and max statistics may not be able to handle UTF8 correctly.
Current processing method is using min_value and max_value statistics introduced by PARQUET-1025 if they are used.
If not, current processing method is temporarily ignored. A better way is try to read min and max statistics if it contains
only ASCII characters. I will improve it in the future PR.
---
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 14 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 15 ++-
.../hive/test_multi_langs.out | 148 +++++++++++++++++++++
.../hive/test_multi_langs.groovy | 61 +++++++++
4 files changed, 231 insertions(+), 7 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
index 8d3057312b..c76fa95f4a 100644
--- a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
+++ b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
@@ -120,7 +120,7 @@ private:
static bool _filter_by_min_max(const ColumnValueRange<primitive_type>& col_val_range,
const ScanPredicate& predicate, const FieldSchema* col_schema,
const std::string& encoded_min, const std::string& encoded_max,
- const cctz::time_zone& ctz) {
+ const cctz::time_zone& ctz, bool use_min_max_value = false) {
using CppType = typename PrimitiveTypeTraits<primitive_type>::CppType;
std::vector<CppType> predicate_values;
for (const void* v : predicate.values) {
@@ -144,6 +144,13 @@ private:
case TYPE_CHAR:
[[fallthrough]];
case TYPE_STRING:
+ // TODO: In parquet, min and max statistics may not be able to handle UTF8 correctly.
+ // Current processing method is using min_value and max_value statistics introduced by PARQUET-1025 if they are used.
+ // If not, current processing method is temporarily ignored. A better way is try to read min and max statistics
+ // if it contains only ASCII characters.
+ if (!use_min_max_value) {
+ return false;
+ }
if constexpr (std::is_same_v<CppType, StringRef>) {
min_value = StringRef(encoded_min);
max_value = StringRef(encoded_max);
@@ -372,7 +379,8 @@ public:
static bool filter_by_stats(const ColumnValueRangeType& col_val_range,
const FieldSchema* col_schema, bool is_set_min_max,
const std::string& encoded_min, const std::string& encoded_max,
- bool is_all_null, const cctz::time_zone& ctz) {
+ bool is_all_null, const cctz::time_zone& ctz,
+ bool use_min_max_value = false) {
bool need_filter = false;
std::visit(
[&](auto&& range) {
@@ -387,7 +395,7 @@ public:
}
for (auto& filter : filters) {
need_filter |= _filter_by_min_max(range, filter, col_schema, encoded_min,
- encoded_max, ctz);
+ encoded_max, ctz, use_min_max_value);
if (need_filter) {
break;
}
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 9b179384e2..fed33b6d28 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -836,15 +836,22 @@ Status ParquetReader::_process_column_stat_filter(const std::vector<tparquet::Co
auto& statistic = meta_data.statistics;
bool is_all_null =
(statistic.__isset.null_count && statistic.null_count == meta_data.num_values);
- bool is_set_min_max = (statistic.__isset.max && statistic.__isset.min);
+ bool is_set_min_max = (statistic.__isset.max && statistic.__isset.min) ||
+ (statistic.__isset.max_value && statistic.__isset.min_value);
if ((!is_set_min_max) && (!is_all_null)) {
continue;
}
const FieldSchema* col_schema = schema_desc.get_column(col_name);
// Min-max of statistic is plain-encoded value
- *filter_group =
- ParquetPredicate::filter_by_stats(slot_iter->second, col_schema, is_set_min_max,
- statistic.min, statistic.max, is_all_null, *_ctz);
+ if (statistic.__isset.min_value) {
+ *filter_group = ParquetPredicate::filter_by_stats(
+ slot_iter->second, col_schema, is_set_min_max, statistic.min_value,
+ statistic.max_value, is_all_null, *_ctz, true);
+ } else {
+ *filter_group = ParquetPredicate::filter_by_stats(
+ slot_iter->second, col_schema, is_set_min_max, statistic.min, statistic.max,
+ is_all_null, *_ctz, false);
+ }
if (*filter_group) {
break;
}
diff --git a/regression-test/data/external_table_emr_p2/hive/test_multi_langs.out b/regression-test/data/external_table_emr_p2/hive/test_multi_langs.out
new file mode 100644
index 0000000000..bebfc26854
--- /dev/null
+++ b/regression-test/data/external_table_emr_p2/hive/test_multi_langs.out
@@ -0,0 +1,148 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !01 --
+2 是
+
+-- !02 --
+1
+2 是
+3 III类户
+
+-- !03 --
+2 1
+
+-- !04 --
+5 ありがとう
+
+-- !05 --
+1 你好
+2 谢谢
+3 再见
+4 こんにちは
+5 ありがとう
+6 さようなら
+7 안녕하세요
+8 감사합니다
+9 안녕히 가세요
+10 Hola
+11 Gracias
+12 Adiós
+13 Hallo
+14 Danke
+15 Auf Wiedersehen
+16 مرحبا
+17 شكرًا
+18 مع السلامة
+19 Bonjour
+20 Merci
+21 Au revoir
+22 Ciao
+23 Grazie
+24 Arrivederci
+25 Olá
+26 Obrigado
+27 Adeus
+28 Hello
+29 Thank you
+30 Goodbye
+
+-- !06 --
+5 1
+
+-- !01 --
+2 是
+
+-- !02 --
+1
+2 是
+3 III类户
+
+-- !03 --
+2 1
+
+-- !04 --
+5 ありがとう
+
+-- !05 --
+1 你好
+2 谢谢
+3 再见
+4 こんにちは
+5 ありがとう
+6 さようなら
+7 안녕하세요
+8 감사합니다
+9 안녕히 가세요
+10 Hola
+11 Gracias
+12 Adiós
+13 Hallo
+14 Danke
+15 Auf Wiedersehen
+16 مرحبا
+17 شكرًا
+18 مع السلامة
+19 Bonjour
+20 Merci
+21 Au revoir
+22 Ciao
+23 Grazie
+24 Arrivederci
+25 Olá
+26 Obrigado
+27 Adeus
+28 Hello
+29 Thank you
+30 Goodbye
+
+-- !06 --
+5 1
+
+-- !01 --
+2 是
+
+-- !02 --
+1
+2 是
+3 III类户
+
+-- !03 --
+2 1
+
+-- !04 --
+5 ありがとう
+
+-- !05 --
+1 你好
+2 谢谢
+3 再见
+4 こんにちは
+5 ありがとう
+6 さようなら
+7 안녕하세요
+8 감사합니다
+9 안녕히 가세요
+10 Hola
+11 Gracias
+12 Adiós
+13 Hallo
+14 Danke
+15 Auf Wiedersehen
+16 مرحبا
+17 شكرًا
+18 مع السلامة
+19 Bonjour
+20 Merci
+21 Au revoir
+22 Ciao
+23 Grazie
+24 Arrivederci
+25 Olá
+26 Obrigado
+27 Adeus
+28 Hello
+29 Thank you
+30 Goodbye
+
+-- !06 --
+5 1
+
diff --git a/regression-test/suites/external_table_emr_p2/hive/test_multi_langs.groovy b/regression-test/suites/external_table_emr_p2/hive/test_multi_langs.groovy
new file mode 100644
index 0000000000..937fd9039a
--- /dev/null
+++ b/regression-test/suites/external_table_emr_p2/hive/test_multi_langs.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_multi_langs", "p2") {
+
+ def formats = ["_parquet", "_orc", "_text"]
+ def q1 = """select * from test_chineseSUFFIX where col1='是' order by id"""
+ def q2 = """select * from test_chineseSUFFIX order by id"""
+ def q3 = """select id, count(col1) from test_chineseSUFFIX where col1='是' group by id order by id"""
+ def q4 = """select * from test_multi_langsSUFFIX where col1='ありがとう' order by id"""
+ def q5 = """select * from test_multi_langsSUFFIX order by id"""
+ def q6 = """select id, count(col1) from test_multi_langsSUFFIX where col1='ありがとう' group by id order by id"""
+
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+ String catalog_name = "test_multi_langs"
+
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """use multi_catalog;"""
+ logger.info("use multi_catalog")
+
+ for (String format in formats) {
+ logger.info("Process format " + format)
+ qt_01 q1.replace("SUFFIX", format)
+ qt_02 q2.replace("SUFFIX", format)
+ qt_03 q3.replace("SUFFIX", format)
+ qt_04 q4.replace("SUFFIX", format)
+ qt_05 q5.replace("SUFFIX", format)
+ qt_06 q6.replace("SUFFIX", format)
+ }
+ sql """drop catalog if exists ${catalog_name}"""
+ } finally {
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/18: [feature](hudi) support hudi time travel in external table (#21739)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9eedda343b0993a7059abc36a0b8f207678fc04c
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Thu Jul 13 22:30:07 2023 +0800
[feature](hudi) support hudi time travel in external table (#21739)
Support hudi time travel in external table:
```
select * from hudi_table for time as of '20230712221248';
```
PR(https://github.com/apache/doris/pull/15418) supports to take timestamp or version as the snapshot ID in iceberg, but hudi only has timestamp as the snapshot ID. Therefore, when querying hudi table with `for version as of`, error will be thrown like:
```
ERROR 1105 (HY000): errCode = 2, detailMessage = Hudi table only supports timestamp as snapshot ID
```
The supported formats of timestamp in hudi are: 'yyyy-MM-dd HH:mm:ss[.SSS]' or 'yyyy-MM-dd' or 'yyyyMMddHHmmss[SSS]', which is consistent with the [time-travel-query.](https://hudi.apache.org/docs/quick-start-guide#time-travel-query)
## Partitioning Strategies
Before this PR, hudi's partitions need to be synchronized to hive through [hive-sync-tool](https://hudi.apache.org/docs/syncing_metastore/#hive-sync-tool), or by setting very complex synchronization parameters in [spark conf](https://hudi.apache.org/docs/syncing_metastore/#sync-template). These processes are exceptionally complex and unnecessary, unless you want to query hudi data through hive.
In addition, partitions are changed in time travel. We cannot guarantee the correctness of time travel through partition synchronization.
So this PR directly obtain partitions by reading hudi meta information. Caching and updating table partition information through hudi instant timestamp, and reusing Doris' partition pruning.
---
docs/en/docs/lakehouse/multi-catalog/hudi.md | 22 +-
docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md | 22 +-
fe/be-java-extensions/hudi-scanner/pom.xml | 3 +-
.../org/apache/doris/hudi/BaseSplitReader.scala | 8 +-
.../java/org/apache/doris/analysis/TableRef.java | 32 ++-
.../org/apache/doris/analysis/TableSnapshot.java | 6 +-
.../doris/catalog/HiveMetaStoreClientHelper.java | 13 +-
.../java/org/apache/doris/catalog/HudiUtils.java | 34 +++
.../java/org/apache/doris/common/ErrorCode.java | 2 +-
.../org/apache/doris/datasource/CatalogMgr.java | 2 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 21 +-
.../doris/planner/external/FileQueryScanNode.java | 7 +-
.../planner/external/TablePartitionValues.java | 255 +++++++++++++++++++++
.../hudi/HudiCachedPartitionProcessor.java | 131 +++++++++++
.../planner/external/hudi/HudiPartitionMgr.java | 86 +++++++
.../external/hudi/HudiPartitionProcessor.java | 124 ++++++++++
.../doris/planner/external/hudi/HudiScanNode.java | 84 ++++++-
17 files changed, 809 insertions(+), 43 deletions(-)
diff --git a/docs/en/docs/lakehouse/multi-catalog/hudi.md b/docs/en/docs/lakehouse/multi-catalog/hudi.md
index 890e7fd92c..4c46ccb0e1 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hudi.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hudi.md
@@ -29,12 +29,12 @@ under the License.
## Usage
-1. Doris supports Snapshot Query on Copy-on-Write Hudi tables and Read Optimized Query / Snapshot on Merge-on-Read tables. In the future, it will support Incremental Query and Time Travel.
+1. The query types supported by the Hudi table are as follows, and the Incremental Query will be supported in the future.
| Table Type | Supported Query types |
| ---- | ---- |
-| Copy On Write | Snapshot Query |
-| Merge On Read | Snapshot Queries + Read Optimized Queries |
+| Copy On Write | Snapshot Query + Time Travel |
+| Merge On Read | Snapshot Queries + Read Optimized Queries + Time Travel |
2. Doris supports Hive Metastore(Including catalogs compatible with Hive MetaStore, like [AWS Glue](./hive.md)/[Alibaba DLF](./dlf.md)) Catalogs.
@@ -82,3 +82,19 @@ Users can view the perfomace of Java SDK through [profile](../../admin-manual/ht
2. `JavaScanTime`: Time to read data by Java SDK
3. `FillBlockTime`: Time co convert Java column data into C++ column data
4. `GetRecordReaderTime`: Time to create and initialize Hudi Record Reader
+
+## Time Travel
+
+Supports reading snapshots specified in Hudi table.
+
+Every write operation to the Hudi table will generate a new snapshot.
+
+By default, query requests will only read the latest version of the snapshot.
+
+You can use the `FOR TIME AS OF` statement, based on the time of the snapshot to read historical version data. Examples are as follows:
+
+`SELECT * FROM hudi_tbl FOR TIME AS OF "2022-10-07 17:20:37";`
+
+`SELECT * FROM hudi_tbl FOR TIME AS OF "20221007172037";`
+
+Hudi table does not support the `FOR VERSION AS OF` statement. Using this syntax to query the Hudi table will throw an error.
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
index d93c4268d8..228be87420 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hudi.md
@@ -29,12 +29,12 @@ under the License.
## 使用限制
-1. 目前支持 Copy On Write 表的 Snapshot Query，以及 Merge On Read 表的 Snapshot Queries 和 Read Optimized Query。后续将支持 Incremental Query 和 Time Travel。
+1. Hudi 表支持的查询类型如下，后续将支持 Incremental Query。
| 表类型 | 支持的查询类型 |
| ---- | ---- |
-| Copy On Write | Snapshot Query |
-| Merge On Read | Snapshot Queries + Read Optimized Queries |
+| Copy On Write | Snapshot Query + Time Travel |
+| Merge On Read | Snapshot Queries + Read Optimized Queries + Time Travel |
2. 目前支持 Hive Metastore 和兼容 Hive Metastore 类型(例如[AWS Glue](./hive.md)/[Alibaba DLF](./dlf.md))的 Catalog。
@@ -83,3 +83,19 @@ Doris 使用 parquet native reader 读取 COW 表的数据文件，使用 Java S
2. `JavaScanTime`: Java SDK 读取数据的时间
3. `FillBlockTime`: Java 数据拷贝为 C++ 数据的时间
4. `GetRecordReaderTime`: 调用 Java SDK 并创建 Hudi Record Reader 的时间
+
+## Time Travel
+
+支持读取 Hudi 表指定的 Snapshot。
+
+每一次对 Hudi 表的写操作都会产生一个新的快照。
+
+默认情况下，查询请求只会读取最新版本的快照。
+
+可以使用 `FOR TIME AS OF` 语句，根据快照的时间([时间格式](https://hudi.apache.org/docs/quick-start-guide#time-travel-query)和Hudi官网保持一致)读取历史版本的数据。示例如下：
+
+`SELECT * FROM hudi_tbl FOR TIME AS OF "2022-10-07 17:20:37";`
+
+`SELECT * FROM hudi_tbl FOR TIME AS OF "20221007172037";`
+
+Hudi 表不支持 `FOR VERSION AS OF` 语句，使用该语法查询 Hudi 表将抛错。
diff --git a/fe/be-java-extensions/hudi-scanner/pom.xml b/fe/be-java-extensions/hudi-scanner/pom.xml
index 1b19da9887..b27c269c23 100644
--- a/fe/be-java-extensions/hudi-scanner/pom.xml
+++ b/fe/be-java-extensions/hudi-scanner/pom.xml
@@ -36,7 +36,6 @@ under the License.
<sparkbundle.version>3.2</sparkbundle.version>
<hudi.version>0.13.0</hudi.version>
<janino.version>3.0.16</janino.version>
- <fasterxml.jackson.version>2.14.3</fasterxml.jackson.version>
</properties>
<dependencies>
@@ -163,7 +162,7 @@ under the License.
<!-- version of spark's jackson module is error -->
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- <version>${fasterxml.jackson.version}</version>
+ <version>${jackson.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
diff --git a/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala b/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
index e2ac89fab8..a4f67feddf 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
+++ b/fe/be-java-extensions/hudi-scanner/src/main/scala/org/apache/doris/hudi/BaseSplitReader.scala
@@ -211,9 +211,7 @@ abstract class BaseSplitReader(val split: HoodieSplit) {
protected lazy val partitionColumns: Array[String] = tableConfig.getPartitionFields.orElse(Array.empty)
- protected lazy val specifiedQueryTimestamp: Option[String] =
- optParams.get(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key)
- .map(HoodieSqlCommonUtils.formatQueryInstant)
+ protected lazy val specifiedQueryTimestamp: Option[String] = Some(split.instantTime)
private def queryTimestamp: Option[String] =
specifiedQueryTimestamp.orElse(toScalaOption(timeline.lastInstant()).map(_.getTimestamp))
@@ -624,9 +622,7 @@ object BaseSplitReader {
// NOTE: We're including compaction here since it's not considering a "commit" operation
val timeline = metaClient.getCommitsAndCompactionTimeline.filterCompletedInstants
- val specifiedQueryTimestamp: Option[String] =
- split.optParams.get(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key)
- .map(HoodieSqlCommonUtils.formatQueryInstant)
+ val specifiedQueryTimestamp: Option[String] = Some(split.instantTime)
val schemaResolver = new TableSchemaResolver(metaClient)
val internalSchemaOpt = if (!isSchemaEvolutionEnabledOnRead(split.optParams, sparkSession)) {
None
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
index fcfbd39b44..730826fa2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
@@ -21,6 +21,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.HudiUtils;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.external.HMSExternalTable;
@@ -537,15 +538,28 @@ public class TableRef implements ParseNode, Writable {
ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
HMSExternalTable extTable = (HMSExternalTable) this.getTable();
- if (extTable.getDlaType() != HMSExternalTable.DLAType.ICEBERG) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
- }
- if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
- String asOfTime = tableSnapshot.getTime();
- Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
- if (!matcher.matches()) {
- throw new AnalysisException("Invalid datetime string: " + asOfTime);
- }
+ switch (extTable.getDlaType()) {
+ case ICEBERG:
+ if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
+ String asOfTime = tableSnapshot.getTime();
+ Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
+ if (!matcher.matches()) {
+ throw new AnalysisException("Invalid datetime string: " + asOfTime);
+ }
+ }
+ break;
+ case HUDI:
+ if (tableSnapshot.getType() == TableSnapshot.VersionType.VERSION) {
+ throw new AnalysisException("Hudi table only supports timestamp as snapshot ID");
+ }
+ try {
+ tableSnapshot.setTime(HudiUtils.formatQueryInstant(tableSnapshot.getTime()));
+ } catch (Exception e) {
+ throw new AnalysisException("Failed to parse hudi timestamp: " + e.getMessage(), e);
+ }
+ break;
+ default:
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_NONSUPPORT_TIME_TRAVEL_TABLE);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java
index e5d43b6d0b..0a851a9fd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java
@@ -19,7 +19,7 @@ package org.apache.doris.analysis;
/**
* Snapshot read for time travel
- * the version in 2022.12.28 just supports external iceberg table
+ * supports external iceberg/hudi table
*/
public class TableSnapshot {
@@ -55,6 +55,10 @@ public class TableSnapshot {
return time;
}
+ public void setTime(String time) {
+ this.time = time;
+ }
+
public long getVersion() {
return version;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 232ce4f285..fbb3c71b29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -922,10 +922,7 @@ public class HiveMetaStoreClientHelper {
return hudiSchema;
}
- public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
- String hudiBasePath = table.getRemoteTable().getSd().getLocation();
-
- Configuration conf = getConfiguration(table);
+ public static UserGroupInformation getUserGroupInformation(Configuration conf) {
UserGroupInformation ugi = null;
String authentication = conf.get(HdfsResource.HADOOP_SECURITY_AUTHENTICATION, null);
if (AuthType.KERBEROS.getDesc().equals(authentication)) {
@@ -945,6 +942,14 @@ public class HiveMetaStoreClientHelper {
ugi = UserGroupInformation.createRemoteUser(hadoopUserName);
}
}
+ return ugi;
+ }
+
+ public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
+ String hudiBasePath = table.getRemoteTable().getSd().getLocation();
+
+ Configuration conf = getConfiguration(table);
+ UserGroupInformation ugi = getUserGroupInformation(conf);
HoodieTableMetaClient metaClient;
if (ugi != null) {
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java
index 0799f7b137..e52b52ab8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HudiUtils.java
@@ -22,12 +22,18 @@ import org.apache.avro.LogicalType;
import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class HudiUtils {
+ private static final SimpleDateFormat defaultDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+
public static String fromAvroHudiTypeToHiveTypeString(Schema avroSchema) {
Schema.Type columnType = avroSchema.getType();
LogicalType logicalType = avroSchema.getLogicalType();
@@ -166,4 +172,32 @@ public class HudiUtils {
}
return Type.UNSUPPORTED;
}
+
+ /**
+ * Convert different query instant time format to the commit time format.
+ * Currently we support three kinds of instant time format for time travel query:
+ * 1、yyyy-MM-dd HH:mm:ss
+ * 2、yyyy-MM-dd
+ * This will convert to 'yyyyMMdd000000'.
+ * 3、yyyyMMddHHmmss
+ */
+ public static String formatQueryInstant(String queryInstant) throws ParseException {
+ int instantLength = queryInstant.length();
+ if (instantLength == 19 || instantLength == 23) { // for yyyy-MM-dd HH:mm:ss[.SSS]
+ if (instantLength == 19) {
+ queryInstant += ".000";
+ }
+ return HoodieInstantTimeGenerator.getInstantForDateString(queryInstant);
+ } else if (instantLength == HoodieInstantTimeGenerator.SECS_INSTANT_ID_LENGTH
+ || instantLength == HoodieInstantTimeGenerator.MILLIS_INSTANT_ID_LENGTH) { // for yyyyMMddHHmmss[SSS]
+ HoodieActiveTimeline.parseDateFromInstantTime(queryInstant); // validate the format
+ return queryInstant;
+ } else if (instantLength == 10) { // for yyyy-MM-dd
+ return HoodieActiveTimeline.formatDate(defaultDateFormat.parse(queryInstant));
+ } else {
+ throw new IllegalArgumentException("Unsupported query instant time format: " + queryInstant
+ + ", Supported time format are: 'yyyy-MM-dd HH:mm:ss[.SSS]' "
+ + "or 'yyyy-MM-dd' or 'yyyyMMddHHmmss[SSS]'");
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index f773bfd5ff..c827a41314 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -1186,7 +1186,7 @@ public enum ErrorCode {
ERR_TABLE_NAME_LENGTH_LIMIT(5089, new byte[]{'4', '2', '0', '0', '0'}, "Table name length exceeds limit, "
+ "the length of table name '%s' is %d which is greater than the configuration 'table_name_length_limit' (%d)."),
- ERR_NONSUPPORT_TIME_TRAVEL_TABLE(5090, new byte[]{'4', '2', '0', '0', '0'}, "Only iceberg external"
+ ERR_NONSUPPORT_TIME_TRAVEL_TABLE(5090, new byte[]{'4', '2', '0', '0', '0'}, "Only iceberg/hudi external"
+ " table supports time travel in current version"),
ERR_NONSSL_HANDSHAKE_RESPONSE(5091, new byte[] {'4', '2', '0', '0'},
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 96a7c1eae3..3ed74e260b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -128,7 +128,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
catalog.onClose();
nameToCatalog.remove(catalog.getName());
lastDBOfCatalog.remove(catalog.getName());
- Env.getCurrentEnv().getExtMetaCacheMgr().removeCache(catalog.getName());
+ Env.getCurrentEnv().getExtMetaCacheMgr().removeCache(catalog.getId());
if (!Strings.isNullOrEmpty(catalog.getResource())) {
Resource catalogResource = Env.getCurrentEnv().getResourceMgr().getResource(catalog.getResource());
if (catalogResource != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index f5ca819c3b..16ffcc71f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -23,6 +23,8 @@ import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
+import org.apache.doris.planner.external.hudi.HudiPartitionMgr;
+import org.apache.doris.planner.external.hudi.HudiPartitionProcessor;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
@@ -41,9 +43,11 @@ public class ExternalMetaCacheMgr {
private static final Logger LOG = LogManager.getLogger(ExternalMetaCacheMgr.class);
// catalog id -> HiveMetaStoreCache
- private Map<Long, HiveMetaStoreCache> cacheMap = Maps.newConcurrentMap();
+ private final Map<Long, HiveMetaStoreCache> cacheMap = Maps.newConcurrentMap();
// catalog id -> table schema cache
private Map<Long, ExternalSchemaCache> schemaCacheMap = Maps.newHashMap();
+ // hudi partition manager
+ private final HudiPartitionMgr hudiPartitionMgr;
private ExecutorService executor;
public ExternalMetaCacheMgr() {
@@ -51,6 +55,7 @@ public class ExternalMetaCacheMgr {
Config.max_external_cache_loader_thread_pool_size,
Config.max_external_cache_loader_thread_pool_size * 1000,
"ExternalMetaCacheMgr", 120, true);
+ hudiPartitionMgr = HudiPartitionMgr.get(executor);
}
public HiveMetaStoreCache getMetaStoreCache(HMSExternalCatalog catalog) {
@@ -79,13 +84,18 @@ public class ExternalMetaCacheMgr {
return cache;
}
- public void removeCache(String catalogId) {
+ public HudiPartitionProcessor getHudiPartitionProcess(ExternalCatalog catalog) {
+ return hudiPartitionMgr.getPartitionProcessor(catalog);
+ }
+
+ public void removeCache(long catalogId) {
if (cacheMap.remove(catalogId) != null) {
- LOG.info("remove hive metastore cache for catalog {}" + catalogId);
+ LOG.info("remove hive metastore cache for catalog {}", catalogId);
}
if (schemaCacheMap.remove(catalogId) != null) {
- LOG.info("remove schema cache for catalog {}" + catalogId);
+ LOG.info("remove schema cache for catalog {}", catalogId);
}
+ hudiPartitionMgr.removePartitionProcessor(catalogId);
}
public void invalidateTableCache(long catalogId, String dbName, String tblName) {
@@ -98,6 +108,7 @@ public class ExternalMetaCacheMgr {
if (metaCache != null) {
metaCache.invalidateTableCache(dbName, tblName);
}
+ hudiPartitionMgr.cleanTablePartitions(catalogId, dbName, tblName);
LOG.debug("invalid table cache for {}.{} in catalog {}", dbName, tblName, catalogId);
}
@@ -111,6 +122,7 @@ public class ExternalMetaCacheMgr {
if (metaCache != null) {
metaCache.invalidateDbCache(dbName);
}
+ hudiPartitionMgr.cleanDatabasePartitions(catalogId, dbName);
LOG.debug("invalid db cache for {} in catalog {}", dbName, catalogId);
}
@@ -123,6 +135,7 @@ public class ExternalMetaCacheMgr {
if (metaCache != null) {
metaCache.invalidateAll();
}
+ hudiPartitionMgr.cleanPartitionProcess(catalogId);
LOG.debug("invalid catalog cache for {}", catalogId);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index 7a7dd76ab7..09ed6361e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -252,9 +252,10 @@ public abstract class FileQueryScanNode extends FileScanNode {
// set hdfs params for hdfs file type.
Map<String, String> locationProperties = getLocationProperties();
- if (fileFormatType == TFileFormatType.FORMAT_JNI) {
+ if (fileFormatType == TFileFormatType.FORMAT_JNI || locationType == TFileType.FILE_S3) {
scanRangeParams.setProperties(locationProperties);
- } else if (locationType == TFileType.FILE_HDFS || locationType == TFileType.FILE_BROKER) {
+ }
+ if (locationType == TFileType.FILE_HDFS || locationType == TFileType.FILE_BROKER) {
String fsName = getFsName(fileSplit);
THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(locationProperties);
tHdfsParams.setFsName(fsName);
@@ -267,8 +268,6 @@ public abstract class FileQueryScanNode extends FileScanNode {
}
scanRangeParams.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
}
- } else if (locationType == TFileType.FILE_S3) {
- scanRangeParams.setProperties(locationProperties);
}
TScanRangeLocations curLocations = newLocations(scanRangeParams);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/TablePartitionValues.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/TablePartitionValues.java
new file mode 100644
index 0000000000..bcc967501c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/TablePartitionValues.java
@@ -0,0 +1,255 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner.external;
+
+import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.CacheException;
+import org.apache.doris.planner.ColumnBound;
+import org.apache.doris.planner.ListPartitionPrunerV2;
+import org.apache.doris.planner.PartitionPrunerV2Base.UniqueId;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.RangeMap;
+import lombok.Data;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+
+@Data
+public class TablePartitionValues {
+ public static final String HIVE_DEFAULT_PARTITION = "__HIVE_DEFAULT_PARTITION__";
+
+ private final ReadWriteLock readWriteLock;
+ private long lastUpdateTimestamp;
+ private long nextPartitionId;
+ private final Map<Long, PartitionItem> idToPartitionItem;
+ private final Map<String, Long> partitionNameToIdMap;
+ private final Map<Long, String> partitionIdToNameMap;
+
+ private Map<Long, List<UniqueId>> idToUniqueIdsMap;
+ private Map<Long, List<String>> partitionValuesMap;
+ //multi pair
+ private Map<UniqueId, Range<PartitionKey>> uidToPartitionRange;
+ private Map<Range<PartitionKey>, UniqueId> rangeToId;
+ //single pair
+ private RangeMap<ColumnBound, UniqueId> singleColumnRangeMap;
+ private Map<UniqueId, Range<ColumnBound>> singleUidToColumnRangeMap;
+
+ public TablePartitionValues() {
+ readWriteLock = new ReentrantReadWriteLock();
+ lastUpdateTimestamp = 0;
+ nextPartitionId = 0;
+ idToPartitionItem = new HashMap<>();
+ partitionNameToIdMap = new HashMap<>();
+ partitionIdToNameMap = new HashMap<>();
+ }
+
+ public TablePartitionValues(List<String> partitionNames, List<List<String>> partitionValues, List<Type> types) {
+ this();
+ addPartitions(partitionNames, partitionValues, types);
+ }
+
+ public TablePartitionValues(List<String> partitionNames, List<Type> types) {
+ this();
+ addPartitions(partitionNames, types);
+ }
+
+ public void addPartitions(List<String> partitionNames, List<List<String>> partitionValues, List<Type> types) {
+ Preconditions.checkState(partitionNames.size() == partitionValues.size());
+ List<String> addPartitionNames = new ArrayList<>();
+ List<PartitionItem> addPartitionItems = new ArrayList<>();
+ partitionNameToIdMap.forEach((partitionName, partitionId) -> {
+ addPartitionNames.add(partitionName);
+ addPartitionItems.add(idToPartitionItem.get(partitionId));
+ });
+
+ for (int i = 0; i < partitionNames.size(); i++) {
+ if (!partitionNameToIdMap.containsKey(partitionNames.get(i))) {
+ addPartitionNames.add(partitionNames.get(i));
+ addPartitionItems.add(toListPartitionItem(partitionValues.get(i), types));
+ }
+ }
+ cleanPartitions();
+
+ addPartitionItems(addPartitionNames, addPartitionItems, types);
+ }
+
+ public void addPartitions(List<String> partitionNames, List<Type> types) {
+ addPartitions(partitionNames,
+ partitionNames.stream().map(this::getHivePartitionValues).collect(Collectors.toList()), types);
+ }
+
+ private void addPartitionItems(List<String> partitionNames, List<PartitionItem> partitionItems, List<Type> types) {
+ Preconditions.checkState(partitionNames.size() == partitionItems.size());
+ Preconditions.checkState(nextPartitionId == 0);
+ for (int i = 0; i < partitionNames.size(); i++) {
+ long partitionId = nextPartitionId++;
+ idToPartitionItem.put(partitionId, partitionItems.get(i));
+ partitionNameToIdMap.put(partitionNames.get(i), partitionId);
+ partitionIdToNameMap.put(partitionId, partitionNames.get(i));
+ }
+
+ // create a new map for partitionId <---> uniqueId
+ idToUniqueIdsMap = new HashMap<>();
+
+ if (types.size() > 1) {
+ // uidToPartitionRange and rangeToId are only used for multi-column partition
+ uidToPartitionRange = ListPartitionPrunerV2.genUidToPartitionRange(idToPartitionItem, idToUniqueIdsMap);
+ rangeToId = ListPartitionPrunerV2.genRangeToId(uidToPartitionRange);
+ } else {
+ Preconditions.checkState(types.size() == 1);
+ // singleColumnRangeMap is only used for single-column partition
+ singleColumnRangeMap = ListPartitionPrunerV2.genSingleColumnRangeMap(idToPartitionItem, idToUniqueIdsMap);
+ singleUidToColumnRangeMap = ListPartitionPrunerV2.genSingleUidToColumnRange(singleColumnRangeMap);
+ }
+ partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem);
+ }
+
+ public void dropPartitions(List<String> partitionNames, List<Type> types) {
+ partitionNames.forEach(p -> {
+ Long removedPartition = partitionNameToIdMap.get(p);
+ if (removedPartition != null) {
+ idToPartitionItem.remove(removedPartition);
+ }
+ });
+ List<String> remainingPartitionNames = new ArrayList<>();
+ List<PartitionItem> remainingPartitionItems = new ArrayList<>();
+ partitionNameToIdMap.forEach((partitionName, partitionId) -> {
+ remainingPartitionNames.add(partitionName);
+ remainingPartitionItems.add(idToPartitionItem.get(partitionId));
+ });
+ cleanPartitions();
+ addPartitionItems(remainingPartitionNames, remainingPartitionItems, types);
+ }
+
+ public long getLastUpdateTimestamp() {
+ return lastUpdateTimestamp;
+ }
+
+ public void setLastUpdateTimestamp(long lastUpdateTimestamp) {
+ this.lastUpdateTimestamp = lastUpdateTimestamp;
+ }
+
+ public Lock readLock() {
+ return readWriteLock.readLock();
+ }
+
+ public Lock writeLock() {
+ return readWriteLock.writeLock();
+ }
+
+ private void cleanPartitions() {
+ nextPartitionId = 0;
+ idToPartitionItem.clear();
+ partitionNameToIdMap.clear();
+ partitionIdToNameMap.clear();
+
+ idToUniqueIdsMap = null;
+ partitionValuesMap = null;
+ uidToPartitionRange = null;
+ rangeToId = null;
+ singleColumnRangeMap = null;
+ singleUidToColumnRangeMap = null;
+ }
+
+ private ListPartitionItem toListPartitionItem(List<String> partitionValues, List<Type> types) {
+ Preconditions.checkState(partitionValues.size() == types.size());
+ try {
+ PartitionKey key = PartitionKey.createListPartitionKeyWithTypes(
+ partitionValues.stream().map(p -> new PartitionValue(p, HIVE_DEFAULT_PARTITION.equals(p)))
+ .collect(Collectors.toList()),
+ types);
+ return new ListPartitionItem(Lists.newArrayList(key));
+ } catch (AnalysisException e) {
+ throw new CacheException("failed to convert partition %s to list partition",
+ e, partitionValues);
+ }
+ }
+
+ private List<String> getHivePartitionValues(String partitionName) {
+ // Partition name will be in format: nation=cn/city=beijing
+ // parse it to get values "cn" and "beijing"
+ return Arrays.stream(partitionName.split("/")).map(part -> {
+ String[] kv = part.split("=");
+ Preconditions.checkState(kv.length == 2, partitionName);
+ String partitionValue;
+ try {
+ // hive partition value maybe contains special characters like '=' and '/'
+ partitionValue = URLDecoder.decode(kv[1], StandardCharsets.UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
+ // It should not be here
+ throw new RuntimeException(e);
+ }
+ return partitionValue;
+ }).collect(Collectors.toList());
+ }
+
+ @Data
+ public static class TablePartitionKey {
+ private String dbName;
+ private String tblName;
+ // not in key
+ private List<Type> types;
+
+ public TablePartitionKey(String dbName, String tblName, List<Type> types) {
+ this.dbName = dbName;
+ this.tblName = tblName;
+ this.types = types;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof TablePartitionKey)) {
+ return false;
+ }
+ return dbName.equals(((TablePartitionKey) obj).dbName)
+ && tblName.equals(((TablePartitionKey) obj).tblName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dbName, tblName);
+ }
+
+ @Override
+ public String toString() {
+ return "TablePartitionKey{" + "dbName='" + dbName + '\'' + ", tblName='" + tblName + '\'' + '}';
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
new file mode 100644
index 0000000000..ab6a8839b5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiCachedPartitionProcessor.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner.external.hudi;
+
+import org.apache.doris.catalog.external.HMSExternalTable;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.CacheException;
+import org.apache.doris.planner.external.TablePartitionValues;
+import org.apache.doris.planner.external.TablePartitionValues.TablePartitionKey;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.Option;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+public class HudiCachedPartitionProcessor extends HudiPartitionProcessor {
+ private final long catalogId;
+ private final LoadingCache<TablePartitionKey, TablePartitionValues> partitionCache;
+
+ public HudiCachedPartitionProcessor(long catalogId, Executor executor) {
+ this.catalogId = catalogId;
+ this.partitionCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_table_cache_num)
+ .expireAfterAccess(Config.external_cache_expire_time_minutes_after_access, TimeUnit.MINUTES)
+ .build(CacheLoader.asyncReloading(
+ new CacheLoader<TablePartitionKey, TablePartitionValues>() {
+ @Override
+ public TablePartitionValues load(TablePartitionKey key) throws Exception {
+ return new TablePartitionValues();
+ }
+ }, executor));
+ }
+
+ @Override
+ public void cleanUp() {
+ partitionCache.cleanUp();
+ }
+
+ @Override
+ public void cleanDatabasePartitions(String dbName) {
+ partitionCache.asMap().keySet().stream().filter(k -> k.getDbName().equals(dbName)).collect(Collectors.toList())
+ .forEach(partitionCache::invalidate);
+
+ }
+
+ @Override
+ public void cleanTablePartitions(String dbName, String tblName) {
+ partitionCache.asMap().keySet().stream()
+ .filter(k -> k.getDbName().equals(dbName) && k.getTblName().equals(tblName))
+ .collect(Collectors.toList())
+ .forEach(partitionCache::invalidate);
+ }
+
+ public TablePartitionValues getPartitionValues(HMSExternalTable table, HoodieTableMetaClient tableMetaClient)
+ throws CacheException {
+ assert (catalogId == table.getCatalog().getId());
+ Option<String[]> partitionColumns = tableMetaClient.getTableConfig().getPartitionFields();
+ if (!partitionColumns.isPresent()) {
+ return null;
+ }
+ HoodieTimeline timeline = tableMetaClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
+ Option<HoodieInstant> lastInstant = timeline.lastInstant();
+ if (!lastInstant.isPresent()) {
+ return null;
+ }
+ try {
+ long lastTimestamp = Long.parseLong(lastInstant.get().getTimestamp());
+ TablePartitionValues partitionValues = partitionCache.get(
+ new TablePartitionKey(table.getDbName(), table.getName(), table.getPartitionColumnTypes()));
+ partitionValues.readLock().lock();
+ try {
+ long lastUpdateTimestamp = partitionValues.getLastUpdateTimestamp();
+ if (lastTimestamp == lastUpdateTimestamp) {
+ return partitionValues;
+ }
+ assert (lastTimestamp > lastUpdateTimestamp);
+ } finally {
+ partitionValues.readLock().unlock();
+ }
+
+ partitionValues.writeLock().lock();
+ try {
+ long lastUpdateTimestamp = partitionValues.getLastUpdateTimestamp();
+ if (lastTimestamp == lastUpdateTimestamp) {
+ return partitionValues;
+ }
+ assert (lastTimestamp > lastUpdateTimestamp);
+ List<String> partitionNames;
+ if (lastUpdateTimestamp == 0) {
+ partitionNames = getAllPartitionNames(tableMetaClient);
+ } else {
+ partitionNames = getPartitionNamesInRange(timeline, String.valueOf(lastUpdateTimestamp),
+ String.valueOf(lastTimestamp));
+ }
+ List<String> partitionColumnsList = Arrays.asList(partitionColumns.get());
+ partitionValues.addPartitions(partitionNames,
+ partitionNames.stream().map(p -> parsePartitionValues(partitionColumnsList, p))
+ .collect(Collectors.toList()), table.getPartitionColumnTypes());
+ partitionValues.setLastUpdateTimestamp(lastTimestamp);
+ return partitionValues;
+ } finally {
+ partitionValues.writeLock().unlock();
+ }
+ } catch (Exception e) {
+ throw new CacheException("Failed to get hudi partitions", e);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionMgr.java
new file mode 100644
index 0000000000..4956a6f58c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionMgr.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner.external.hudi;
+
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.HMSExternalCatalog;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+import java.util.concurrent.Executor;
+
+public class HudiPartitionMgr {
+ private static volatile HudiPartitionMgr partitionMgr = null;
+
+ private static final Map<Long, HudiPartitionProcessor> partitionProcessors = Maps.newConcurrentMap();
+ private final Executor executor;
+
+ private HudiPartitionMgr(Executor executor) {
+ this.executor = executor;
+ }
+
+ public HudiPartitionProcessor getPartitionProcessor(ExternalCatalog catalog) {
+ return partitionProcessors.computeIfAbsent(catalog.getId(), catalogId -> {
+ if (catalog instanceof HMSExternalCatalog) {
+ return new HudiCachedPartitionProcessor(catalogId, executor);
+ } else {
+ throw new RuntimeException("Hudi only supports hive(or compatible) catalog now");
+ }
+ });
+ }
+
+ public void removePartitionProcessor(long catalogId) {
+ HudiPartitionProcessor processor = partitionProcessors.remove(catalogId);
+ if (processor != null) {
+ processor.cleanUp();
+ }
+ }
+
+ public void cleanPartitionProcess(long catalogId) {
+ HudiPartitionProcessor processor = partitionProcessors.get(catalogId);
+ if (processor != null) {
+ processor.cleanUp();
+ }
+ }
+
+ public void cleanDatabasePartitions(long catalogId, String dbName) {
+ HudiPartitionProcessor processor = partitionProcessors.get(catalogId);
+ if (processor != null) {
+ processor.cleanDatabasePartitions(dbName);
+ }
+ }
+
+ public void cleanTablePartitions(long catalogId, String dbName, String tblName) {
+ HudiPartitionProcessor processor = partitionProcessors.get(catalogId);
+ if (processor != null) {
+ processor.cleanTablePartitions(dbName, tblName);
+ }
+ }
+
+ public static HudiPartitionMgr get(Executor executor) {
+ if (partitionMgr == null) {
+ synchronized (HudiPartitionMgr.class) {
+ if (partitionMgr == null) {
+ partitionMgr = new HudiPartitionMgr(executor);
+ }
+ }
+ }
+ return partitionMgr;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionProcessor.java
new file mode 100644
index 0000000000..3be3e1f080
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiPartitionProcessor.java
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner.external.hudi;
+
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineUtils;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public abstract class HudiPartitionProcessor {
+
+ public abstract void cleanUp();
+
+ public abstract void cleanDatabasePartitions(String dbName);
+
+ public abstract void cleanTablePartitions(String dbName, String tblName);
+
+ public String[] getPartitionColumns(HoodieTableMetaClient tableMetaClient) {
+ return tableMetaClient.getTableConfig().getPartitionFields().get();
+ }
+
+ public List<String> getAllPartitionNames(HoodieTableMetaClient tableMetaClient) throws IOException {
+ TypedProperties configProperties = new TypedProperties();
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
+ .fromProperties(configProperties)
+ .enable(configProperties.getBoolean(HoodieMetadataConfig.ENABLE.key(),
+ HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS)
+ && HoodieTableMetadataUtil.isFilesPartitionAvailable(tableMetaClient))
+ .build();
+
+ HoodieTableMetadata newTableMetadata = HoodieTableMetadata.create(
+ new HoodieLocalEngineContext(tableMetaClient.getHadoopConf()), metadataConfig,
+ tableMetaClient.getBasePathV2().toString(),
+ FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(), true);
+
+ return newTableMetadata.getPartitionPathWithPathPrefixes(Collections.singletonList(""));
+ }
+
+ public List<String> getPartitionNamesInRange(HoodieTimeline timeline, String startTimestamp, String endTimestamp) {
+ return new ArrayList<>(HoodieInputFormatUtils.getWritePartitionPaths(
+ timeline.findInstantsInRange(startTimestamp, endTimestamp).getInstants().stream()
+ .map(instant -> {
+ try {
+ return TimelineUtils.getCommitMetadata(instant, timeline);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }).collect(Collectors.toList())));
+ }
+
+ public static List<String> parsePartitionValues(List<String> partitionColumns, String partitionPath) {
+ if (partitionColumns.size() == 0) {
+ // This is a non-partitioned table
+ return Collections.emptyList();
+ }
+ String[] partitionFragments = partitionPath.split("/");
+ if (partitionFragments.length != partitionColumns.size()) {
+ if (partitionColumns.size() == 1) {
+ // If the partition column size is not equal to the partition fragment size
+ // and the partition column size is 1, we map the whole partition path
+ // to the partition column which can benefit from the partition prune.
+ String prefix = partitionColumns.get(0) + "=";
+ String partitionValue;
+ if (partitionPath.startsWith(prefix)) {
+ // support hive style partition path
+ partitionValue = partitionPath.substring(prefix.length());
+ } else {
+ partitionValue = partitionPath;
+ }
+ // TODO: In hive, the specific characters like '=', '/' will be url encoded
+ return Collections.singletonList(partitionValue);
+ } else {
+ // If the partition column size is not equal to the partition fragments size
+ // and the partition column size > 1, we do not know how to map the partition
+ // fragments to the partition columns and therefore return an empty tuple. We don't
+ // fail outright so that in some cases we can fallback to reading the table as non-partitioned
+ // one
+ throw new RuntimeException("Failed to parse partition values of path: " + partitionPath);
+ }
+ } else {
+ // If partitionSeqs.length == partitionSchema.fields.length
+ // Append partition name to the partition value if the
+ // HIVE_STYLE_PARTITIONING is disable.
+ // e.g. convert "/xx/xx/2021/02" to "/xx/xx/year=2021/month=02"
+ List<String> partitionValues = new ArrayList<>(partitionFragments.length);
+ for (int i = 0; i < partitionFragments.length; i++) {
+ String prefix = partitionColumns.get(i) + "=";
+ if (partitionFragments[i].startsWith(prefix)) {
+ partitionValues.add(partitionFragments[i].substring(prefix.length()));
+ } else {
+ partitionValues.add(partitionFragments[i]);
+ }
+ }
+ return partitionValues;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
index 3c4fb0d1fa..734b3943a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
@@ -18,16 +18,21 @@
package org.apache.doris.planner.external.hudi;
import org.apache.doris.analysis.TupleDescriptor;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.HudiUtils;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.hive.HivePartition;
+import org.apache.doris.planner.ListPartitionPrunerV2;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.FileSplit;
import org.apache.doris.planner.external.HiveScanNode;
import org.apache.doris.planner.external.TableFormatType;
+import org.apache.doris.planner.external.TablePartitionValues;
import org.apache.doris.spi.Split;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.thrift.TExplainLevel;
@@ -36,10 +41,12 @@ import org.apache.doris.thrift.TFileRangeDesc;
import org.apache.doris.thrift.THudiFileDesc;
import org.apache.doris.thrift.TTableFormatFileDesc;
+import com.google.common.collect.Lists;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.BaseFile;
@@ -54,7 +61,10 @@ import org.apache.hudi.common.util.Option;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -137,6 +147,53 @@ public class HudiScanNode extends HiveScanNode {
rangeDesc.setTableFormatParams(tableFormatFileDesc);
}
+ private List<HivePartition> getPrunedPartitions(HoodieTableMetaClient metaClient) throws AnalysisException {
+ List<Type> partitionColumnTypes = hmsTable.getPartitionColumnTypes();
+ if (!partitionColumnTypes.isEmpty()) {
+ HudiCachedPartitionProcessor processor = (HudiCachedPartitionProcessor) Env.getCurrentEnv()
+ .getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog());
+ TablePartitionValues partitionValues = processor.getPartitionValues(hmsTable, metaClient);
+ if (partitionValues != null) {
+ // 2. prune partitions by expr
+ partitionValues.readLock().lock();
+ try {
+ Map<Long, PartitionItem> idToPartitionItem = partitionValues.getIdToPartitionItem();
+ this.totalPartitionNum = idToPartitionItem.size();
+ ListPartitionPrunerV2 pruner = new ListPartitionPrunerV2(idToPartitionItem,
+ hmsTable.getPartitionColumns(), columnNameToRange,
+ partitionValues.getUidToPartitionRange(),
+ partitionValues.getRangeToId(),
+ partitionValues.getSingleColumnRangeMap(),
+ true);
+ Collection<Long> filteredPartitionIds = pruner.prune();
+ this.readPartitionNum = filteredPartitionIds.size();
+ // 3. get partitions from cache
+ String dbName = hmsTable.getDbName();
+ String tblName = hmsTable.getName();
+ String inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat();
+ String basePath = metaClient.getBasePathV2().toString();
+ Map<Long, String> partitionIdToNameMap = partitionValues.getPartitionIdToNameMap();
+ Map<Long, List<String>> partitionValuesMap = partitionValues.getPartitionValuesMap();
+ return filteredPartitionIds.stream().map(id -> {
+ String path = basePath + "/" + partitionIdToNameMap.get(id);
+ return new HivePartition(
+ dbName, tblName, false, inputFormat, path, partitionValuesMap.get(id));
+ }).collect(Collectors.toList());
+ } finally {
+ partitionValues.readLock().unlock();
+ }
+ }
+ }
+ // unpartitioned table, create a dummy partition to save location and inputformat,
+ // so that we can unify the interface.
+ HivePartition dummyPartition = new HivePartition(hmsTable.getDbName(), hmsTable.getName(), true,
+ hmsTable.getRemoteTable().getSd().getInputFormat(),
+ hmsTable.getRemoteTable().getSd().getLocation(), null);
+ this.totalPartitionNum = 1;
+ this.readPartitionNum = 1;
+ return Lists.newArrayList(dummyPartition);
+ }
+
@Override
public List<Split> getSplits() throws UserException {
HoodieTableMetaClient hudiClient = HiveMetaStoreClientHelper.getHudiClient(hmsTable);
@@ -173,13 +230,30 @@ public class HudiScanNode extends HiveScanNode {
}
HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants();
- Option<HoodieInstant> latestInstant = timeline.lastInstant();
- if (!latestInstant.isPresent()) {
- return new ArrayList<>();
+ String queryInstant;
+ if (desc.getRef().getTableSnapshot() != null) {
+ queryInstant = desc.getRef().getTableSnapshot().getTime();
+ } else {
+ Option<HoodieInstant> snapshotInstant = timeline.lastInstant();
+ if (!snapshotInstant.isPresent()) {
+ return Collections.emptyList();
+ }
+ queryInstant = snapshotInstant.get().getTimestamp();
}
- String queryInstant = latestInstant.get().getTimestamp();
// Non partition table will get one dummy partition
- List<HivePartition> partitions = getPartitions();
+ UserGroupInformation ugi = HiveMetaStoreClientHelper.getUserGroupInformation(
+ HiveMetaStoreClientHelper.getConfiguration(hmsTable));
+ List<HivePartition> partitions;
+ if (ugi != null) {
+ try {
+ partitions = ugi.doAs(
+ (PrivilegedExceptionAction<List<HivePartition>>) () -> getPrunedPartitions(hudiClient));
+ } catch (Exception e) {
+ throw new UserException(e);
+ }
+ } else {
+ partitions = getPrunedPartitions(hudiClient);
+ }
try {
for (HivePartition partition : partitions) {
String globPath;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/18: [Agg](exec) support aggregation_node limit short circuit (#21767)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4d595d548fcc34492396e9d474edec6429e8fff8
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Fri Jul 14 00:29:19 2023 +0800
[Agg](exec) support aggregation_node limit short circuit (#21767)
---
be/src/vec/exec/vaggregation_node.cpp | 4 ++++
be/src/vec/exec/vaggregation_node.h | 5 +++++
2 files changed, 9 insertions(+)
diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp
index 591ad3e55d..5f021e9c95 100644
--- a/be/src/vec/exec/vaggregation_node.cpp
+++ b/be/src/vec/exec/vaggregation_node.cpp
@@ -151,6 +151,10 @@ Status AggregationNode::init(const TPlanNode& tnode, RuntimeState* state) {
// init aggregate functions
_aggregate_evaluators.reserve(tnode.agg_node.aggregate_functions.size());
+ // In case of : `select * from (select GoodEvent from hits union select CounterID from hits) as h limit 10;`
+ // only union with limit: we can short circuit query the pipeline exec engine.
+ _can_short_circuit =
+ tnode.agg_node.aggregate_functions.empty() && state->enable_pipeline_exec();
TSortInfo dummy;
for (int i = 0; i < tnode.agg_node.aggregate_functions.size(); ++i) {
diff --git a/be/src/vec/exec/vaggregation_node.h b/be/src/vec/exec/vaggregation_node.h
index ce7070b817..b30d1cfa0c 100644
--- a/be/src/vec/exec/vaggregation_node.h
+++ b/be/src/vec/exec/vaggregation_node.h
@@ -897,6 +897,7 @@ private:
std::vector<size_t> _probe_key_sz;
std::vector<AggFnEvaluator*> _aggregate_evaluators;
+ bool _can_short_circuit = false;
// may be we don't have to know the tuple id
TupleId _intermediate_tuple_id;
@@ -1060,6 +1061,10 @@ private:
if (_should_limit_output) {
_reach_limit = _get_hash_table_size() >= _limit;
+ if (_reach_limit && _can_short_circuit) {
+ _can_read = true;
+ return Status::Error<ErrorCode::END_OF_FILE>("");
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 12/18: [enhance](Nereids) Pushdown Project Through OuterJoin. (#21730)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d7960b805365597be6447f71b860eab1e9c472cd
Author: jakevin <ja...@gmail.com>
AuthorDate: Fri Jul 14 11:46:29 2023 +0800
[enhance](Nereids) Pushdown Project Through OuterJoin. (#21730)
PushdownJoinOtherCondition will pushdown expression in condition into project, it will block JoinReorder, so we need to pushdown project to help JoinReorder
---
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +--
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
...a => PushdownProjectThroughInnerOuterJoin.java} | 37 ++++++++++++++++++----
... PushdownProjectThroughInnerOuterJoinTest.java} | 10 +++---
4 files changed, 38 insertions(+), 15 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 6eb68e3ec5..98e42eb311 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -35,7 +35,7 @@ import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssoc;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinAssocProject;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscom;
import org.apache.doris.nereids.rules.exploration.join.OuterJoinLAsscomProject;
-import org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughInnerJoin;
+import org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughInnerOuterJoin;
import org.apache.doris.nereids.rules.exploration.join.PushdownProjectThroughSemiJoin;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
@@ -111,7 +111,7 @@ public class RuleSet {
.add(SemiJoinSemiJoinTransposeProject.INSTANCE)
.add(LogicalJoinSemiJoinTranspose.INSTANCE)
.add(LogicalJoinSemiJoinTransposeProject.INSTANCE)
- .add(PushdownProjectThroughInnerJoin.INSTANCE)
+ .add(PushdownProjectThroughInnerOuterJoin.INSTANCE)
.add(PushdownProjectThroughSemiJoin.INSTANCE)
.add(TransposeAggSemiJoin.INSTANCE)
.add(TransposeAggSemiJoinProject.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index a537eb2cbf..03aa1cf3e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -268,7 +268,7 @@ public enum RuleType {
TRANSPOSE_LOGICAL_AGG_SEMI_JOIN_PROJECT(RuleTypeClass.EXPLORATION),
TRANSPOSE_LOGICAL_JOIN_UNION(RuleTypeClass.EXPLORATION),
PUSH_DOWN_PROJECT_THROUGH_SEMI_JOIN(RuleTypeClass.EXPLORATION),
- PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN(RuleTypeClass.EXPLORATION),
+ PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN(RuleTypeClass.EXPLORATION),
EAGER_COUNT(RuleTypeClass.EXPLORATION),
EAGER_GROUP_BY(RuleTypeClass.EXPLORATION),
EAGER_GROUP_BY_COUNT(RuleTypeClass.EXPLORATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
similarity index 79%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
index bd9d146426..6d01078cdc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoin.java
@@ -33,7 +33,9 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -48,13 +50,15 @@ import java.util.stream.Collectors;
* A B A B
* </pre>
*/
-public class PushdownProjectThroughInnerJoin implements ExplorationRuleFactory {
- public static final PushdownProjectThroughInnerJoin INSTANCE = new PushdownProjectThroughInnerJoin();
+public class PushdownProjectThroughInnerOuterJoin implements ExplorationRuleFactory {
+ public static final PushdownProjectThroughInnerOuterJoin INSTANCE = new PushdownProjectThroughInnerOuterJoin();
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalJoin(logicalProject(innerLogicalJoin()), group())
+ logicalJoin(logicalProject(logicalJoin()), group())
+ .when(j -> j.left().child().getJoinType().isOuterJoin() || j.left().child().getJoinType()
+ .isInnerJoin())
// Just pushdown project with non-column expr like (t.id + 1)
.whenNot(j -> j.left().isAllSlots())
.whenNot(j -> j.left().child().hasJoinHint())
@@ -65,8 +69,10 @@ public class PushdownProjectThroughInnerJoin implements ExplorationRuleFactory {
return null;
}
return topJoin.withChildren(newLeft, topJoin.right());
- }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN),
- logicalJoin(group(), logicalProject(innerLogicalJoin()))
+ }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN),
+ logicalJoin(group(), logicalProject(logicalJoin()))
+ .when(j -> j.right().child().getJoinType().isOuterJoin() || j.right().child().getJoinType()
+ .isInnerJoin())
// Just pushdown project with non-column expr like (t.id + 1)
.whenNot(j -> j.right().isAllSlots())
.whenNot(j -> j.right().child().hasJoinHint())
@@ -77,7 +83,7 @@ public class PushdownProjectThroughInnerJoin implements ExplorationRuleFactory {
return null;
}
return topJoin.withChildren(topJoin.left(), newRight);
- }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_JOIN)
+ }).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN)
);
}
@@ -97,7 +103,24 @@ public class PushdownProjectThroughInnerJoin implements ExplorationRuleFactory {
List<NamedExpression> aProjects = new ArrayList<>();
List<NamedExpression> bProjects = new ArrayList<>();
- for (NamedExpression namedExpression : project.getProjects()) {
+ List<NamedExpression> projects;
+ if (join.getJoinType().isInnerJoin()) {
+ projects = project.getProjects();
+ } else {
+ Map<Slot, Slot> childrenSlots = new HashMap<>();
+ join.left().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot));
+ join.right().getOutputSet().forEach(slot -> childrenSlots.put(slot, slot));
+ join.getOutputSet().forEach(slot -> {
+ if (childrenSlots.containsKey(slot)) {
+ childrenSlots.put(slot, childrenSlots.get(slot));
+ }
+ });
+
+ projects = project.getProjects().stream().map(expr -> expr.rewriteUp(e ->
+ e instanceof Slot ? childrenSlots.get((Slot) e) : e
+ )).map(e -> (NamedExpression) e).collect(Collectors.toList());
+ }
+ for (NamedExpression namedExpression : projects) {
Set<ExprId> usedExprIds = namedExpression.getInputSlotExprIds();
if (aOutputExprIdSet.containsAll(usedExprIds)) {
aProjects.add(namedExpression);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
similarity index 94%
rename from fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
index d739039676..19be848332 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushdownProjectThroughInnerOuterJoinTest.java
@@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
-class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported {
+class PushdownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSupported {
private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0);
@@ -59,7 +59,7 @@ class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+ .applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -90,7 +90,7 @@ class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+ .applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -121,7 +121,7 @@ class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+ .applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
@@ -158,7 +158,7 @@ class PushdownProjectThroughInnerJoinTest implements MemoPatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyExploration(PushdownProjectThroughInnerJoin.INSTANCE.buildRules())
+ .applyExploration(PushdownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.checkMemo(memo -> Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/18: [Improvement](multi catalog)Cache file system to improve list remote files performance (#21700)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c3c87ab3dbb1e0cf1083ffdb0c698a5e3f00c70f
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Jul 14 09:59:46 2023 +0800
[Improvement](multi catalog)Cache file system to improve list remote files performance (#21700)
Use file system type and Conf as key to cache remote file system.
This could avoid get a new file system for each external table partition's location.
The time cost for fetching 100000 partitions with 1 file for each partition is reduced to 22s from about 15 minutes.
---
.../main/java/org/apache/doris/common/Config.java | 4 +
.../doris/datasource/ExternalMetaCacheMgr.java | 8 ++
.../doris/datasource/hive/HiveMetaStoreCache.java | 24 ++++--
.../java/org/apache/doris/fs/FileSystemCache.java | 91 ++++++++++++++++++++++
.../org/apache/doris/fs/FileSystemFactory.java | 43 ++++++----
.../java/org/apache/doris/fs/FileSystemType.java | 25 ++++++
.../doris/planner/external/HiveScanNode.java | 2 +-
.../doris/statistics/util/StatisticsUtil.java | 3 +-
8 files changed, 177 insertions(+), 23 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index a7af0a0c3b..373cb777e4 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1700,6 +1700,10 @@ public class Config extends ConfigBase {
"Max number of hive partition values to return while list partitions, -1 means no limitation."})
public static short max_hive_list_partition_num = -1;
+ @ConfField(mutable = false, masterOnly = false, description = {"远程文件系统缓存的最大数量",
+ "Max cache number of remote file system."})
+ public static long max_remote_file_system_cache_num = 100;
+
/**
* Max cache loader thread-pool size.
* Max thread pool size for loading external meta cache
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 16ffcc71f6..57f473f1ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -23,6 +23,7 @@ import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
+import org.apache.doris.fs.FileSystemCache;
import org.apache.doris.planner.external.hudi.HudiPartitionMgr;
import org.apache.doris.planner.external.hudi.HudiPartitionProcessor;
@@ -49,6 +50,8 @@ public class ExternalMetaCacheMgr {
// hudi partition manager
private final HudiPartitionMgr hudiPartitionMgr;
private ExecutorService executor;
+ // all catalogs could share the same fsCache.
+ private FileSystemCache fsCache;
public ExternalMetaCacheMgr() {
executor = ThreadPoolManager.newDaemonFixedThreadPool(
@@ -56,6 +59,7 @@ public class ExternalMetaCacheMgr {
Config.max_external_cache_loader_thread_pool_size * 1000,
"ExternalMetaCacheMgr", 120, true);
hudiPartitionMgr = HudiPartitionMgr.get(executor);
+ fsCache = new FileSystemCache(executor);
}
public HiveMetaStoreCache getMetaStoreCache(HMSExternalCatalog catalog) {
@@ -88,6 +92,10 @@ public class ExternalMetaCacheMgr {
return hudiPartitionMgr.getPartitionProcessor(catalog);
}
+ public FileSystemCache getFsCache() {
+ return fsCache;
+ }
+
public void removeCache(long catalogId) {
if (cacheMap.remove(catalogId) != null) {
LOG.info("remove hive metastore cache for catalog {}", catalogId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 1b5ed54ed1..51653afc68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.backup.Status;
import org.apache.doris.backup.Status.ErrCode;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.catalog.ListPartitionItem;
import org.apache.doris.catalog.PartitionItem;
@@ -37,6 +38,7 @@ import org.apache.doris.datasource.CacheException;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.external.hive.util.HiveUtil;
+import org.apache.doris.fs.FileSystemCache;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.RemoteFiles;
import org.apache.doris.fs.remote.RemoteFile;
@@ -357,11 +359,12 @@ public class HiveMetaStoreCache {
// Get File Status by using FileSystem API.
private FileCacheValue getFileCache(String location, InputFormat<?, ?> inputFormat,
- JobConf jobConf,
- List<String> partitionValues) throws UserException {
+ JobConf jobConf,
+ List<String> partitionValues) throws UserException {
FileCacheValue result = new FileCacheValue();
result.setSplittable(HiveUtil.isSplittable(inputFormat, new Path(location), jobConf));
- RemoteFileSystem fs = FileSystemFactory.getByLocation(location, jobConf);
+ RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
+ new FileSystemCache.FileSystemCacheKey(FileSystemFactory.getLocationType(location), jobConf));
try {
// For Tez engine, it may generate subdirectoies for "union" query.
// So there may be files and directories in the table directory at the same time. eg:
@@ -493,7 +496,8 @@ public class HiveMetaStoreCache {
}
}
- public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions, boolean useSelfSplitter) {
+ public List<FileCacheValue> getFilesByPartitions(List<HivePartition> partitions,
+ boolean useSelfSplitter) {
long start = System.currentTimeMillis();
List<FileCacheKey> keys = partitions.stream().map(p -> {
FileCacheKey fileCacheKey = p.isDummyPartition()
@@ -726,7 +730,7 @@ public class HiveMetaStoreCache {
}
public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions, ValidWriteIdList validWriteIds,
- boolean isFullAcid) {
+ boolean isFullAcid, long tableId) {
List<FileCacheValue> fileCacheValues = Lists.newArrayList();
String remoteUser = jobConf.get(HdfsResource.HADOOP_USER_NAME);
try {
@@ -755,7 +759,9 @@ public class HiveMetaStoreCache {
!directory.getCurrentDirectories().isEmpty() ? directory.getCurrentDirectories().get(0)
.getPath() : null;
String acidVersionPath = new Path(baseOrDeltaPath, "_orc_acid_version").toUri().toString();
- RemoteFileSystem fs = FileSystemFactory.getByLocation(baseOrDeltaPath.toUri().toString(), jobConf);
+ RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
+ new FileSystemCache.FileSystemCacheKey(
+ FileSystemFactory.getLocationType(baseOrDeltaPath.toUri().toString()), jobConf));
Status status = fs.exists(acidVersionPath);
if (status != Status.OK) {
if (status.getErrCode() == ErrCode.NOT_FOUND) {
@@ -775,7 +781,8 @@ public class HiveMetaStoreCache {
List<DeleteDeltaInfo> deleteDeltas = new ArrayList<>();
for (AcidUtils.ParsedDelta delta : directory.getCurrentDirectories()) {
String location = delta.getPath().toString();
- RemoteFileSystem fs = FileSystemFactory.getByLocation(location, jobConf);
+ RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
+ new FileSystemCache.FileSystemCacheKey(FileSystemFactory.getLocationType(location), jobConf));
RemoteFiles locatedFiles = fs.listLocatedFiles(location, true, false);
if (delta.isDeleteDelta()) {
List<String> deleteDeltaFileNames = locatedFiles.files().stream().map(f -> f.getName()).filter(
@@ -792,7 +799,8 @@ public class HiveMetaStoreCache {
// base
if (directory.getBaseDirectory() != null) {
String location = directory.getBaseDirectory().toString();
- RemoteFileSystem fs = FileSystemFactory.getByLocation(location, jobConf);
+ RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem(
+ new FileSystemCache.FileSystemCacheKey(FileSystemFactory.getLocationType(location), jobConf));
RemoteFiles locatedFiles = fs.listLocatedFiles(location, true, false);
locatedFiles.files().stream().filter(
f -> f.getName().startsWith(HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
new file mode 100644
index 0000000000..aa6123d807
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.fs;
+
+import org.apache.doris.common.Config;
+import org.apache.doris.common.util.CacheBulkLoader;
+import org.apache.doris.datasource.CacheException;
+import org.apache.doris.fs.remote.RemoteFileSystem;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.mapred.JobConf;
+
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class FileSystemCache {
+
+ private LoadingCache<FileSystemCacheKey, RemoteFileSystem> fileSystemCache;
+
+ public FileSystemCache(ExecutorService executor) {
+ fileSystemCache = CacheBuilder.newBuilder().maximumSize(Config.max_remote_file_system_cache_num)
+ .expireAfterAccess(Config.external_cache_expire_time_minutes_after_access, TimeUnit.MINUTES)
+ .build(new CacheBulkLoader<FileSystemCacheKey, RemoteFileSystem>() {
+ @Override
+ protected ExecutorService getExecutor() {
+ return executor;
+ }
+
+ @Override
+ public RemoteFileSystem load(FileSystemCacheKey key) {
+ return loadFileSystem(key);
+ }
+ });
+ }
+
+ private RemoteFileSystem loadFileSystem(FileSystemCacheKey key) {
+ return FileSystemFactory.getByType(key.type, key.conf);
+ }
+
+ public RemoteFileSystem getRemoteFileSystem(FileSystemCacheKey key) {
+ try {
+ return fileSystemCache.get(key);
+ } catch (ExecutionException e) {
+ throw new CacheException("failed to get remote filesystem for type[%s]", e, key.type);
+ }
+ }
+
+ public static class FileSystemCacheKey {
+ private final FileSystemType type;
+ private final JobConf conf;
+
+ public FileSystemCacheKey(FileSystemType type, JobConf conf) {
+ this.type = type;
+ this.conf = conf;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof FileSystemCacheKey)) {
+ return false;
+ }
+ return type.equals(((FileSystemCacheKey) obj).type) && conf == ((FileSystemCacheKey) obj).conf;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(conf, type);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index 5ca3e1f21f..73d0c19472 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -19,6 +19,7 @@ package org.apache.doris.fs;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.S3Util;
import org.apache.doris.fs.remote.BrokerFileSystem;
import org.apache.doris.fs.remote.RemoteFileSystem;
import org.apache.doris.fs.remote.S3FileSystem;
@@ -52,23 +53,39 @@ public class FileSystemFactory {
}
}
- public static RemoteFileSystem getByLocation(String location, Configuration conf) {
- // TODO: need optimize the method. the conf is converted many times.
- Map<String, String> properties = new HashMap<>();
- conf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
- if (location.startsWith(FeConstants.FS_PREFIX_S3)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_OBS)) {
- return new S3FileSystem(properties);
+ public static FileSystemType getLocationType(String location) {
+ if (S3Util.isObjStorage(location)) {
+ if (S3Util.isHdfsOnOssEndpoint(location)) {
+ // if hdfs service is enabled on oss, use hdfs lib to access oss.
+ return FileSystemType.DFS;
+ }
+ return FileSystemType.S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) || location.startsWith(FeConstants.FS_PREFIX_GFS)) {
- return new DFSFileSystem(properties);
+ return FileSystemType.DFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
- return new OFSFileSystem(properties);
+ return FileSystemType.OFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
- return new JFSFileSystem(properties);
+ return FileSystemType.JFS;
+ } else {
+ throw new UnsupportedOperationException("Unknown file system for location: " + location);
+ }
+ }
+
+ public static RemoteFileSystem getByType(FileSystemType type, Configuration conf) {
+ Map<String, String> properties = new HashMap<>();
+ conf.iterator().forEachRemaining(e -> properties.put(e.getKey(), e.getValue()));
+ switch (type) {
+ case S3:
+ return new S3FileSystem(properties);
+ case DFS:
+ return new DFSFileSystem(properties);
+ case OFS:
+ return new OFSFileSystem(properties);
+ case JFS:
+ return new JFSFileSystem(properties);
+ default:
+ throw new IllegalStateException("Not supported file system type: " + type);
}
- throw new UnsupportedOperationException("Can not create file system for: " + location);
}
public static RemoteFileSystem getS3FileSystem(Map<String, String> properties) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemType.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemType.java
new file mode 100644
index 0000000000..e3147943c2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemType.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.fs;
+
+public enum FileSystemType {
+ S3,
+ DFS,
+ OFS,
+ JFS
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
index 52bcc67c32..d629914775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanNode.java
@@ -218,7 +218,7 @@ public class HiveScanNode extends FileQueryScanNode {
}
ValidWriteIdList validWriteIds = hiveTransaction.getValidWriteIds(
((HMSExternalCatalog) hmsTable.getCatalog()).getClient());
- return cache.getFilesByTransaction(partitions, validWriteIds, hiveTransaction.isFullAcid());
+ return cache.getFilesByTransaction(partitions, validWriteIds, hiveTransaction.isFullAcid(), hmsTable.getId());
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index eef64ef1c9..6a52bb36b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -577,7 +577,8 @@ public class StatisticsUtil {
table.getRemoteTable().getSd().getLocation(), null));
}
// Get files for all partitions.
- List<HiveMetaStoreCache.FileCacheValue> filesByPartitions = cache.getFilesByPartitions(hivePartitions, true);
+ List<HiveMetaStoreCache.FileCacheValue> filesByPartitions = cache.getFilesByPartitions(
+ hivePartitions, true);
long totalSize = 0;
// Calculate the total file size.
for (HiveMetaStoreCache.FileCacheValue files : filesByPartitions) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 15/18: [Enhancement] (binlog) TBinlog and BinlogManager V2 (#21674)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7e0c9f551efb38880d898b6431db12baf5f390fc
Author: DeadlineFen <11...@users.noreply.github.com>
AuthorDate: Fri Jul 14 16:59:32 2023 +0800
[Enhancement] (binlog) TBinlog and BinlogManager V2 (#21674)
---
.../org/apache/doris/binlog/BinlogComparator.java | 24 ++
.../java/org/apache/doris/binlog/BinlogGcer.java | 5 +-
.../org/apache/doris/binlog/BinlogManager.java | 120 +++++----
.../org/apache/doris/binlog/BinlogTombstone.java | 8 +-
.../java/org/apache/doris/binlog/BinlogUtils.java | 53 ++++
.../java/org/apache/doris/binlog/DBBinlog.java | 298 +++++++++++++++------
.../java/org/apache/doris/binlog/TableBinlog.java | 167 ++++++++----
gensrc/thrift/FrontendService.thrift | 5 +-
8 files changed, 478 insertions(+), 202 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogComparator.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogComparator.java
new file mode 100644
index 0000000000..9e35cc3bd6
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogComparator.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.binlog;
+
+import org.apache.doris.thrift.TBinlog;
+
+public interface BinlogComparator {
+ boolean isExpired(TBinlog binlog, long expired);
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogGcer.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogGcer.java
index a2e6afd1fc..96d41946ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogGcer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogGcer.java
@@ -135,9 +135,8 @@ public class BinlogGcer extends MasterDaemon {
}
}
- private void sendTableGcInfoToBe(Map<Long, BinlogGcTask> beBinlogGcTaskMap, OlapTable table,
+ private void sendTableGcInfoToBe(Map<Long, BinlogGcTask> beBinlogGcTaskMap, OlapTable olapTable,
UpsertRecord.TableRecord tableRecord) {
- OlapTable olapTable = (OlapTable) table;
olapTable.readLock();
try {
@@ -173,7 +172,7 @@ public class BinlogGcer extends MasterDaemon {
}
}
} finally {
- table.readUnlock();
+ olapTable.readUnlock();
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 63e773af4d..b07072955f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -38,15 +38,10 @@ import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TMemoryBuffer;
import org.apache.thrift.transport.TMemoryInputTransport;
-import org.apache.thrift.transport.TTransportException;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -83,7 +78,7 @@ public class BinlogManager {
try {
dbBinlog = dbBinlogMap.get(dbId);
if (dbBinlog == null) {
- dbBinlog = new DBBinlog(dbId);
+ dbBinlog = new DBBinlog(binlog);
dbBinlogMap.put(dbId, dbBinlog);
}
} finally {
@@ -105,6 +100,7 @@ public class BinlogManager {
if (tableIds != null && !tableIds.isEmpty()) {
binlog.setTableIds(tableIds);
}
+ binlog.setTableRef(0);
addBinlog(binlog);
}
@@ -121,7 +117,7 @@ public class BinlogManager {
public void addAddPartitionRecord(AddPartitionRecord addPartitionRecord) {
long dbId = addPartitionRecord.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(addPartitionRecord.getTableId());
long commitSeq = addPartitionRecord.getCommitSeq();
long timestamp = -1;
@@ -133,7 +129,7 @@ public class BinlogManager {
public void addCreateTableRecord(CreateTableRecord createTableRecord) {
long dbId = createTableRecord.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(createTableRecord.getTableId());
long commitSeq = createTableRecord.getCommitSeq();
long timestamp = -1;
@@ -145,7 +141,7 @@ public class BinlogManager {
public void addDropPartitionRecord(DropPartitionInfo dropPartitionInfo, long commitSeq) {
long dbId = dropPartitionInfo.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(dropPartitionInfo.getTableId());
long timestamp = -1;
TBinlogType type = TBinlogType.DROP_PARTITION;
@@ -156,7 +152,7 @@ public class BinlogManager {
public void addDropTableRecord(DropTableRecord record) {
long dbId = record.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(record.getTableId());
long commitSeq = record.getCommitSeq();
long timestamp = -1;
@@ -168,7 +164,7 @@ public class BinlogManager {
public void addAlterJobV2(AlterJobV2 alterJob, long commitSeq) {
long dbId = alterJob.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(alterJob.getTableId());
long timestamp = -1;
TBinlogType type = TBinlogType.ALTER_JOB;
@@ -179,7 +175,7 @@ public class BinlogManager {
public void addModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info, long commitSeq) {
long dbId = info.getDbId();
- List<Long> tableIds = new ArrayList<Long>();
+ List<Long> tableIds = Lists.newArrayList();
tableIds.add(info.getTableId());
long timestamp = -1;
TBinlogType type = TBinlogType.MODIFY_TABLE_ADD_OR_DROP_COLUMNS;
@@ -228,9 +224,9 @@ public class BinlogManager {
LOG.info("begin gc binlog");
lock.writeLock().lock();
- Map<Long, DBBinlog> gcDbBinlogMap = null;
+ Map<Long, DBBinlog> gcDbBinlogMap;
try {
- gcDbBinlogMap = new HashMap<Long, DBBinlog>(dbBinlogMap);
+ gcDbBinlogMap = Maps.newHashMap(dbBinlogMap);
} finally {
lock.writeLock().unlock();
}
@@ -242,9 +238,9 @@ public class BinlogManager {
List<BinlogTombstone> tombstones = Lists.newArrayList();
for (DBBinlog dbBinlog : gcDbBinlogMap.values()) {
- List<BinlogTombstone> dbTombstones = dbBinlog.gc();
+ BinlogTombstone dbTombstones = dbBinlog.gc();
if (dbTombstones != null) {
- tombstones.addAll(dbTombstones);
+ tombstones.add(dbTombstones);
}
}
return tombstones;
@@ -252,9 +248,9 @@ public class BinlogManager {
public void replayGc(BinlogGcInfo binlogGcInfo) {
lock.writeLock().lock();
- Map<Long, DBBinlog> gcDbBinlogMap = null;
+ Map<Long, DBBinlog> gcDbBinlogMap;
try {
- gcDbBinlogMap = new HashMap<Long, DBBinlog>(dbBinlogMap);
+ gcDbBinlogMap = Maps.newHashMap(dbBinlogMap);
} finally {
lock.writeLock().unlock();
}
@@ -309,18 +305,11 @@ public class BinlogManager {
return checksum;
}
- List<TBinlog> binlogs = new ArrayList<TBinlog>();
+ List<TBinlog> binlogs = Lists.newArrayList();
// Step 1: get all binlogs
for (DBBinlog dbBinlog : dbBinlogMap.values()) {
dbBinlog.getAllBinlogs(binlogs);
}
- // sort binlogs by commitSeq
- Collections.sort(binlogs, new Comparator<TBinlog>() {
- @Override
- public int compare(TBinlog o1, TBinlog o2) {
- return Long.compare(o1.getCommitSeq(), o2.getCommitSeq());
- }
- });
// Step 2: write binlogs length
dos.writeInt(binlogs.size());
@@ -339,31 +328,6 @@ public class BinlogManager {
return checksum;
}
- public void read(DataInputStream dis) throws IOException {
- // Step 1: read binlogs length
- int length = dis.readInt();
-
- // Step 2: read all binlogs from dis && add binlog
- TMemoryBuffer buffer;
- TBinaryProtocol protocol;
- try {
- buffer = new TMemoryBuffer(BUFFER_SIZE);
- protocol = new TBinaryProtocol(buffer);
- } catch (TTransportException e) {
- throw new IOException("failed to create TMemoryBuffer");
- }
-
- for (int i = 0; i < length; i++) {
- TBinlog binlog = new TBinlog();
- try {
- binlog.read(protocol);
- } catch (TException e) {
- throw new IOException("failed to read binlog from TMemoryBuffer");
- }
- addBinlog(binlog);
- }
- }
-
public TBinlog readTBinlogFromStream(DataInputStream dis) throws TException, IOException {
// We assume that the first int is the length of the serialized data.
int length = dis.readInt();
@@ -376,19 +340,63 @@ public class BinlogManager {
return binlog;
}
+ // db TBinlogs in file struct:
+ // (tableDummy)TBinlog.belong == tableId, (dbDummy)TBinlog.belong == -1
+ // +---------------------------+------------------+-----------------------------------+
+ // | (tableDummy)TBinlog | ... | (dbDummy)TBinlog | TBinlog | TBinlog | TBinlog | ... |
+ // +---------------------------+------------------+-----------------------------------+
+ // | Unnecessary | Necessary | Unnecessary |
+ // +---------------------------+------------------+-----------------------------------+
public long read(DataInputStream dis, long checksum) throws IOException {
// Step 1: read binlogs length
int size = dis.readInt();
LOG.info("read binlogs length: {}", size);
// Step 2: read all binlogs from dis
- for (int i = 0; i < size; i++) {
- try {
+ long currentDbId = -1;
+ boolean currentDbBinlogEnable = false;
+ List<TBinlog> tableDummies = Lists.newArrayList();
+ try {
+ for (int i = 0; i < size; i++) {
+ // Step 2.1: read a binlog
TBinlog binlog = readTBinlogFromStream(dis);
- addBinlog(binlog);
- } catch (TException e) {
- throw new IOException("failed to read binlog from TMemoryBuffer", e);
+
+ // Step 2.2: check if there is in next db Binlogs region
+ long dbId = binlog.getDbId();
+ if (dbId != currentDbId) {
+ // if there is in next db Binlogs region, check and update metadata
+ Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null) {
+ LOG.warn("db not found. dbId: {}", dbId);
+ continue;
+ }
+ currentDbId = dbId;
+ currentDbBinlogEnable = db.getBinlogConfig().isEnable();
+ tableDummies = Lists.newArrayList();
+ }
+
+ // step 2.3: recover binlog
+ if (binlog.getType() == TBinlogType.DUMMY) {
+ // collect tableDummyBinlogs and dbDummyBinlog to recover DBBinlog and TableBinlog
+ if (binlog.getBelong() == -1) {
+ DBBinlog dbBinlog = DBBinlog.recoverDbBinlog(binlog, tableDummies, currentDbBinlogEnable);
+ dbBinlogMap.put(dbId, dbBinlog);
+ } else {
+ tableDummies.add(binlog);
+ }
+ } else {
+ // recover common binlogs
+ DBBinlog dbBinlog = dbBinlogMap.get(dbId);
+ if (dbBinlog == null) {
+ LOG.warn("dbBinlog recover fail! binlog {} is before dummy. dbId: {}", binlog, dbId);
+ continue;
+ }
+ binlog.setTableRef(0);
+ dbBinlog.recoverBinlog(binlog, currentDbBinlogEnable);
+ }
}
+ } catch (TException e) {
+ throw new IOException("failed to read binlog from TMemoryBuffer", e);
}
return checksum;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogTombstone.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogTombstone.java
index 22e0fd9eba..50d9f90a01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogTombstone.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogTombstone.java
@@ -71,8 +71,8 @@ public class BinlogTombstone {
tableVersionMap.put(tableId, tableRecord);
}
- public void addTableRecord(long tableId, UpsertRecord.TableRecord record) {
- tableVersionMap.put(tableId, record);
+ public void addTableRecord(Map<Long, UpsertRecord.TableRecord> records) {
+ tableVersionMap.putAll(records);
}
public boolean isDbBinlogTomstone() {
@@ -91,6 +91,10 @@ public class BinlogTombstone {
return commitSeq;
}
+ public void setCommitSeq(long seq) {
+ commitSeq = seq;
+ }
+
public Map<Long, UpsertRecord.TableRecord> getTableVersionMap() {
return tableVersionMap;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogUtils.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogUtils.java
index 9742bed23d..3af4053ef1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogUtils.java
@@ -17,14 +17,24 @@
package org.apache.doris.binlog;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
import org.apache.doris.common.Pair;
import org.apache.doris.thrift.TBinlog;
+import org.apache.doris.thrift.TBinlogType;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.TreeSet;
public class BinlogUtils {
+ private static final Logger LOG = LogManager.getLogger(BinlogUtils.class);
+
public static Pair<TStatus, TBinlog> getBinlog(TreeSet<TBinlog> binlogs, long prevCommitSeq) {
TStatus status = new TStatus(TStatusCode.OK);
TBinlog firstBinlog = binlogs.first();
@@ -69,4 +79,47 @@ public class BinlogUtils {
return Pair.of(status, Long.valueOf(binlogs.tailSet(binlog).size()));
}
}
+
+ public static TBinlog newDummyBinlog(long dbId, long tableId) {
+ TBinlog dummy = new TBinlog();
+ dummy.setCommitSeq(-1);
+ dummy.setTimestamp(-1);
+ dummy.setType(TBinlogType.DUMMY);
+ dummy.setDbId(dbId);
+ dummy.setBelong(tableId);
+ return dummy;
+ }
+
+ public static boolean tableEnabledBinlog(long dbId, long tableId) {
+ Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
+ if (db == null) {
+ LOG.error("db not found. dbId: {}", dbId);
+ return false;
+ }
+
+ OlapTable table;
+ try {
+ Table tbl = db.getTableOrMetaException(tableId);
+ if (tbl == null) {
+ LOG.warn("fail to get table. db: {}, table id: {}", db.getFullName(), tableId);
+ return false;
+ }
+ if (!(tbl instanceof OlapTable)) {
+ LOG.warn("table is not olap table. db: {}, table id: {}", db.getFullName(), tableId);
+ return false;
+ }
+ table = (OlapTable) tbl;
+ } catch (Exception e) {
+ LOG.warn("fail to get table. db: {}, table id: {}", db.getFullName(), tableId);
+ return false;
+ }
+
+ return table.getBinlogConfig().isEnable();
+ }
+
+ public static long getExpiredMs(long ttlSeconds) {
+ long currentSeconds = System.currentTimeMillis() / 1000;
+ long expireSeconds = currentSeconds - ttlSeconds;
+ return expireSeconds * 1000;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
index 0f113ff491..58708c8fe6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java
@@ -21,15 +21,17 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Pair;
import org.apache.doris.thrift.TBinlog;
+import org.apache.doris.thrift.TBinlogType;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -52,21 +54,74 @@ public class DBBinlog {
// need UpsertRecord to add timestamps for gc
private List<Pair<Long, Long>> timestamps;
- public DBBinlog(long dbId) {
+ private List<TBinlog> tableDummyBinlogs;
+
+ public DBBinlog(TBinlog binlog) {
lock = new ReentrantReadWriteLock();
- this.dbId = dbId;
+ this.dbId = binlog.getDbId();
+
// allBinlogs treeset order by commitSeq
- allBinlogs = new TreeSet<TBinlog>((o1, o2) -> {
- if (o1.getCommitSeq() < o2.getCommitSeq()) {
- return -1;
- } else if (o1.getCommitSeq() > o2.getCommitSeq()) {
- return 1;
- } else {
- return 0;
+ allBinlogs = Sets.newTreeSet(Comparator.comparingLong(TBinlog::getCommitSeq));
+ tableDummyBinlogs = Lists.newArrayList();
+ tableBinlogMap = Maps.newHashMap();
+ timestamps = Lists.newArrayList();
+
+ TBinlog dummy;
+ if (binlog.getType() == TBinlogType.DUMMY) {
+ dummy = binlog;
+ } else {
+ dummy = BinlogUtils.newDummyBinlog(dbId, -1);
+ }
+ allBinlogs.add(dummy);
+ }
+
+ public static DBBinlog recoverDbBinlog(TBinlog dbDummy, List<TBinlog> tableDummies, boolean dbBinlogEnable) {
+ DBBinlog dbBinlog = new DBBinlog(dbDummy);
+ for (TBinlog tableDummy : tableDummies) {
+ long tableId = tableDummy.getBelong();
+ if (!dbBinlogEnable && !BinlogUtils.tableEnabledBinlog(dbBinlog.getDbId(), tableId)) {
+ continue;
+ }
+ dbBinlog.tableBinlogMap.put(tableId, new TableBinlog(tableDummy, tableId));
+ dbBinlog.tableDummyBinlogs.add(tableDummy);
+ }
+
+ return dbBinlog;
+ }
+
+ // not thread safety, do this without lock
+ public void recoverBinlog(TBinlog binlog, boolean dbBinlogEnable) {
+ List<Long> tableIds = binlog.getTableIds();
+
+ if (binlog.getTimestamp() > 0 && dbBinlogEnable) {
+ timestamps.add(Pair.of(binlog.getCommitSeq(), binlog.getTimestamp()));
+ }
+
+ allBinlogs.add(binlog);
+
+ if (tableIds == null) {
+ return;
+ }
+
+ for (long tableId : tableIds) {
+ TableBinlog tableBinlog = getTableBinlog(binlog, tableId, dbBinlogEnable);
+ if (tableBinlog == null) {
+ continue;
+ }
+ tableBinlog.recoverBinlog(binlog);
+ }
+ }
+
+ private TableBinlog getTableBinlog(TBinlog binlog, long tableId, boolean dbBinlogEnable) {
+ TableBinlog tableBinlog = tableBinlogMap.get(tableId);
+ if (tableBinlog == null) {
+ if (dbBinlogEnable || BinlogUtils.tableEnabledBinlog(dbId, tableId)) {
+ tableBinlog = new TableBinlog(binlog, tableId);
+ tableBinlogMap.put(tableId, tableBinlog);
+ tableDummyBinlogs.add(tableBinlog.getDummyBinlog());
}
- });
- tableBinlogMap = new HashMap<Long, TableBinlog>();
- timestamps = new ArrayList<Pair<Long, Long>>();
+ }
+ return tableBinlog;
}
public void addBinlog(TBinlog binlog, boolean dbBinlogEnable) {
@@ -97,16 +152,21 @@ public class DBBinlog {
}
for (long tableId : tableIds) {
- TableBinlog tableBinlog = tableBinlogMap.get(tableId);
- if (tableBinlog == null) {
- tableBinlog = new TableBinlog(tableId);
- tableBinlogMap.put(tableId, tableBinlog);
+ TableBinlog tableBinlog = getTableBinlog(binlog, tableId, dbBinlogEnable);
+ if (tableBinlog != null) {
+ tableBinlog.addBinlog(binlog);
}
- tableBinlog.addBinlog(binlog);
}
} finally {
lock.writeLock().unlock();
}
+
+ lock.readLock().lock();
+ try {
+ LOG.info("[deadlinefen] after add, db {} binlogs: {}, dummys: {}", dbId, allBinlogs, tableDummyBinlogs);
+ } finally {
+ lock.readLock().unlock();
+ }
}
public long getDbId() {
@@ -151,7 +211,7 @@ public class DBBinlog {
}
}
- public List<BinlogTombstone> gc() {
+ public BinlogTombstone gc() {
// check db
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
if (db == null) {
@@ -160,33 +220,55 @@ public class DBBinlog {
}
boolean dbBinlogEnable = db.getBinlogConfig().isEnable();
+ BinlogTombstone tombstone;
if (dbBinlogEnable) {
- // db binlog is enable, only one binlogTombstones
+ // db binlog is enabled, only one binlogTombstones
long ttlSeconds = db.getBinlogConfig().getTtlSeconds();
- long currentSeconds = System.currentTimeMillis() / 1000;
- long expireSeconds = currentSeconds - ttlSeconds;
- long expireMs = expireSeconds * 1000;
+ long expiredMs = BinlogUtils.getExpiredMs(ttlSeconds);
- BinlogTombstone tombstone = dbBinlogEnableGc(expireMs);
- List<BinlogTombstone> tombstones = new ArrayList<BinlogTombstone>();
- if (tombstone != null) {
- tombstones.add(tombstone);
- }
- return tombstones;
+ tombstone = dbBinlogEnableGc(expiredMs);
} else {
- return dbBinlogDisableGc(db);
+ tombstone = dbBinlogDisableGc(db);
}
+
+ return tombstone;
}
- private List<BinlogTombstone> dbBinlogDisableGc(Database db) {
- List<BinlogTombstone> tombstones = new ArrayList<BinlogTombstone>();
- List<TableBinlog> tableBinlogs = null;
+ private BinlogTombstone collectTableTombstone(List<BinlogTombstone> tableTombstones) {
+ if (tableTombstones.isEmpty()) {
+ return null;
+ }
- lock.writeLock().lock();
+ List<Long> tableIds = Lists.newArrayList();
+ long largestExpiredCommitSeq = -1;
+ BinlogTombstone dbTombstone = new BinlogTombstone(dbId, tableIds, -1);
+ for (BinlogTombstone tableTombstone : tableTombstones) {
+ long commitSeq = tableTombstone.getCommitSeq();
+ if (largestExpiredCommitSeq < commitSeq) {
+ largestExpiredCommitSeq = commitSeq;
+ }
+ Map<Long, UpsertRecord.TableRecord> tableVersionMap = tableTombstone.getTableVersionMap();
+ if (tableVersionMap.size() > 1) {
+ LOG.warn("tableVersionMap size is greater than 1. tableVersionMap: {}", tableVersionMap);
+ }
+ tableIds.addAll(tableTombstone.getTableIds());
+ dbTombstone.addTableRecord(tableVersionMap);
+ }
+
+ dbTombstone.setCommitSeq(largestExpiredCommitSeq);
+
+ return dbTombstone;
+ }
+
+ private BinlogTombstone dbBinlogDisableGc(Database db) {
+ List<BinlogTombstone> tombstones = Lists.newArrayList();
+ List<TableBinlog> tableBinlogs;
+
+ lock.readLock().lock();
try {
- tableBinlogs = new ArrayList<TableBinlog>(tableBinlogMap.values());
+ tableBinlogs = Lists.newArrayList(tableBinlogMap.values());
} finally {
- lock.writeLock().unlock();
+ lock.readLock().unlock();
}
for (TableBinlog tableBinlog : tableBinlogs) {
@@ -195,70 +277,110 @@ public class DBBinlog {
tombstones.add(tombstone);
}
}
- return tombstones;
- }
+ BinlogTombstone tombstone = collectTableTombstone(tombstones);
+ if (tombstone != null) {
+ removeExpiredMetaData(tombstone.getCommitSeq());
+
+ lock.readLock().lock();
+ try {
+ LOG.info("[deadlinefen] after gc, db {} binlogs: {}, tombstone.seq: {}, dummys: {}",
+ dbId, allBinlogs, tombstone.getCommitSeq(), tableDummyBinlogs);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
- private BinlogTombstone dbBinlogEnableGc(long expireMs) {
- // find commitSeq from timestamps, if commitSeq's timestamp is less than expireSeconds, then remove it
- long largestExpiredCommitSeq = -1;
- TBinlog tombstoneBinlog = null;
- List<Long> tableIds = null;
- List<TableBinlog> tableBinlogs = null;
+ return tombstone;
+ }
+ private void removeExpiredMetaData(long largestExpiredCommitSeq) {
lock.writeLock().lock();
try {
- Iterator<Pair<Long, Long>> iterator = timestamps.iterator();
- while (iterator.hasNext()) {
- Pair<Long, Long> pair = iterator.next();
- if (pair.second < expireMs) {
- largestExpiredCommitSeq = pair.first;
- iterator.remove();
+ Iterator<TBinlog> binlogIter = allBinlogs.iterator();
+ TBinlog dummy = binlogIter.next();
+ boolean foundFirstUsingBinlog = false;
+ long lastCommitSeq = -1;
+
+ while (binlogIter.hasNext()) {
+ TBinlog binlog = binlogIter.next();
+ long commitSeq = binlog.getCommitSeq();
+ if (commitSeq <= largestExpiredCommitSeq) {
+ if (binlog.table_ref <= 0) {
+ binlogIter.remove();
+ if (!foundFirstUsingBinlog) {
+ lastCommitSeq = commitSeq;
+ }
+ } else {
+ foundFirstUsingBinlog = true;
+ }
} else {
break;
}
}
- Iterator<TBinlog> binlogIterator = allBinlogs.iterator();
- while (binlogIterator.hasNext()) {
- TBinlog binlog = binlogIterator.next();
- if (binlog.getCommitSeq() <= largestExpiredCommitSeq) {
- tombstoneBinlog = binlog;
- binlogIterator.remove();
+ if (lastCommitSeq != -1) {
+ dummy.setCommitSeq(lastCommitSeq);
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private BinlogTombstone dbBinlogEnableGc(long expiredMs) {
+ // step 1: get current tableBinlog info and expiredCommitSeq
+ long expiredCommitSeq = -1;
+ lock.readLock().lock();
+ try {
+ Iterator<Pair<Long, Long>> timeIter = timestamps.iterator();
+ while (timeIter.hasNext()) {
+ Pair<Long, Long> pair = timeIter.next();
+ if (pair.second <= expiredMs) {
+ expiredCommitSeq = pair.first;
+ timeIter.remove();
} else {
break;
}
}
- tableIds = new ArrayList<Long>(tableBinlogMap.keySet());
- tableBinlogs = new ArrayList<TableBinlog>(tableBinlogMap.values());
+ Iterator<TBinlog> binlogIter = allBinlogs.iterator();
+ TBinlog dummy = binlogIter.next();
+ dummy.setCommitSeq(expiredCommitSeq);
+
+ while (binlogIter.hasNext()) {
+ TBinlog binlog = binlogIter.next();
+ if (binlog.getCommitSeq() <= expiredCommitSeq) {
+ binlogIter.remove();
+ } else {
+ break;
+ }
+ }
} finally {
- lock.writeLock().unlock();
+ lock.readLock().unlock();
}
- LOG.info("gc binlog. dbId: {}, expireMs: {}, largestExpiredCommitSeq: {}",
- dbId, expireMs, largestExpiredCommitSeq);
- if (tombstoneBinlog == null) {
+
+ if (expiredCommitSeq == -1) {
return null;
}
- BinlogTombstone tombstone = new BinlogTombstone(dbId, tableIds, tombstoneBinlog.getCommitSeq());
- for (TableBinlog tableBinlog : tableBinlogs) {
- BinlogTombstone binlogTombstone = tableBinlog.gc(largestExpiredCommitSeq);
- if (binlogTombstone == null) {
- continue;
+ // step 2: gc every tableBinlog in dbBinlog, get table tombstone to complete db tombstone
+ List<BinlogTombstone> tableTombstones = Lists.newArrayList();
+ for (TableBinlog tableBinlog : tableBinlogMap.values()) {
+ // step 2.1: gc tableBinlog，and get table tombstone
+ BinlogTombstone tableTombstone = tableBinlog.gc(expiredCommitSeq);
+ if (tableTombstone != null) {
+ tableTombstones.add(tableTombstone);
}
+ }
- Map<Long, UpsertRecord.TableRecord> tableVersionMap = binlogTombstone.getTableVersionMap();
- if (tableVersionMap.size() > 1) {
- LOG.warn("tableVersionMap size is greater than 1. tableVersionMap: {}", tableVersionMap);
- }
- for (Map.Entry<Long, UpsertRecord.TableRecord> entry : tableVersionMap.entrySet()) {
- long tableId = entry.getKey();
- UpsertRecord.TableRecord record = entry.getValue();
- tombstone.addTableRecord(tableId, record);
- }
+ lock.readLock().lock();
+ try {
+ LOG.info("[deadlinefen] after gc, db {} binlogs: {}, tombstone.seq: {}, dummys: {}",
+ dbId, allBinlogs, expiredCommitSeq, tableDummyBinlogs);
+ } finally {
+ lock.readLock().unlock();
}
- return tombstone;
+ return collectTableTombstone(tableTombstones);
}
public void replayGc(BinlogTombstone tombstone) {
@@ -266,6 +388,7 @@ public class DBBinlog {
dbBinlogEnableReplayGc(tombstone);
} else {
dbBinlogDisableReplayGc(tombstone);
+ removeExpiredMetaData(tombstone.getCommitSeq());
}
}
@@ -274,11 +397,11 @@ public class DBBinlog {
lock.writeLock().lock();
try {
- Iterator<Pair<Long, Long>> iterator = timestamps.iterator();
- while (iterator.hasNext()) {
- Pair<Long, Long> pair = iterator.next();
- if (pair.first <= largestExpiredCommitSeq) {
- iterator.remove();
+ Iterator<Pair<Long, Long>> timeIter = timestamps.iterator();
+ while (timeIter.hasNext()) {
+ long commitSeq = timeIter.next().first;
+ if (commitSeq <= largestExpiredCommitSeq) {
+ timeIter.remove();
} else {
break;
}
@@ -301,11 +424,11 @@ public class DBBinlog {
}
public void dbBinlogDisableReplayGc(BinlogTombstone tombstone) {
- List<TableBinlog> tableBinlogs = null;
+ List<TableBinlog> tableBinlogs;
lock.writeLock().lock();
try {
- tableBinlogs = new ArrayList<TableBinlog>(tableBinlogMap.values());
+ tableBinlogs = Lists.newArrayList(tableBinlogMap.values());
} finally {
lock.writeLock().unlock();
}
@@ -314,7 +437,7 @@ public class DBBinlog {
return;
}
- Set<Long> tableIds = new HashSet<Long>(tombstone.getTableIds());
+ Set<Long> tableIds = Sets.newHashSet(tombstone.getTableIds());
long largestExpiredCommitSeq = tombstone.getCommitSeq();
for (TableBinlog tableBinlog : tableBinlogs) {
if (tableIds.contains(tableBinlog.getTableId())) {
@@ -325,6 +448,7 @@ public class DBBinlog {
// not thread safety, do this without lock
public void getAllBinlogs(List<TBinlog> binlogs) {
+ binlogs.addAll(tableDummyBinlogs);
binlogs.addAll(allBinlogs);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java
index 2b0d45b694..9b82272f63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/TableBinlog.java
@@ -25,9 +25,11 @@ import org.apache.doris.thrift.TBinlog;
import org.apache.doris.thrift.TBinlogType;
import org.apache.doris.thrift.TStatus;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.TreeSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -39,29 +41,43 @@ public class TableBinlog {
private ReentrantReadWriteLock lock;
private TreeSet<TBinlog> binlogs;
- public TableBinlog(long tableId) {
+ public TableBinlog(TBinlog binlog, long tableId) {
this.tableId = tableId;
lock = new ReentrantReadWriteLock();
- // binlogs treeset order by commitSeq
- binlogs = new TreeSet<TBinlog>((o1, o2) -> {
- if (o1.getCommitSeq() < o2.getCommitSeq()) {
- return -1;
- } else if (o1.getCommitSeq() > o2.getCommitSeq()) {
- return 1;
- } else {
- return 0;
- }
- });
+ binlogs = Sets.newTreeSet(Comparator.comparingLong(TBinlog::getCommitSeq));
+
+ TBinlog dummy;
+ if (binlog.getType() == TBinlogType.DUMMY) {
+ dummy = binlog;
+ } else {
+ dummy = BinlogUtils.newDummyBinlog(binlog.getDbId(), tableId);
+ }
+ binlogs.add(dummy);
+ }
+
+ public TBinlog getDummyBinlog() {
+ return binlogs.first();
}
public long getTableId() {
return tableId;
}
+ // not thread safety, do this without lock
+ public void recoverBinlog(TBinlog binlog) {
+ TBinlog dummy = getDummyBinlog();
+ if (binlog.getCommitSeq() > dummy.getCommitSeq()) {
+ binlogs.add(binlog);
+ ++binlog.table_ref;
+ }
+ }
+
public void addBinlog(TBinlog binlog) {
lock.writeLock().lock();
try {
binlogs.add(binlog);
+ ++binlog.table_ref;
+ LOG.info("[deadlinefen] after add, table {} binlogs: {}", tableId, binlogs);
} finally {
lock.writeLock().unlock();
}
@@ -85,41 +101,80 @@ public class TableBinlog {
}
}
- // this method call when db binlog enable
- public BinlogTombstone gc(long largestExpiredCommitSeq) {
+ private Pair<TBinlog, Long> getLastUpsertAndLargestCommitSeq(long expired, BinlogComparator check) {
+ if (binlogs.size() <= 1) {
+ return null;
+ }
+
+ Iterator<TBinlog> iter = binlogs.iterator();
+ TBinlog dummyBinlog = iter.next();
TBinlog tombstoneUpsert = null;
+ TBinlog lastExpiredBinlog = null;
+ while (iter.hasNext()) {
+ TBinlog binlog = iter.next();
+ if (check.isExpired(binlog, expired)) {
+ lastExpiredBinlog = binlog;
+ --binlog.table_ref;
+ if (binlog.getType() == TBinlogType.UPSERT) {
+ tombstoneUpsert = binlog;
+ }
+ iter.remove();
+ } else {
+ break;
+ }
+ }
+
+ if (lastExpiredBinlog == null) {
+ return null;
+ }
+
+ dummyBinlog.setCommitSeq(lastExpiredBinlog.getCommitSeq());
+
+ return Pair.of(tombstoneUpsert, lastExpiredBinlog.getCommitSeq());
+ }
+
+ // this method call when db binlog enable
+ public BinlogTombstone gc(long expiredCommitSeq) {
+ Pair<TBinlog, Long> tombstoneInfo;
+ // step 1: get tombstoneUpsertBinlog and dummyBinlog
lock.writeLock().lock();
try {
- Iterator<TBinlog> iter = binlogs.iterator();
- while (iter.hasNext()) {
- TBinlog binlog = iter.next();
- if (binlog.getCommitSeq() <= largestExpiredCommitSeq) {
- if (binlog.getType() == TBinlogType.UPSERT) {
- tombstoneUpsert = binlog;
- }
- iter.remove();
- } else {
- break;
- }
- }
+ BinlogComparator check = (binlog, expire) -> binlog.getCommitSeq() <= expire;
+ tombstoneInfo = getLastUpsertAndLargestCommitSeq(expiredCommitSeq, check);
} finally {
lock.writeLock().unlock();
}
- if (tombstoneUpsert == null) {
+ // step 2: set tombstone by tombstoneInfo
+ // if there have expired Binlogs, tombstoneInfo != null
+ if (tombstoneInfo == null) {
return null;
}
- BinlogTombstone tombstone = new BinlogTombstone(-1, largestExpiredCommitSeq);
- UpsertRecord upsertRecord = UpsertRecord.fromJson(tombstoneUpsert.getData());
- tombstone.addTableRecord(tableId, upsertRecord);
+ TBinlog lastUpsertBinlog = tombstoneInfo.first;
+ long largestCommitSeq = tombstoneInfo.second;
+ BinlogTombstone tombstone = new BinlogTombstone(-1, largestCommitSeq);
+ if (lastUpsertBinlog != null) {
+ UpsertRecord upsertRecord = UpsertRecord.fromJson(lastUpsertBinlog.getData());
+ tombstone.addTableRecord(tableId, upsertRecord);
+ }
+
+ lock.readLock().lock();
+ try {
+ LOG.info("[deadlinefen] after gc, table {} binlogs: {}, tombstone.seq: {}",
+ tableId, binlogs, tombstone.getCommitSeq());
+ } finally {
+ lock.readLock().unlock();
+ }
+
return tombstone;
}
// this method call when db binlog disable
public BinlogTombstone gc(Database db) {
- OlapTable table = null;
+ // step 1: get expire time
+ OlapTable table;
try {
Table tbl = db.getTableOrMetaException(tableId);
if (tbl == null) {
@@ -138,39 +193,45 @@ public class TableBinlog {
long dbId = db.getId();
long ttlSeconds = table.getBinlogConfig().getTtlSeconds();
- long currentSeconds = System.currentTimeMillis() / 1000;
- long expireSeconds = currentSeconds - ttlSeconds;
- long expireMs = expireSeconds * 1000;
+ long expiredMs = BinlogUtils.getExpiredMs(ttlSeconds);
- TBinlog tombstoneUpsert = null;
- long largestExpiredCommitSeq = 0;
+ if (expiredMs < 0) {
+ return null;
+ }
+
+ // step 2: get tombstoneUpsertBinlog and dummyBinlog
+ Pair<TBinlog, Long> tombstoneInfo;
lock.writeLock().lock();
try {
- Iterator<TBinlog> iter = binlogs.iterator();
- while (iter.hasNext()) {
- TBinlog binlog = iter.next();
- long timestamp = binlog.getTimestamp();
-
- if (timestamp > expireMs) {
- break;
- }
-
- if (binlog.getType() == TBinlogType.UPSERT) {
- tombstoneUpsert = binlog;
- }
- largestExpiredCommitSeq = binlog.getCommitSeq();
- iter.remove();
- }
+ BinlogComparator check = (binlog, expire) -> binlog.getTimestamp() <= expire;
+ tombstoneInfo = getLastUpsertAndLargestCommitSeq(expiredMs, check);
} finally {
lock.writeLock().unlock();
}
- BinlogTombstone tombstone = new BinlogTombstone(dbId, tableId, largestExpiredCommitSeq);
- if (tombstoneUpsert != null) {
- UpsertRecord upsertRecord = UpsertRecord.fromJson(tombstoneUpsert.getData());
+ // step 3: set tombstone by tombstoneInfo
+ // if have expired Binlogs, tombstoneInfo != null
+ if (tombstoneInfo == null) {
+ return null;
+ }
+
+ TBinlog lastUpsertBinlog = tombstoneInfo.first;
+ long largestCommitSeq = tombstoneInfo.second;
+ BinlogTombstone tombstone = new BinlogTombstone(dbId, tableId, largestCommitSeq);
+ if (lastUpsertBinlog != null) {
+ UpsertRecord upsertRecord = UpsertRecord.fromJson(lastUpsertBinlog.getData());
tombstone.addTableRecord(tableId, upsertRecord);
}
+ lock.readLock().lock();
+ try {
+ LOG.info("[deadlinefen] after gc, table {} binlogs: {}, tombstone.seq: {}",
+ tableId, binlogs, tombstone.getCommitSeq());
+ } finally {
+ lock.readLock().unlock();
+ }
+
+
return tombstone;
}
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 4630f8bdd0..619337321f 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -961,8 +961,9 @@ enum TBinlogType {
CREATE_TABLE = 2,
DROP_PARTITION = 3,
DROP_TABLE = 4,
- ALTER_JOB = 5,
+ ALTER_JOB = 5,
MODIFY_TABLE_ADD_OR_DROP_COLUMNS = 6,
+ DUMMY = 7,
}
struct TBinlog {
@@ -972,6 +973,8 @@ struct TBinlog {
4: optional i64 db_id
5: optional list<i64> table_ids
6: optional string data
+ 7: optional i64 belong // belong == -1 if type is not DUMMY
+ 8: optional i64 table_ref // only use for gc
}
struct TGetBinlogResult {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 14/18: [feature](nereids) adjust min/max of column stats for cast function (#21772)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 26f5bea217b46f0686aece9210ae7ff31f25c59a
Author: minghong <en...@gmail.com>
AuthorDate: Fri Jul 14 12:54:04 2023 +0800
[feature](nereids) adjust min/max of column stats for cast function (#21772)
cast(A as date), where A is a string column. the min/max of result column stats should be calc like this:
convert A.minExpr to a date dateA, and then get double value from dateA.
add "explain memo plan select ..." to print memo from mysql client
dump column stats for FileScanNode, used in datalake.
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +++
.../java/org/apache/doris/nereids/memo/Memo.java | 11 +++++---
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +++
.../doris/nereids/stats/ExpressionEstimation.java | 31 +++++++++++++++++++++-
.../trees/plans/commands/ExplainCommand.java | 1 +
.../apache/doris/statistics/ColumnStatistic.java | 4 +--
8 files changed, 49 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 78071820ff..2f23376d0a 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -251,6 +251,7 @@ MATCH_ANY: 'MATCH_ANY';
MATCH_ALL: 'MATCH_ALL';
MATCH_PHRASE: 'MATCH_PHRASE';
MATCHED: 'MATCHED';
+MEMO:'MEMO';
MERGE: 'MERGE';
MINUTE: 'MINUTE';
MONTH: 'MONTH';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 3fd8e8b1af..c32a09f3f6 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -78,6 +78,7 @@ planType
| REWRITTEN | LOGICAL // same type
| OPTIMIZED | PHYSICAL // same type
| SHAPE
+ | MEMO
| ALL // default type
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index bf74eceee3..25064c1c68 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -408,6 +408,10 @@ public class NereidsPlanner extends Planner {
return "cost = " + cost + "\n" + optimizedPlan.treeString();
case SHAPE_PLAN:
return optimizedPlan.shape("");
+ case MEMO_PLAN:
+ return cascadesContext.getMemo().toString()
+ + "\n\n========== OPTIMIZED PLAN ==========\n"
+ + optimizedPlan.treeString();
case ALL_PLAN:
return "========== PARSED PLAN ==========\n"
+ parsedPlan.treeString() + "\n\n"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 42acda2511..643ffe819c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.LeafPlan;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -737,10 +738,12 @@ public class Memo {
builder.append("\n\n").append(group);
builder.append(" stats=").append(group.getStatistics()).append("\n");
Statistics stats = group.getStatistics();
- if (stats != null && !group.getLogicalExpressions().isEmpty()
- && group.getLogicalExpressions().get(0).getPlan() instanceof LogicalOlapScan) {
- for (Entry e : stats.columnStatistics().entrySet()) {
- builder.append(" ").append(e.getKey()).append(":").append(e.getValue()).append("\n");
+ if (stats != null && !group.getLogicalExpressions().isEmpty()) {
+ Plan plan = group.getLogicalExpressions().get(0).getPlan();
+ if (plan instanceof LogicalOlapScan || plan instanceof LogicalFileScan) {
+ for (Entry e : stats.columnStatistics().entrySet()) {
+ builder.append(" ").append(e.getKey()).append(":").append(e.getValue()).append("\n");
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index dffc10d7df..d4c044ad73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -1901,6 +1901,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
if (planTypeContext.SHAPE() != null) {
return ExplainLevel.SHAPE_PLAN;
}
+ if (planTypeContext.MEMO() != null) {
+ return ExplainLevel.MEMO_PLAN;
+ }
return ExplainLevel.ALL_PLAN;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
index eca7511684..9a3c54f739 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java
@@ -18,6 +18,8 @@
package org.apache.doris.nereids.stats;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.AggregateExpression;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -84,6 +86,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff;
import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub;
+import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
@@ -140,12 +143,38 @@ public class ExpressionEstimation extends ExpressionVisitor<ColumnStatistic, Sta
return columnStat.build();
}
+ @Override
public ColumnStatistic visitCast(Cast cast, Statistics context) {
ColumnStatistic stats = context.findColumnStatistics(cast);
if (stats != null) {
return stats;
}
- return cast.child().accept(this, context);
+ ColumnStatistic childColStats = cast.child().accept(this, context);
+
+ return castMinMax(childColStats, cast.getDataType());
+ }
+
+ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType) {
+ if (colStats.minExpr instanceof StringLiteral && targetType.isDateLikeType()) {
+ ColumnStatisticBuilder builder = new ColumnStatisticBuilder(colStats);
+ if (colStats.minExpr != null && colStats.maxExpr != null) {
+ String strMin = colStats.minExpr.getStringValue();
+ try {
+ DateLiteral dateMinLiteral = new DateLiteral(strMin);
+ long min = dateMinLiteral.getValue();
+ builder.setMinValue(min);
+
+ String strMax = colStats.maxExpr.getStringValue();
+ DateLiteral dateMaxLiteral = new DateLiteral(strMax);
+ long max = dateMaxLiteral.getValue();
+ builder.setMaxValue(max);
+ } catch (AnalysisException e) {
+ // ignore exception. do not convert min max
+ }
+ }
+ return builder.build();
+ }
+ return colStats;
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
index 820e3861d8..bfb0e82a17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
@@ -46,6 +46,7 @@ public class ExplainCommand extends Command implements NoForward {
REWRITTEN_PLAN(true),
OPTIMIZED_PLAN(true),
SHAPE_PLAN(true),
+ MEMO_PLAN(true),
ALL_PLAN(true)
;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 744c4af7a0..c2c987cf27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -314,8 +314,8 @@ public class ColumnStatistic {
@Override
public String toString() {
- return isUnKnown ? "unKnown" : String.format("ndv=%.4f, min=%f, max=%f, sel=%f, count=%.4f",
- ndv, minValue, maxValue, selectivity, count);
+ return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f",
+ ndv, minValue, minExpr, maxValue, maxExpr, count);
}
public JSONObject toJson() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 17/18: [regression] add order by in test case for stable output (#21815)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a51741cba6d608b05f95151579b56e034f06068e
Author: Kang <kx...@gmail.com>
AuthorDate: Fri Jul 14 18:01:43 2023 +0800
[regression] add order by in test case for stable output (#21815)
---
.../data/index_p0/test_ngram_bloomfilter_index.out | 8 +++----
.../index_p0/test_ngram_bloomfilter_index.groovy | 26 +++++++++++-----------
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index.out
index 7849739f42..531ce8119e 100644
--- a/regression-test/data/index_p0/test_ngram_bloomfilter_index.out
+++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index.out
@@ -1,7 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_all_1 --
1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699% /test 100 false
-1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
+2 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
-- !select_eq_1 --
1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699% /test 100 false
@@ -11,17 +11,17 @@
-- !select_like_1 --
1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699% /test 100 false
-1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
+2 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
-- !select_all_2 --
-1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
+2 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
-- !select_eq_2 --
-- !select_in_2 --
-- !select_like_2 --
-1 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
+2 dt_bjn001 p9-webcast-sign.douyinpic.com test /%/7212503657802320699%xxx /test 100 false
-- !select_all_3 --
diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
index 7619adedc3..c56eed967a 100644
--- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
+++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy
@@ -37,26 +37,26 @@ suite("test_ngram_bloomfilter_index") {
"""
sql "INSERT INTO ${tableName} values (1, 'dt_bjn001', 'p9-webcast-sign.douyinpic.com', 'test', '/%/7212503657802320699%', '/test', 100, false);"
- sql "INSERT INTO ${tableName} values (1, 'dt_bjn001', 'p9-webcast-sign.douyinpic.com', 'test', '/%/7212503657802320699%xxx', '/test', 100, false);"
+ sql "INSERT INTO ${tableName} values (2, 'dt_bjn001', 'p9-webcast-sign.douyinpic.com', 'test', '/%/7212503657802320699%xxx', '/test', 100, false);"
sql "SET enable_function_pushdown = true"
- qt_select_all_1 "SELECT * FROM ${tableName}"
- qt_select_eq_1 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%'"
- qt_select_in_1 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%')"
- qt_select_like_1 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%'"
+ qt_select_all_1 "SELECT * FROM ${tableName} ORDER BY key_id"
+ qt_select_eq_1 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%' ORDER BY key_id"
+ qt_select_in_1 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%') ORDER BY key_id"
+ qt_select_like_1 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%' ORDER BY key_id"
// delete and then select
sql "DELETE FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%')"
- qt_select_all_2 "SELECT * FROM ${tableName}"
- qt_select_eq_2 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%'"
- qt_select_in_2 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%')"
- qt_select_like_2 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%'"
+ qt_select_all_2 "SELECT * FROM ${tableName} ORDER BY key_id"
+ qt_select_eq_2 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%' ORDER BY key_id"
+ qt_select_in_2 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%') ORDER BY key_id"
+ qt_select_like_2 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%' ORDER BY key_id"
sql "DELETE FROM ${tableName} WHERE http_url = '/%/7212503657802320699%xxx'"
- qt_select_all_3 "SELECT * FROM ${tableName}"
- qt_select_eq_3 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%'"
- qt_select_in_3 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%')"
- qt_select_like_3 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%'"
+ qt_select_all_3 "SELECT * FROM ${tableName} ORDER BY key_id"
+ qt_select_eq_3 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%' ORDER BY key_id"
+ qt_select_in_3 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%') ORDER BY key_id"
+ qt_select_like_3 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%' ORDER BY key_id"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 11/18: [fix](Nereids) use groupExpr's children to make logicalPlan (#21794)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 466b7b854dc2878f28608386da9f7893c6021f2f
Author: 谢健 <ji...@gmail.com>
AuthorDate: Fri Jul 14 11:41:38 2023 +0800
[fix](Nereids) use groupExpr's children to make logicalPlan (#21794)
After mergeGroup, the children of the plan are different from GroupExpr. To avoid optimizing out-dated group, we should construct new plan with groupExpr's children rather than plan's children
---
.../nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
index be7e462b49..8023d806da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java
@@ -306,7 +306,7 @@ public class PlanReceiver implements AbstractReceiver {
}
hasGenerated.add(groupExpression);
- // process child first
+ // process child first, plan's child may be changed due to mergeGroup
Plan physicalPlan = groupExpression.getPlan();
for (Group child : groupExpression.children()) {
makeLogicalExpression(child);
@@ -316,12 +316,12 @@ public class PlanReceiver implements AbstractReceiver {
if (physicalPlan instanceof PhysicalProject) {
PhysicalProject physicalProject = (PhysicalProject) physicalPlan;
logicalPlan = new LogicalProject<>(physicalProject.getProjects(),
- physicalProject.child(0));
+ new GroupPlan(groupExpression.child(0)));
} else if (physicalPlan instanceof AbstractPhysicalJoin) {
AbstractPhysicalJoin physicalJoin = (AbstractPhysicalJoin) physicalPlan;
logicalPlan = new LogicalJoin<>(physicalJoin.getJoinType(), physicalJoin.getHashJoinConjuncts(),
physicalJoin.getOtherJoinConjuncts(), JoinHint.NONE, physicalJoin.getMarkJoinSlotReference(),
- physicalJoin.children());
+ groupExpression.children().stream().map(g -> new GroupPlan(g)).collect(Collectors.toList()));
} else {
throw new RuntimeException("DPhyp can only handle join and project operator");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/18: [improve](nereids)inner join estimation: assume children output at least one tuple #21792
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0456342faadadd5cd038472d2c479040af14947f
Author: minghong <en...@gmail.com>
AuthorDate: Fri Jul 14 11:30:25 2023 +0800
[improve](nereids)inner join estimation: assume children output at least one tuple #21792
this assumption is good to eliminate error propagation, when the filter estimation is too low, less than one.
---
.../src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
index 228b6936a7..867ad1527c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
@@ -124,7 +124,7 @@ public class JoinEstimation {
Statistics innerJoinStats;
Statistics crossJoinStats = new StatisticsBuilder()
- .setRowCount(Math.max(1, leftStats.getRowCount() * rightStats.getRowCount()))
+ .setRowCount(Math.max(1, leftStats.getRowCount()) * Math.max(1, rightStats.getRowCount()))
.putColumnStatistics(leftStats.columnStatistics())
.putColumnStatistics(rightStats.columnStatistics())
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/18: [fix](merge-on-write) fix be core and delete unused pending publish info for async publish when tablet dropped (#21793)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3ee83c77b054765ca0263e7956fc19182ed9168f
Author: Xin Liao <li...@126.com>
AuthorDate: Thu Jul 13 21:09:51 2023 +0800
[fix](merge-on-write) fix be core and delete unused pending publish info for async publish when tablet dropped (#21793)
---
be/src/olap/olap_server.cpp | 6 +++++-
be/src/olap/storage_engine.cpp | 27 +++++++++++++++++++++++++++
be/src/olap/storage_engine.h | 2 ++
3 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 09875b9e52..dabf07d176 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1216,6 +1216,11 @@ void StorageEngine::add_async_publish_task(int64_t partition_id, int64_t tablet_
bool is_recovery) {
if (!is_recovery) {
TabletSharedPtr tablet = tablet_manager()->get_tablet(tablet_id);
+ if (tablet == nullptr) {
+ LOG(INFO) << "tablet may be dropped when add async publish task, tablet_id: "
+ << tablet_id;
+ return;
+ }
PendingPublishInfoPB pending_publish_info_pb;
pending_publish_info_pb.set_partition_id(partition_id);
pending_publish_info_pb.set_transaction_id(transaction_id);
@@ -1259,7 +1264,6 @@ void StorageEngine::_async_publish_callback() {
if (!tablet) {
LOG(WARNING) << "tablet does not exist when async publush, tablet_id: "
<< tablet_id;
- // TODO(liaoxin) remove pending publish info from db
tablet_iter = _async_publish_tasks.erase(tablet_iter);
continue;
}
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index a182fccd31..a448a11f46 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -695,6 +695,9 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) {
// cleand unused delete bitmap for deleted tablet
_clean_unused_delete_bitmap();
+ // cleand unused pending publish info for deleted tablet
+ _clean_unused_pending_publish_info();
+
// clean unused rowsets in remote storage backends
for (auto data_dir : get_stores()) {
data_dir->perform_remote_rowset_gc();
@@ -795,6 +798,30 @@ void StorageEngine::_clean_unused_delete_bitmap() {
}
}
+void StorageEngine::_clean_unused_pending_publish_info() {
+ std::vector<std::pair<int64_t, int64_t>> removed_infos;
+ auto clean_pending_publish_info_func = [this, &removed_infos](int64_t tablet_id,
+ int64_t publish_version,
+ const string& info) -> bool {
+ TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id);
+ if (tablet == nullptr) {
+ removed_infos.emplace_back(tablet_id, publish_version);
+ }
+ return true;
+ };
+ auto data_dirs = get_stores();
+ for (auto data_dir : data_dirs) {
+ TabletMetaManager::traverse_pending_publish(data_dir->get_meta(),
+ clean_pending_publish_info_func);
+ for (auto& [tablet_id, publish_version] : removed_infos) {
+ TabletMetaManager::remove_pending_publish_info(data_dir, tablet_id, publish_version);
+ }
+ LOG(INFO) << "removed invalid pending publish info from dir: " << data_dir->path()
+ << ", deleted pending publish info size: " << removed_infos.size();
+ removed_infos.clear();
+ }
+}
+
void StorageEngine::gc_binlogs(const std::unordered_map<int64_t, int64_t>& gc_tablet_infos) {
for (auto [tablet_id, version] : gc_tablet_infos) {
LOG(INFO) << fmt::format("start to gc binlogs for tablet_id: {}, version: {}", tablet_id,
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 2bd01ba2cb..a40e1792d9 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -255,6 +255,8 @@ private:
void _clean_unused_delete_bitmap();
+ void _clean_unused_pending_publish_info();
+
Status _do_sweep(const std::string& scan_root, const time_t& local_tm_now,
const int32_t expire);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/18: [Fix](MoW) Fix bug about caculating all committed rowsets delete bitmaps when do comapction (#21760)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 26497c49a40394fec5ca664791784f75fba945b4
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Thu Jul 13 21:10:15 2023 +0800
[Fix](MoW) Fix bug about caculating all committed rowsets delete bitmaps when do comapction (#21760)
---
be/src/olap/compaction.cpp | 46 +++++++++++++++++++++++++++++++++++-----------
be/src/olap/compaction.h | 3 +++
2 files changed, 38 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index ec4f6a6ff9..e4c0b59d8c 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -614,17 +614,26 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) {
// Step2: calculate all rowsets' delete bitmaps which are published during compaction.
for (auto& it : commit_tablet_txn_info_vec) {
- DeleteBitmap output_delete_bitmap(_tablet->tablet_id());
- _tablet->calc_compaction_output_rowset_delete_bitmap(
- _input_rowsets, _rowid_conversion, 0, UINT64_MAX, &missed_rows,
- &location_map, *it.delete_bitmap.get(), &output_delete_bitmap);
- it.delete_bitmap->merge(output_delete_bitmap);
- // Step3: write back updated delete bitmap and tablet info.
- it.rowset_ids.insert(_output_rowset->rowset_id());
- StorageEngine::instance()->txn_manager()->set_txn_related_delete_bitmap(
- it.partition_id, it.transaction_id, _tablet->tablet_id(),
- _tablet->schema_hash(), _tablet->tablet_uid(), true, it.delete_bitmap,
- it.rowset_ids);
+ if (!_check_if_includes_input_rowsets(it.rowset_ids)) {
+ // When calculating the delete bitmap of all committed rowsets relative to the compaction,
+ // there may be cases where the compacted rowsets are newer than the committed rowsets.
+ // At this time, row number conversion cannot be performed, otherwise data will be missing.
+ // Therefore, we need to check if every committed rowset has calculated delete bitmap for
+ // all compaction input rowsets.
+ continue;
+ } else {
+ DeleteBitmap output_delete_bitmap(_tablet->tablet_id());
+ _tablet->calc_compaction_output_rowset_delete_bitmap(
+ _input_rowsets, _rowid_conversion, 0, UINT64_MAX, &missed_rows,
+ &location_map, *it.delete_bitmap.get(), &output_delete_bitmap);
+ it.delete_bitmap->merge(output_delete_bitmap);
+ // Step3: write back updated delete bitmap and tablet info.
+ it.rowset_ids.insert(_output_rowset->rowset_id());
+ StorageEngine::instance()->txn_manager()->set_txn_related_delete_bitmap(
+ it.partition_id, it.transaction_id, _tablet->tablet_id(),
+ _tablet->schema_hash(), _tablet->tablet_uid(), true, it.delete_bitmap,
+ it.rowset_ids);
+ }
}
// Convert the delete bitmap of the input rowsets to output rowset for
@@ -668,6 +677,21 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) {
return Status::OK();
}
+bool Compaction::_check_if_includes_input_rowsets(
+ const RowsetIdUnorderedSet& commit_rowset_ids_set) const {
+ std::vector<RowsetId> commit_rowset_ids {};
+ commit_rowset_ids.insert(commit_rowset_ids.end(), commit_rowset_ids_set.begin(),
+ commit_rowset_ids_set.end());
+ std::sort(commit_rowset_ids.begin(), commit_rowset_ids.end());
+ std::vector<RowsetId> input_rowset_ids {};
+ for (const auto& rowset : _input_rowsets) {
+ input_rowset_ids.emplace_back(rowset->rowset_meta()->rowset_id());
+ }
+ std::sort(input_rowset_ids.begin(), input_rowset_ids.end());
+ return std::includes(commit_rowset_ids.begin(), commit_rowset_ids.end(),
+ input_rowset_ids.begin(), input_rowset_ids.end());
+}
+
void Compaction::gc_output_rowset() {
if (_state != CompactionState::SUCCESS && _output_rowset != nullptr) {
if (!_output_rowset->is_local()) {
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index bed9ae403f..853d970a48 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -94,6 +94,9 @@ protected:
void init_profile(const std::string& label);
+private:
+ bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const;
+
protected:
// the root tracker for this compaction
std::shared_ptr<MemTrackerLimiter> _mem_tracker;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 18/18: [imporve](bloomfilter) refactor runtime_filter_mgr with bloomfilter and fix bug in change_to_bloom_filter (#21783)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 976a9e0197b636b838efaf9276180314566dead7
Author: Mryange <59...@users.noreply.github.com>
AuthorDate: Fri Jul 14 17:47:32 2023 +0800
[imporve](bloomfilter) refactor runtime_filter_mgr with bloomfilter and fix bug in change_to_bloom_filter (#21783)
---
be/src/exprs/runtime_filter.cpp | 5 ++++-
be/src/runtime/runtime_filter_mgr.cpp | 38 +++++++++++++++++++----------------
be/src/runtime/runtime_filter_mgr.h | 13 ++++++------
3 files changed, 32 insertions(+), 24 deletions(-)
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index e6639dadf0..6932ad9c6b 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -440,7 +440,10 @@ public:
<< "Can not change to bloom filter because of runtime filter type is "
<< to_string(_filter_type);
_is_bloomfilter = true;
- insert_to_bloom_filter(_context.bloom_filter_func.get());
+ BloomFilterFuncBase* bf = _context.bloom_filter_func.get();
+ // BloomFilter may be not init
+ bf->init_with_fixed_length();
+ insert_to_bloom_filter(bf);
// release in filter
_context.hybrid_set.reset(create_set(_column_return_type));
}
diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp
index 4380716671..6614532d85 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -39,6 +39,7 @@
#include "runtime/runtime_state.h"
#include "runtime/thread_context.h"
#include "util/brpc_client_cache.h"
+#include "util/spinlock.h"
namespace doris {
@@ -208,7 +209,7 @@ Status RuntimeFilterMergeControllerEntity::_init_with_desc(
const TRuntimeFilterDesc* runtime_filter_desc, const TQueryOptions* query_options,
const std::vector<doris::TRuntimeFilterTargetParams>* target_info,
const int producer_size) {
- std::lock_guard<std::mutex> guard(_filter_map_mutex);
+ std::unique_lock<std::shared_mutex> guard(_filter_map_mutex);
std::shared_ptr<RuntimeFilterCntlVal> cntVal = std::make_shared<RuntimeFilterCntlVal>();
// runtime_filter_desc and target will be released,
// so we need to copy to cntVal
@@ -219,10 +220,10 @@ Status RuntimeFilterMergeControllerEntity::_init_with_desc(
cntVal->filter =
cntVal->pool->add(new IRuntimeFilter(_state, &_state->get_query_ctx()->obj_pool));
- std::string filter_id = std::to_string(runtime_filter_desc->filter_id);
+ auto filter_id = runtime_filter_desc->filter_id;
// LOG(INFO) << "entity filter id:" << filter_id;
cntVal->filter->init_with_desc(&cntVal->runtime_filter_desc, query_options, -1, false);
- _filter_map.emplace(filter_id, cntVal);
+ _filter_map.emplace(filter_id, CntlValwithLock {cntVal, std::make_unique<SpinLock>()});
return Status::OK();
}
@@ -230,7 +231,7 @@ Status RuntimeFilterMergeControllerEntity::_init_with_desc(
const TRuntimeFilterDesc* runtime_filter_desc, const TQueryOptions* query_options,
const std::vector<doris::TRuntimeFilterTargetParamsV2>* targetv2_info,
const int producer_size) {
- std::lock_guard<std::mutex> guard(_filter_map_mutex);
+ std::unique_lock<std::shared_mutex> guard(_filter_map_mutex);
std::shared_ptr<RuntimeFilterCntlVal> cntVal = std::make_shared<RuntimeFilterCntlVal>();
// runtime_filter_desc and target will be released,
// so we need to copy to cntVal
@@ -241,10 +242,10 @@ Status RuntimeFilterMergeControllerEntity::_init_with_desc(
cntVal->filter =
cntVal->pool->add(new IRuntimeFilter(_state, &_state->get_query_ctx()->obj_pool));
- std::string filter_id = std::to_string(runtime_filter_desc->filter_id);
+ auto filter_id = runtime_filter_desc->filter_id;
// LOG(INFO) << "entity filter id:" << filter_id;
cntVal->filter->init_with_desc(&cntVal->runtime_filter_desc, query_options);
- _filter_map.emplace(filter_id, cntVal);
+ _filter_map.emplace(filter_id, CntlValwithLock {cntVal, std::make_unique<SpinLock>()});
return Status::OK();
}
@@ -312,34 +313,37 @@ Status RuntimeFilterMergeControllerEntity::merge(const PMergeFilterRequest* requ
std::shared_ptr<RuntimeFilterCntlVal> cntVal;
int merged_size = 0;
int64_t merge_time = 0;
+ int64_t start_merge = MonotonicMillis();
+ auto filter_id = request->filter_id();
+ std::map<int, CntlValwithLock>::iterator iter;
{
- int64_t start_merge = MonotonicMillis();
- std::lock_guard<std::mutex> guard(_filter_map_mutex);
- auto iter = _filter_map.find(std::to_string(request->filter_id()));
+ std::shared_lock<std::shared_mutex> guard(_filter_map_mutex);
+ iter = _filter_map.find(filter_id);
VLOG_ROW << "recv filter id:" << request->filter_id() << " " << request->ShortDebugString();
if (iter == _filter_map.end()) {
return Status::InvalidArgument("unknown filter id {}",
std::to_string(request->filter_id()));
}
- cntVal = iter->second;
- if (auto bf = cntVal->filter->get_bloomfilter()) {
- RETURN_IF_ERROR(bf->init_with_fixed_length());
- }
+ }
+ // iter->second = pair{CntlVal,SpinLock}
+ cntVal = iter->second.first;
+ {
+ std::lock_guard<SpinLock> l(*iter->second.second);
MergeRuntimeFilterParams params(request, attach_data);
- ObjectPool* pool = iter->second->pool.get();
+ ObjectPool* pool = cntVal->pool.get();
RuntimeFilterWrapperHolder holder;
RETURN_IF_ERROR(IRuntimeFilter::create_wrapper(_state, ¶ms, pool, holder.getHandle()));
RETURN_IF_ERROR(cntVal->filter->merge_from(holder.getHandle()->get()));
- cntVal->arrive_id.insert(UniqueId(request->fragment_id()).to_string());
+ cntVal->arrive_id.insert(UniqueId(request->fragment_id()));
merged_size = cntVal->arrive_id.size();
// TODO: avoid log when we had acquired a lock
VLOG_ROW << "merge size:" << merged_size << ":" << cntVal->producer_size;
DCHECK_LE(merged_size, cntVal->producer_size);
- iter->second->merge_time += (MonotonicMillis() - start_merge);
+ cntVal->merge_time += (MonotonicMillis() - start_merge);
if (merged_size < cntVal->producer_size) {
return Status::OK();
} else {
- merge_time = iter->second->merge_time;
+ merge_time = cntVal->merge_time;
}
}
diff --git a/be/src/runtime/runtime_filter_mgr.h b/be/src/runtime/runtime_filter_mgr.h
index 7d7e7a0ba1..2048229cd2 100644
--- a/be/src/runtime/runtime_filter_mgr.h
+++ b/be/src/runtime/runtime_filter_mgr.h
@@ -26,6 +26,7 @@
#include <map>
#include <memory>
#include <mutex>
+#include <shared_mutex>
#include <string>
#include <unordered_map>
#include <unordered_set>
@@ -145,12 +146,12 @@ public:
std::vector<doris::TRuntimeFilterTargetParams> target_info;
std::vector<doris::TRuntimeFilterTargetParamsV2> targetv2_info;
IRuntimeFilter* filter;
- std::unordered_set<std::string> arrive_id; // fragment_instance_id ?
+ std::unordered_set<UniqueId> arrive_id; // fragment_instance_id ?
std::shared_ptr<ObjectPool> pool;
};
public:
- RuntimeFilterCntlVal* get_filter(int id) { return _filter_map[std::to_string(id)].get(); }
+ RuntimeFilterCntlVal* get_filter(int id) { return _filter_map[id].first.get(); }
private:
Status _init_with_desc(const TRuntimeFilterDesc* runtime_filter_desc,
@@ -166,11 +167,11 @@ private:
UniqueId _query_id;
UniqueId _fragment_instance_id;
// protect _filter_map
- std::mutex _filter_map_mutex;
+ std::shared_mutex _filter_map_mutex;
std::shared_ptr<MemTracker> _mem_tracker;
- // TODO: convert filter id to i32
- // filter-id -> val
- std::map<std::string, std::shared_ptr<RuntimeFilterCntlVal>> _filter_map;
+ using CntlValwithLock =
+ std::pair<std::shared_ptr<RuntimeFilterCntlVal>, std::unique_ptr<SpinLock>>;
+ std::map<int, CntlValwithLock> _filter_map;
RuntimeState* _state;
bool _opt_remote_rf = true;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/18: [imporve](udaf) refactor java-udaf executor by using for loop (#21713)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 46ab7587e23a08b511485ae4959a79d4c19b802e
Author: zhangstar333 <87...@users.noreply.github.com>
AuthorDate: Fri Jul 14 11:37:19 2023 +0800
[imporve](udaf) refactor java-udaf executor by using for loop (#21713)
refactor java-udaf executor by using for loop
---
.../aggregate_function_java_udaf.h | 119 ++++++----
fe/be-java-extensions/java-udf/pom.xml | 6 +
.../java/org/apache/doris/udf/BaseExecutor.java | 181 ++++++++++++++
.../java/org/apache/doris/udf/UdafExecutor.java | 89 ++++++-
.../main/java/org/apache/doris/udf/UdfConvert.java | 262 +++++++++++----------
.../java/org/apache/doris/udf/UdfExecutor.java | 165 +------------
6 files changed, 485 insertions(+), 337 deletions(-)
diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
index fa0c4efd9d..6fe4742064 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h
@@ -128,63 +128,80 @@ public:
return Status::OK();
}
- Status add(const int64_t places_address[], bool is_single_place, const IColumn** columns,
- size_t row_num_start, size_t row_num_end, const DataTypes& argument_types) {
+ Status add(int64_t places_address, bool is_single_place, const IColumn** columns,
+ int row_num_start, int row_num_end, const DataTypes& argument_types,
+ int place_offset) {
JNIEnv* env = nullptr;
RETURN_NOT_OK_STATUS_WITH_WARN(JniUtil::GetJNIEnv(&env), "Java-Udaf add function");
+ jclass obj_class = env->FindClass("[Ljava/lang/Object;");
+ jobjectArray arg_objects = env->NewObjectArray(argument_size, obj_class, nullptr);
+ int64_t nullmap_address = 0;
+
for (int arg_idx = 0; arg_idx < argument_size; ++arg_idx) {
+ bool arg_column_nullable = false;
auto data_col = columns[arg_idx];
if (auto* nullable = check_and_get_column<const ColumnNullable>(*columns[arg_idx])) {
+ arg_column_nullable = true;
+ auto null_col = nullable->get_null_map_column_ptr();
data_col = nullable->get_nested_column_ptr();
- auto null_col = check_and_get_column<ColumnVector<UInt8>>(
- nullable->get_null_map_column_ptr());
- input_nulls_buffer_ptr.get()[arg_idx] =
- reinterpret_cast<int64_t>(null_col->get_data().data());
- } else {
- input_nulls_buffer_ptr.get()[arg_idx] = -1;
+ nullmap_address = reinterpret_cast<int64_t>(
+ check_and_get_column<ColumnVector<UInt8>>(null_col)->get_data().data());
}
- if (data_col->is_column_string()) {
- const ColumnString* str_col = check_and_get_column<ColumnString>(data_col);
- input_values_buffer_ptr.get()[arg_idx] =
- reinterpret_cast<int64_t>(str_col->get_chars().data());
- input_offsets_ptrs.get()[arg_idx] =
- reinterpret_cast<int64_t>(str_col->get_offsets().data());
- } else if (data_col->is_numeric() || data_col->is_column_decimal()) {
- input_values_buffer_ptr.get()[arg_idx] =
- reinterpret_cast<int64_t>(data_col->get_raw_data().data);
+ // convert argument column data into java type
+ jobjectArray arr_obj = nullptr;
+ if (data_col->is_numeric() || data_col->is_column_decimal()) {
+ arr_obj = (jobjectArray)env->CallObjectMethod(
+ executor_obj, executor_convert_basic_argument_id, arg_idx,
+ arg_column_nullable, row_num_start, row_num_end, nullmap_address,
+ reinterpret_cast<int64_t>(data_col->get_raw_data().data), 0);
+ } else if (data_col->is_column_string()) {
+ const ColumnString* str_col = assert_cast<const ColumnString*>(data_col);
+ arr_obj = (jobjectArray)env->CallObjectMethod(
+ executor_obj, executor_convert_basic_argument_id, arg_idx,
+ arg_column_nullable, row_num_start, row_num_end, nullmap_address,
+ reinterpret_cast<int64_t>(str_col->get_chars().data()),
+ reinterpret_cast<int64_t>(str_col->get_offsets().data()));
} else if (data_col->is_column_array()) {
const ColumnArray* array_col = assert_cast<const ColumnArray*>(data_col);
- input_offsets_ptrs.get()[arg_idx] = reinterpret_cast<int64_t>(
- array_col->get_offsets_column().get_raw_data().data);
const ColumnNullable& array_nested_nullable =
assert_cast<const ColumnNullable&>(array_col->get_data());
auto data_column_null_map = array_nested_nullable.get_null_map_column_ptr();
auto data_column = array_nested_nullable.get_nested_column_ptr();
- input_array_nulls_buffer_ptr.get()[arg_idx] = reinterpret_cast<int64_t>(
+ auto offset_address = reinterpret_cast<int64_t>(
+ array_col->get_offsets_column().get_raw_data().data);
+ auto nested_nullmap_address = reinterpret_cast<int64_t>(
check_and_get_column<ColumnVector<UInt8>>(data_column_null_map)
->get_data()
.data());
-
- //need pass FE, nullamp and offset, chars
+ int64_t nested_data_address = 0, nested_offset_address = 0;
+ // array type need pass address: [nullmap_address], offset_address, nested_nullmap_address, nested_data_address/nested_char_address,nested_offset_address
if (data_column->is_column_string()) {
const ColumnString* col = assert_cast<const ColumnString*>(data_column.get());
- input_values_buffer_ptr.get()[arg_idx] =
- reinterpret_cast<int64_t>(col->get_chars().data());
- input_array_string_offsets_ptrs.get()[arg_idx] =
- reinterpret_cast<int64_t>(col->get_offsets().data());
+ nested_data_address = reinterpret_cast<int64_t>(col->get_chars().data());
+ nested_offset_address = reinterpret_cast<int64_t>(col->get_offsets().data());
} else {
- input_values_buffer_ptr.get()[arg_idx] =
+ nested_data_address =
reinterpret_cast<int64_t>(data_column->get_raw_data().data);
}
+ arr_obj = (jobjectArray)env->CallObjectMethod(
+ executor_obj, executor_convert_array_argument_id, arg_idx,
+ arg_column_nullable, row_num_start, row_num_end, nullmap_address,
+ offset_address, nested_nullmap_address, nested_data_address,
+ nested_offset_address);
} else {
return Status::InvalidArgument(
strings::Substitute("Java UDAF doesn't support type is $0 now !",
argument_types[arg_idx]->get_name()));
}
+ env->SetObjectArrayElement(arg_objects, arg_idx, arr_obj);
+ env->DeleteLocalRef(arr_obj);
}
- *input_place_ptrs = reinterpret_cast<int64_t>(places_address);
- env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_add_id, is_single_place,
- row_num_start, row_num_end);
+ RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env));
+ // invoke add batch
+ env->CallObjectMethod(executor_obj, executor_add_batch_id, is_single_place, row_num_start,
+ row_num_end, places_address, place_offset, arg_objects);
+ env->DeleteLocalRef(arg_objects);
+ env->DeleteLocalRef(obj_class);
return JniUtil::GetJniExceptionMsg(env);
}
@@ -392,6 +409,12 @@ private:
register_id("getValue", UDAF_EXECUTOR_RESULT_SIGNATURE, executor_result_id));
RETURN_IF_ERROR(
register_id("destroy", UDAF_EXECUTOR_DESTROY_SIGNATURE, executor_destroy_id));
+ RETURN_IF_ERROR(register_id("convertBasicArguments", "(IZIIJJJ)[Ljava/lang/Object;",
+ executor_convert_basic_argument_id));
+ RETURN_IF_ERROR(register_id("convertArrayArguments", "(IZIIJJJJJ)[Ljava/lang/Object;",
+ executor_convert_array_argument_id));
+ RETURN_IF_ERROR(
+ register_id("addBatch", "(ZIIJI[Ljava/lang/Object;)V", executor_add_batch_id));
return Status::OK();
}
@@ -403,12 +426,15 @@ private:
jmethodID executor_ctor_id;
jmethodID executor_add_id;
+ jmethodID executor_add_batch_id;
jmethodID executor_merge_id;
jmethodID executor_serialize_id;
jmethodID executor_result_id;
jmethodID executor_reset_id;
jmethodID executor_close_id;
jmethodID executor_destroy_id;
+ jmethodID executor_convert_basic_argument_id;
+ jmethodID executor_convert_array_argument_id;
std::unique_ptr<int64_t[]> input_values_buffer_ptr;
std::unique_ptr<int64_t[]> input_nulls_buffer_ptr;
@@ -481,11 +507,10 @@ public:
void add(AggregateDataPtr __restrict place, const IColumn** columns, size_t row_num,
Arena*) const override {
- int64_t places_address[1];
- places_address[0] = reinterpret_cast<int64_t>(place);
- Status st =
- this->data(_exec_place)
- .add(places_address, true, columns, row_num, row_num + 1, argument_types);
+ int64_t places_address = reinterpret_cast<int64_t>(place);
+ Status st = this->data(_exec_place)
+ .add(places_address, true, columns, row_num, row_num + 1,
+ argument_types, 0);
if (UNLIKELY(st != Status::OK())) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, st.to_string());
}
@@ -493,25 +518,20 @@ public:
void add_batch(size_t batch_size, AggregateDataPtr* places, size_t place_offset,
const IColumn** columns, Arena* /*arena*/, bool /*agg_many*/) const override {
- int64_t places_address[batch_size];
- for (size_t i = 0; i < batch_size; ++i) {
- places_address[i] = reinterpret_cast<int64_t>(places[i] + place_offset);
- }
+ int64_t places_address = reinterpret_cast<int64_t>(places);
Status st = this->data(_exec_place)
- .add(places_address, false, columns, 0, batch_size, argument_types);
+ .add(places_address, false, columns, 0, batch_size, argument_types,
+ place_offset);
if (UNLIKELY(st != Status::OK())) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, st.to_string());
}
}
- // TODO: Here we calling method by jni, And if we get a thrown from FE,
- // But can't let user known the error, only return directly and output error to log file.
void add_batch_single_place(size_t batch_size, AggregateDataPtr place, const IColumn** columns,
Arena* /*arena*/) const override {
- int64_t places_address[1];
- places_address[0] = reinterpret_cast<int64_t>(place);
+ int64_t places_address = reinterpret_cast<int64_t>(place);
Status st = this->data(_exec_place)
- .add(places_address, true, columns, 0, batch_size, argument_types);
+ .add(places_address, true, columns, 0, batch_size, argument_types, 0);
if (UNLIKELY(st != Status::OK())) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, st.to_string());
}
@@ -522,11 +542,10 @@ public:
Arena* arena) const override {
frame_start = std::max<int64_t>(frame_start, partition_start);
frame_end = std::min<int64_t>(frame_end, partition_end);
- int64_t places_address[1];
- places_address[0] = reinterpret_cast<int64_t>(place);
- Status st =
- this->data(_exec_place)
- .add(places_address, true, columns, frame_start, frame_end, argument_types);
+ int64_t places_address = reinterpret_cast<int64_t>(place);
+ Status st = this->data(_exec_place)
+ .add(places_address, true, columns, frame_start, frame_end,
+ argument_types, 0);
if (UNLIKELY(st != Status::OK())) {
throw doris::Exception(ErrorCode::INTERNAL_ERROR, st.to_string());
}
diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml
index bf05aeafc2..67921aa2cf 100644
--- a/fe/be-java-extensions/java-udf/pom.xml
+++ b/fe/be-java-extensions/java-udf/pom.xml
@@ -41,6 +41,12 @@ under the License.
<artifactId>java-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <!-- https://mvnrepository.com/artifact/com.esotericsoftware/reflectasm -->
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>reflectasm</artifactId>
+ <version>1.11.9</version>
+ </dependency>
</dependencies>
<build>
diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
index 3dbe10ca27..ef405197d6 100644
--- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
+++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java
@@ -25,12 +25,14 @@ import org.apache.doris.common.jni.utils.UdfUtils;
import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType;
import org.apache.doris.thrift.TJavaUdfExecutorCtorParams;
+import com.google.common.base.Preconditions;
import org.apache.log4j.Logger;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import java.io.IOException;
+import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
@@ -1021,4 +1023,183 @@ public abstract class BaseExecutor {
protected void updateOutputOffset(long offset) {
}
+
+ public Object[] convertBasicArg(boolean isUdf, int argIdx, boolean isNullable, int rowStart, int rowEnd,
+ long nullMapAddr, long columnAddr, long strOffsetAddr) {
+ switch (argTypes[argIdx]) {
+ case BOOLEAN:
+ return UdfConvert.convertBooleanArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case TINYINT:
+ return UdfConvert.convertTinyIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case SMALLINT:
+ return UdfConvert.convertSmallIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case INT:
+ return UdfConvert.convertIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case BIGINT:
+ return UdfConvert.convertBigIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case LARGEINT:
+ return UdfConvert.convertLargeIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case FLOAT:
+ return UdfConvert.convertFloatArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case DOUBLE:
+ return UdfConvert.convertDoubleArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr);
+ case CHAR:
+ case VARCHAR:
+ case STRING:
+ return UdfConvert
+ .convertStringArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr, strOffsetAddr);
+ case DATE: // udaf maybe argClass[i + argClassOffset] need add +1
+ return UdfConvert
+ .convertDateArg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, rowEnd,
+ nullMapAddr, columnAddr);
+ case DATETIME:
+ return UdfConvert
+ .convertDateTimeArg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart,
+ rowEnd, nullMapAddr, columnAddr);
+ case DATEV2:
+ return UdfConvert
+ .convertDateV2Arg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, rowEnd,
+ nullMapAddr, columnAddr);
+ case DATETIMEV2:
+ return UdfConvert
+ .convertDateTimeV2Arg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart,
+ rowEnd, nullMapAddr, columnAddr);
+ case DECIMALV2:
+ case DECIMAL128:
+ return UdfConvert
+ .convertDecimalArg(argTypes[argIdx].getScale(), 16L, isNullable, rowStart, rowEnd, nullMapAddr,
+ columnAddr);
+ case DECIMAL32:
+ return UdfConvert
+ .convertDecimalArg(argTypes[argIdx].getScale(), 4L, isNullable, rowStart, rowEnd, nullMapAddr,
+ columnAddr);
+ case DECIMAL64:
+ return UdfConvert
+ .convertDecimalArg(argTypes[argIdx].getScale(), 8L, isNullable, rowStart, rowEnd, nullMapAddr,
+ columnAddr);
+ default: {
+ LOG.info("Not support type: " + argTypes[argIdx].toString());
+ Preconditions.checkState(false, "Not support type: " + argTypes[argIdx].toString());
+ break;
+ }
+ }
+ return null;
+ }
+
+ public Object[] convertArrayArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr,
+ long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) {
+ Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart);
+ for (int row = rowStart; row < rowEnd; ++row) {
+ long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1));
+ long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row));
+ int currentRowNum = (int) (offsetEnd - offsetStart);
+ switch (argTypes[argIdx].getItemType().getPrimitiveType()) {
+ case BOOLEAN: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case TINYINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case SMALLINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case INT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case BIGINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case LARGEINT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case FLOAT: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DOUBLE: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case CHAR:
+ case VARCHAR:
+ case STRING: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr, strOffsetAddr);
+ break;
+ }
+ case DATE: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATETIME: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATEV2: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr,
+ nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DATETIMEV2: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable,
+ nullMapAddr, nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMALV2:
+ case DECIMAL128: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMAL32: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr);
+ break;
+ }
+ case DECIMAL64: {
+ argument[row - rowStart] = UdfConvert
+ .convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum,
+ offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr);
+ break;
+ }
+ default: {
+ LOG.info("Not support: " + argTypes[argIdx]);
+ Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString());
+ break;
+ }
+ }
+ }
+ return argument;
+ }
}
diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
index e2d8ab1b75..a0736b5a72 100644
--- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
+++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.jni.utils.UdfUtils;
import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType;
import org.apache.doris.thrift.TJavaUdfExecutorCtorParams;
+import com.esotericsoftware.reflectasm.MethodAccess;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import org.apache.log4j.Logger;
@@ -36,6 +37,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
/**
@@ -49,6 +51,8 @@ public class UdafExecutor extends BaseExecutor {
private HashMap<String, Method> allMethods;
private HashMap<Long, Object> stateObjMap;
private Class retClass;
+ private int addIndex;
+ private MethodAccess methodAccess;
/**
* Constructor to create an object.
@@ -66,6 +70,84 @@ public class UdafExecutor extends BaseExecutor {
super.close();
}
+ public Object[] convertBasicArguments(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr,
+ long columnAddr, long strOffsetAddr) {
+ return convertBasicArg(false, argIdx, isNullable, rowStart, rowEnd, nullMapAddr, columnAddr, strOffsetAddr);
+ }
+
+ public Object[] convertArrayArguments(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr,
+ long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) {
+ return convertArrayArg(argIdx, isNullable, rowStart, rowEnd, nullMapAddr, offsetsAddr, nestedNullMapAddr,
+ dataAddr, strOffsetAddr);
+ }
+
+ public void addBatch(boolean isSinglePlace, int rowStart, int rowEnd, long placeAddr, int offset, Object[] column)
+ throws UdfRuntimeException {
+ if (isSinglePlace) {
+ addBatchSingle(rowStart, rowEnd, placeAddr, column);
+ } else {
+ addBatchPlaces(rowStart, rowEnd, placeAddr, offset, column);
+ }
+ }
+
+ public void addBatchSingle(int rowStart, int rowEnd, long placeAddr, Object[] column) throws UdfRuntimeException {
+ try {
+ Long curPlace = placeAddr;
+ Object[] inputArgs = new Object[argTypes.length + 1];
+ Object state = stateObjMap.get(curPlace);
+ if (state != null) {
+ inputArgs[0] = state;
+ } else {
+ Object newState = createAggState();
+ stateObjMap.put(curPlace, newState);
+ inputArgs[0] = newState;
+ }
+
+ Object[][] inputs = (Object[][]) column;
+ for (int i = 0; i < (rowEnd - rowStart); ++i) {
+ for (int j = 0; j < column.length; ++j) {
+ inputArgs[j + 1] = inputs[j][i];
+ }
+ methodAccess.invoke(udf, addIndex, inputArgs);
+ }
+ } catch (Exception e) {
+ LOG.warn("invoke add function meet some error: " + e.getCause().toString());
+ throw new UdfRuntimeException("UDAF failed to addBatchSingle: ", e);
+ }
+ }
+
+ public void addBatchPlaces(int rowStart, int rowEnd, long placeAddr, int offset, Object[] column)
+ throws UdfRuntimeException {
+ try {
+ Object[][] inputs = (Object[][]) column;
+ ArrayList<Object> placeState = new ArrayList<>(rowEnd - rowStart);
+ for (int row = rowStart; row < rowEnd; ++row) {
+ Long curPlace = UdfUtils.UNSAFE.getLong(null, placeAddr + (8L * row)) + offset;
+ Object state = stateObjMap.get(curPlace);
+ if (state != null) {
+ placeState.add(state);
+ } else {
+ Object newState = createAggState();
+ stateObjMap.put(curPlace, newState);
+ placeState.add(newState);
+ }
+ }
+ //spilt into two for loop
+
+ Object[] inputArgs = new Object[argTypes.length + 1];
+ for (int row = 0; row < (rowEnd - rowStart); ++row) {
+ inputArgs[0] = placeState.get(row);
+ for (int j = 0; j < column.length; ++j) {
+ inputArgs[j + 1] = inputs[j][row];
+ }
+ methodAccess.invoke(udf, addIndex, inputArgs);
+ }
+ } catch (Exception e) {
+ LOG.warn("invoke add function meet some error: " + Arrays.toString(e.getStackTrace()));
+ throw new UdfRuntimeException("UDAF failed to addBatchPlaces: ", e);
+ }
+ }
+
/**
* invoke add function, add row in loop [rowStart, rowEnd).
*/
@@ -224,10 +306,10 @@ public class UdafExecutor extends BaseExecutor {
protected long getCurrentOutputOffset(long row, boolean isArrayType) {
if (isArrayType) {
return Integer.toUnsignedLong(
- UdfUtils.UNSAFE.getInt(null, UdfUtils.UNSAFE.getLong(null, outputOffsetsPtr) + 8L * (row - 1)));
+ UdfUtils.UNSAFE.getInt(null, UdfUtils.UNSAFE.getLong(null, outputOffsetsPtr) + 8L * (row - 1)));
} else {
return Integer.toUnsignedLong(
- UdfUtils.UNSAFE.getInt(null, UdfUtils.UNSAFE.getLong(null, outputOffsetsPtr) + 4L * (row - 1)));
+ UdfUtils.UNSAFE.getInt(null, UdfUtils.UNSAFE.getLong(null, outputOffsetsPtr) + 4L * (row - 1)));
}
}
@@ -251,6 +333,7 @@ public class UdafExecutor extends BaseExecutor {
loader = ClassLoader.getSystemClassLoader();
}
Class<?> c = Class.forName(className, true, loader);
+ methodAccess = MethodAccess.get(c);
Constructor<?> ctor = c.getConstructor();
udf = ctor.newInstance();
Method[] methods = c.getDeclaredMethods();
@@ -281,7 +364,7 @@ public class UdafExecutor extends BaseExecutor {
}
case UDAF_ADD_FUNCTION: {
allMethods.put(methods[idx].getName(), methods[idx]);
-
+ addIndex = methodAccess.getIndex(UDAF_ADD_FUNCTION);
argClass = methods[idx].getParameterTypes();
if (argClass.length != parameterTypes.length + 1) {
LOG.debug("add function parameterTypes length not equal " + argClass.length + " "
diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java
index 4519b23a54..fb2ead5a3f 100644
--- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java
+++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java
@@ -37,263 +37,269 @@ import java.util.Arrays;
public class UdfConvert {
private static final Logger LOG = Logger.getLogger(UdfConvert.class);
- public static Object[] convertBooleanArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Boolean[] argument = new Boolean[numRows];
+ public static Object[] convertBooleanArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Boolean[] argument = new Boolean[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i);
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i);
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i);
}
}
return argument;
}
- public static Object[] convertTinyIntArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Byte[] argument = new Byte[numRows];
+ public static Object[] convertTinyIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Byte[] argument = new Byte[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getByte(null, columnAddr + i);
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getByte(null, columnAddr + i);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getByte(null, columnAddr + i);
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getByte(null, columnAddr + i);
}
}
return argument;
}
- public static Object[] convertSmallIntArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Short[] argument = new Short[numRows];
+ public static Object[] convertSmallIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Short[] argument = new Short[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L));
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L));
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L));
}
}
return argument;
}
- public static Object[] convertIntArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Integer[] argument = new Integer[numRows];
+ public static Object[] convertIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Integer[] argument = new Integer[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
}
}
return argument;
}
- public static Object[] convertBigIntArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Long[] argument = new Long[numRows];
+ public static Object[] convertBigIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Long[] argument = new Long[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
}
}
return argument;
}
- public static Object[] convertFloatArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Float[] argument = new Float[numRows];
+ public static Object[] convertFloatArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Float[] argument = new Float[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L));
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L));
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L));
}
}
return argument;
}
- public static Object[] convertDoubleArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- Double[] argument = new Double[numRows];
+ public static Object[] convertDoubleArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ Double[] argument = new Double[rowsEnd - rowsStart];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
- argument[i] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L));
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
- argument[i] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L));
+ for (int i = rowsStart; i < rowsEnd; ++i) {
+ argument[i - rowsStart] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L));
}
}
return argument;
}
- public static Object[] convertDateArg(Class argTypeClass, boolean isNullable, int numRows, long nullMapAddr,
- long columnAddr) {
- Object[] argument = (Object[]) Array.newInstance(argTypeClass, numRows);
+ public static Object[] convertDateArg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd,
+ long nullMapAddr, long columnAddr) {
+ Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart);
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
- argument[i] = UdfUtils.convertDateToJavaDate(value, argTypeClass);
+ argument[i - rowsStart] = UdfUtils.convertDateToJavaDate(value, argTypeClass);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
- argument[i] = UdfUtils.convertDateToJavaDate(value, argTypeClass);
+ argument[i - rowsStart] = UdfUtils.convertDateToJavaDate(value, argTypeClass);
}
}
return argument;
}
- public static Object[] convertDateTimeArg(Class argTypeClass, boolean isNullable, int numRows, long nullMapAddr,
- long columnAddr) {
- Object[] argument = (Object[]) Array.newInstance(argTypeClass, numRows);
+ public static Object[] convertDateTimeArg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd,
+ long nullMapAddr, long columnAddr) {
+ Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart);
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
- argument[i] = UdfUtils
+ argument[i - rowsStart] = UdfUtils
.convertDateTimeToJavaDateTime(value, argTypeClass);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
- argument[i] = UdfUtils.convertDateTimeToJavaDateTime(value, argTypeClass);
+ argument[i - rowsStart] = UdfUtils.convertDateTimeToJavaDateTime(value, argTypeClass);
}
}
return argument;
}
- public static Object[] convertDateV2Arg(Class argTypeClass, boolean isNullable, int numRows, long nullMapAddr,
- long columnAddr) {
- Object[] argument = (Object[]) Array.newInstance(argTypeClass, numRows);
+ public static Object[] convertDateV2Arg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd,
+ long nullMapAddr, long columnAddr) {
+ Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart);
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
int value = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
- argument[i] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass);
+ argument[i - rowsStart] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
int value = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L));
- argument[i] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass);
+ argument[i - rowsStart] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass);
}
}
return argument;
}
- public static Object[] convertDateTimeV2Arg(Class argTypeClass, boolean isNullable, int numRows, long nullMapAddr,
- long columnAddr) {
- Object[] argument = (Object[]) Array.newInstance(argTypeClass, numRows);
+ public static Object[] convertDateTimeV2Arg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd,
+ long nullMapAddr, long columnAddr) {
+ Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart);
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(null, nullMapAddr + i) == 0) {
long value = UdfUtils.UNSAFE.getLong(columnAddr + (i * 8L));
- argument[i] = UdfUtils
+ argument[i - rowsStart] = UdfUtils
.convertDateTimeV2ToJavaDateTime(value, argTypeClass);
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L));
- argument[i] = UdfUtils
+ argument[i - rowsStart] = UdfUtils
.convertDateTimeV2ToJavaDateTime(value, argTypeClass);
}
}
return argument;
}
- public static Object[] convertLargeIntArg(boolean isNullable, int numRows, long nullMapAddr, long columnAddr) {
- BigInteger[] argument = new BigInteger[numRows];
+ public static Object[] convertLargeIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
+ long columnAddr) {
+ BigInteger[] argument = new BigInteger[rowsEnd - rowsStart];
byte[] bytes = new byte[16];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
UdfUtils.copyMemory(null, columnAddr + (i * 16L), bytes, UdfUtils.BYTE_ARRAY_OFFSET, 16);
- argument[i] = new BigInteger(UdfUtils.convertByteOrder(bytes));
+ argument[i - rowsStart] = new BigInteger(UdfUtils.convertByteOrder(bytes));
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
UdfUtils.copyMemory(null, columnAddr + (i * 16L), bytes, UdfUtils.BYTE_ARRAY_OFFSET, 16);
- argument[i] = new BigInteger(UdfUtils.convertByteOrder(bytes));
+ argument[i - rowsStart] = new BigInteger(UdfUtils.convertByteOrder(bytes));
}
}
return argument;
}
- public static Object[] convertDecimalArg(int scale, long typeLen, boolean isNullable, int numRows, long nullMapAddr,
- long columnAddr) {
- BigDecimal[] argument = new BigDecimal[numRows];
+ public static Object[] convertDecimalArg(int scale, long typeLen, boolean isNullable, int rowsStart, int rowsEnd,
+ long nullMapAddr, long columnAddr) {
+ BigDecimal[] argument = new BigDecimal[rowsEnd - rowsStart];
byte[] bytes = new byte[(int) typeLen];
if (isNullable) {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) {
UdfUtils.copyMemory(null, columnAddr + (i * typeLen), bytes, UdfUtils.BYTE_ARRAY_OFFSET, typeLen);
BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes));
- argument[i] = new BigDecimal(bigInteger, scale); //show to pass scale info
+ argument[i - rowsStart] = new BigDecimal(bigInteger, scale); //show to pass scale info
} // else is the current row is null
}
} else {
- for (int i = 0; i < numRows; ++i) {
+ for (int i = rowsStart; i < rowsEnd; ++i) {
UdfUtils.copyMemory(null, columnAddr + (i * typeLen), bytes, UdfUtils.BYTE_ARRAY_OFFSET, typeLen);
BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes));
- argument[i] = new BigDecimal(bigInteger, scale);
+ argument[i - rowsStart] = new BigDecimal(bigInteger, scale);
}
}
return argument;
}
- public static Object[] convertStringArg(boolean isNullable, int numRows, long nullMapAddr,
+ public static Object[] convertStringArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr,
long charsAddr, long offsetsAddr) {
- String[] argument = new String[numRows];
+ String[] argument = new String[rowsEnd - rowsStart];
Preconditions.checkState(UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (0 - 1)) == 0,
"offsetsAddr[-1] should be 0;");
-
+ final int totalLen = UdfUtils.UNSAFE.getInt(null, offsetsAddr + (rowsEnd - 1) * 4L);
+ byte[] bytes = new byte[totalLen];
+ UdfUtils.copyMemory(null, charsAddr, bytes, UdfUtils.BYTE_ARRAY_OFFSET, totalLen);
if (isNullable) {
- for (int row = 0; row < numRows; ++row) {
+ for (int row = rowsStart; row < rowsEnd; ++row) {
if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) {
- int offset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + row * 4L);
- int numBytes = offset - UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1));
- long base = charsAddr + offset - numBytes;
- byte[] bytes = new byte[numBytes];
- UdfUtils.copyMemory(null, base, bytes, UdfUtils.BYTE_ARRAY_OFFSET, numBytes);
- argument[row] = new String(bytes, StandardCharsets.UTF_8);
+ int prevOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1));
+ int currOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + row * 4L);
+ argument[row - rowsStart] = new String(bytes, prevOffset, currOffset - prevOffset,
+ StandardCharsets.UTF_8);
} // else is the current row is null
}
} else {
- for (int row = 0; row < numRows; ++row) {
- int offset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + row * 4L);
- int numBytes = offset - UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1));
- long base = charsAddr + offset - numBytes;
- byte[] bytes = new byte[numBytes];
- UdfUtils.copyMemory(null, base, bytes, UdfUtils.BYTE_ARRAY_OFFSET, numBytes);
- argument[row] = new String(bytes, StandardCharsets.UTF_8);
+ for (int row = rowsStart; row < rowsEnd; ++row) {
+ int prevOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1));
+ int currOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * row);
+ argument[row - rowsStart] = new String(bytes, prevOffset, currOffset - prevOffset,
+ StandardCharsets.UTF_8);
}
}
return argument;
@@ -1314,7 +1320,7 @@ public class UdfConvert {
}
//////////////////////////////////////////convertArray///////////////////////////////////////////////////////////
- public static void convertArrayBooleanArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Boolean> convertArrayBooleanArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Boolean> data = null;
if (isNullable) {
@@ -1340,10 +1346,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayTinyIntArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Byte> convertArrayTinyIntArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Byte> data = null;
if (isNullable) {
@@ -1369,10 +1375,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArraySmallIntArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Short> convertArraySmallIntArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Short> data = null;
if (isNullable) {
@@ -1398,10 +1404,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayIntArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Integer> convertArrayIntArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Integer> data = null;
if (isNullable) {
@@ -1427,10 +1433,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayBigIntArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Long> convertArrayBigIntArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Long> data = null;
if (isNullable) {
@@ -1456,10 +1462,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayFloatArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Float> convertArrayFloatArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Float> data = null;
if (isNullable) {
@@ -1485,10 +1491,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDoubleArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<Double> convertArrayDoubleArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<Double> data = null;
if (isNullable) {
@@ -1514,10 +1520,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDateArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<LocalDate> convertArrayDateArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<LocalDate> data = null;
if (isNullable) {
@@ -1549,10 +1555,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDateTimeArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<LocalDateTime> convertArrayDateTimeArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<LocalDateTime> data = null;
if (isNullable) {
@@ -1582,10 +1588,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDateV2Arg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<LocalDate> convertArrayDateV2Arg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<LocalDate> data = null;
if (isNullable) {
@@ -1613,10 +1619,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDateTimeV2Arg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<LocalDateTime> convertArrayDateTimeV2Arg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<LocalDateTime> data = null;
if (isNullable) {
@@ -1646,10 +1652,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayLargeIntArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<BigInteger> convertArrayLargeIntArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<BigInteger> data = null;
byte[] bytes = new byte[16];
@@ -1678,10 +1684,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayDecimalArg(int scale, long typeLen, Object[] argument, int row, int currentRowNum,
+ public static ArrayList<BigDecimal> convertArrayDecimalArg(int scale, long typeLen, int row, int currentRowNum,
long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) {
ArrayList<BigDecimal> data = null;
@@ -1713,10 +1719,10 @@ public class UdfConvert {
}
} // for loop
} // end for all current row
- argument[row] = data;
+ return data;
}
- public static void convertArrayStringArg(Object[] argument, int row, int currentRowNum, long offsetStart,
+ public static ArrayList<String> convertArrayStringArg(int row, int currentRowNum, long offsetStart,
boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) {
ArrayList<String> data = null;
if (isNullable) {
@@ -1755,6 +1761,6 @@ public class UdfConvert {
}
}
}
- argument[row] = data;
+ return data;
}
}
diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
index f1993ec488..50528d007b 100644
--- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
+++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.jni.utils.UdfUtils;
import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType;
import org.apache.doris.thrift.TJavaUdfExecutorCtorParams;
+import com.esotericsoftware.reflectasm.MethodAccess;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -50,6 +51,8 @@ public class UdfExecutor extends BaseExecutor {
private long rowIdx;
private long batchSizePtr;
+ private int evaluateIndex;
+ private MethodAccess methodAccess;
/**
* Create a UdfExecutor, using parameters from a serialized thrift object. Used by
@@ -113,166 +116,14 @@ public class UdfExecutor extends BaseExecutor {
public Object[] convertBasicArguments(int argIdx, boolean isNullable, int numRows, long nullMapAddr,
long columnAddr, long strOffsetAddr) {
- switch (argTypes[argIdx]) {
- case BOOLEAN:
- return UdfConvert.convertBooleanArg(isNullable, numRows, nullMapAddr, columnAddr);
- case TINYINT:
- return UdfConvert.convertTinyIntArg(isNullable, numRows, nullMapAddr, columnAddr);
- case SMALLINT:
- return UdfConvert.convertSmallIntArg(isNullable, numRows, nullMapAddr, columnAddr);
- case INT:
- return UdfConvert.convertIntArg(isNullable, numRows, nullMapAddr, columnAddr);
- case BIGINT:
- return UdfConvert.convertBigIntArg(isNullable, numRows, nullMapAddr, columnAddr);
- case LARGEINT:
- return UdfConvert.convertLargeIntArg(isNullable, numRows, nullMapAddr, columnAddr);
- case FLOAT:
- return UdfConvert.convertFloatArg(isNullable, numRows, nullMapAddr, columnAddr);
- case DOUBLE:
- return UdfConvert.convertDoubleArg(isNullable, numRows, nullMapAddr, columnAddr);
- case CHAR:
- case VARCHAR:
- case STRING:
- return UdfConvert.convertStringArg(isNullable, numRows, nullMapAddr, columnAddr, strOffsetAddr);
- case DATE: // udaf maybe argClass[i + argClassOffset] need add +1
- return UdfConvert.convertDateArg(argClass[argIdx], isNullable, numRows, nullMapAddr, columnAddr);
- case DATETIME:
- return UdfConvert.convertDateTimeArg(argClass[argIdx], isNullable, numRows, nullMapAddr, columnAddr);
- case DATEV2:
- return UdfConvert.convertDateV2Arg(argClass[argIdx], isNullable, numRows, nullMapAddr, columnAddr);
- case DATETIMEV2:
- return UdfConvert.convertDateTimeV2Arg(argClass[argIdx], isNullable, numRows, nullMapAddr, columnAddr);
- case DECIMALV2:
- case DECIMAL128:
- return UdfConvert.convertDecimalArg(argTypes[argIdx].getScale(), 16L, isNullable, numRows, nullMapAddr,
- columnAddr);
- case DECIMAL32:
- return UdfConvert.convertDecimalArg(argTypes[argIdx].getScale(), 4L, isNullable, numRows, nullMapAddr,
- columnAddr);
- case DECIMAL64:
- return UdfConvert.convertDecimalArg(argTypes[argIdx].getScale(), 8L, isNullable, numRows, nullMapAddr,
- columnAddr);
- default: {
- LOG.info("Not support type: " + argTypes[argIdx].toString());
- Preconditions.checkState(false, "Not support type: " + argTypes[argIdx].toString());
- break;
- }
- }
- return null;
+ return convertBasicArg(true, argIdx, isNullable, 0, numRows, nullMapAddr, columnAddr, strOffsetAddr);
}
public Object[] convertArrayArguments(int argIdx, boolean isNullable, int numRows, long nullMapAddr,
long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) {
- Object[] argument = (Object[]) Array.newInstance(ArrayList.class, numRows);
- for (int row = 0; row < numRows; ++row) {
- long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1));
- long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row));
- int currentRowNum = (int) (offsetEnd - offsetStart);
- switch (argTypes[argIdx].getItemType().getPrimitiveType()) {
- case BOOLEAN: {
- UdfConvert
- .convertArrayBooleanArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case TINYINT: {
- UdfConvert
- .convertArrayTinyIntArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case SMALLINT: {
- UdfConvert
- .convertArraySmallIntArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case INT: {
- UdfConvert.convertArrayIntArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case BIGINT: {
- UdfConvert.convertArrayBigIntArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case LARGEINT: {
- UdfConvert
- .convertArrayLargeIntArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case FLOAT: {
- UdfConvert.convertArrayFloatArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DOUBLE: {
- UdfConvert.convertArrayDoubleArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case CHAR:
- case VARCHAR:
- case STRING: {
- UdfConvert.convertArrayStringArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr, strOffsetAddr);
- break;
- }
- case DATE: {
- UdfConvert.convertArrayDateArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DATETIME: {
- UdfConvert
- .convertArrayDateTimeArg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DATEV2: {
- UdfConvert.convertArrayDateV2Arg(argument, row, currentRowNum, offsetStart, isNullable, nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DATETIMEV2: {
- UdfConvert.convertArrayDateTimeV2Arg(argument, row, currentRowNum, offsetStart, isNullable,
- nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DECIMALV2:
- case DECIMAL128: {
- UdfConvert.convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, argument, row, currentRowNum,
- offsetStart, isNullable,
- nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DECIMAL32: {
- UdfConvert.convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, argument, row, currentRowNum,
- offsetStart, isNullable,
- nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- case DECIMAL64: {
- UdfConvert.convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, argument, row, currentRowNum,
- offsetStart, isNullable,
- nullMapAddr,
- nestedNullMapAddr, dataAddr);
- break;
- }
- default: {
- LOG.info("Not support: " + argTypes[argIdx]);
- Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString());
- break;
- }
- }
- }
- return argument;
+ return convertArrayArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, nestedNullMapAddr, dataAddr,
+ strOffsetAddr);
}
/**
@@ -287,7 +138,7 @@ public class UdfExecutor extends BaseExecutor {
for (int j = 0; j < column.length; ++j) {
parameters[j] = inputs[j][i];
}
- result[i] = method.invoke(udf, parameters);
+ result[i] = methodAccess.invoke(udf, evaluateIndex, parameters);
}
return result;
} catch (Exception e) {
@@ -581,6 +432,7 @@ public class UdfExecutor extends BaseExecutor {
loader = ClassLoader.getSystemClassLoader();
}
Class<?> c = Class.forName(className, true, loader);
+ methodAccess = MethodAccess.get(c);
Constructor<?> ctor = c.getConstructor();
udf = ctor.newInstance();
Method[] methods = c.getMethods();
@@ -597,6 +449,7 @@ public class UdfExecutor extends BaseExecutor {
continue;
}
method = m;
+ evaluateIndex = methodAccess.getIndex(UDF_FUNCTION_NAME);
Pair<Boolean, JavaUdfDataType> returnType;
if (argClass.length == 0 && parameterTypes.length == 0) {
// Special case where the UDF doesn't take any input args
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 13/18: [enhancement](multi-table) enable mullti table routine load on pipeline engine (#21729)
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5170c50ca9ae0b0ae29f1e0cc564ca83f88e1150
Author: Siyang Tang <82...@users.noreply.github.com>
AuthorDate: Fri Jul 14 12:16:32 2023 +0800
[enhancement](multi-table) enable mullti table routine load on pipeline engine (#21729)
---
be/src/io/file_factory.cpp | 27 +++++---
be/src/io/file_factory.h | 2 +-
be/src/io/fs/multi_table_pipe.cpp | 85 +++++++++++++++++++------
be/src/io/fs/multi_table_pipe.h | 3 +
be/src/io/fs/stream_load_pipe.cpp | 7 ++
be/src/io/fs/stream_load_pipe.h | 3 +
be/src/pipeline/pipeline_fragment_context.cpp | 6 +-
be/src/runtime/runtime_state.cpp | 6 +-
be/src/runtime/runtime_state.h | 9 ++-
be/src/vec/exec/format/csv/csv_reader.cpp | 3 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 3 +-
11 files changed, 114 insertions(+), 40 deletions(-)
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index adc02a0aa2..d46d2c5b4c 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -144,7 +144,7 @@ Status FileFactory::create_file_reader(const io::FileSystemProperties& system_pr
// file scan node/stream load pipe
Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderSPtr* file_reader,
- const TUniqueId& fragment_instance_id) {
+ RuntimeState* runtime_state) {
auto stream_load_ctx = ExecEnv::GetInstance()->new_load_stream_mgr()->get(load_id);
if (!stream_load_ctx) {
return Status::InternalError("unknown stream load id: {}", UniqueId(load_id).to_string());
@@ -152,15 +152,26 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS
*file_reader = stream_load_ctx->pipe;
- if (file_reader->get() != nullptr) {
- auto multi_table_pipe = std::dynamic_pointer_cast<io::MultiTablePipe>(*file_reader);
- if (multi_table_pipe != nullptr) {
- *file_reader = multi_table_pipe->getPipe(fragment_instance_id);
- LOG(INFO) << "create pipe reader for fragment instance: " << fragment_instance_id
- << " pipe: " << (*file_reader).get();
- }
+ if (file_reader->get() == nullptr) {
+ return Status::OK();
}
+ auto multi_table_pipe = std::dynamic_pointer_cast<io::MultiTablePipe>(*file_reader);
+ if (multi_table_pipe == nullptr || runtime_state == nullptr) {
+ return Status::OK();
+ }
+
+ TUniqueId pipe_id;
+ if (runtime_state->enable_pipeline_exec()) {
+ pipe_id = io::StreamLoadPipe::calculate_pipe_id(runtime_state->query_id(),
+ runtime_state->fragment_id());
+ } else {
+ pipe_id = runtime_state->fragment_instance_id();
+ }
+ *file_reader = multi_table_pipe->getPipe(pipe_id);
+ LOG(INFO) << "create pipe reader for fragment instance: " << pipe_id
+ << " pipe: " << (*file_reader).get();
+
return Status::OK();
}
diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h
index 42589df531..a311b8d58b 100644
--- a/be/src/io/file_factory.h
+++ b/be/src/io/file_factory.h
@@ -66,7 +66,7 @@ public:
// Create FileReader for stream load pipe
static Status create_pipe_reader(const TUniqueId& load_id, io::FileReaderSPtr* file_reader,
- const TUniqueId& fragment_instance_id);
+ RuntimeState* runtime_state);
static Status create_hdfs_reader(const THdfsParams& hdfs_params, const io::FileDescription& fd,
const io::FileReaderOptions& reader_options,
diff --git a/be/src/io/fs/multi_table_pipe.cpp b/be/src/io/fs/multi_table_pipe.cpp
index 853baa4f12..12de3fa8d4 100644
--- a/be/src/io/fs/multi_table_pipe.cpp
+++ b/be/src/io/fs/multi_table_pipe.cpp
@@ -20,11 +20,15 @@
#include <gen_cpp/FrontendService.h>
#include <gen_cpp/FrontendService_types.h>
#include <gen_cpp/HeartbeatService_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/Types_types.h>
#include <thrift/protocol/TDebugProtocol.h>
+#include <type_traits>
+
#include "common/status.h"
#include "runtime/client_cache.h"
+#include "runtime/exec_env.h"
#include "runtime/fragment_mgr.h"
#include "runtime/runtime_state.h"
#include "runtime/stream_load/new_load_stream_mgr.h"
@@ -130,7 +134,9 @@ Status MultiTablePipe::dispatch(const std::string& table, const char* data, size
#ifndef BE_TEST
Status MultiTablePipe::request_and_exec_plans() {
- if (_unplanned_pipes.empty()) return Status::OK();
+ if (_unplanned_pipes.empty()) {
+ return Status::OK();
+ }
// get list of table names in unplanned pipes
std::vector<std::string> tables;
@@ -175,24 +181,52 @@ Status MultiTablePipe::request_and_exec_plans() {
return plan_status;
}
+ Status st;
+ if (_ctx->multi_table_put_result.__isset.params &&
+ !_ctx->multi_table_put_result.__isset.pipeline_params) {
+ st = exec_plans(exec_env, _ctx->multi_table_put_result.params);
+ } else if (!_ctx->multi_table_put_result.__isset.params &&
+ _ctx->multi_table_put_result.__isset.pipeline_params) {
+ st = exec_plans(exec_env, _ctx->multi_table_put_result.pipeline_params);
+ } else {
+ return Status::Aborted("too many or too few params are set in multi_table_put_result.");
+ }
+
+ return st;
+}
+
+template <typename ExecParam>
+Status MultiTablePipe::exec_plans(ExecEnv* exec_env, std::vector<ExecParam> params) {
// put unplanned pipes into planned pipes and clear unplanned pipes
for (auto& pipe : _unplanned_pipes) {
_ctx->table_list.push_back(pipe.first);
_planned_pipes.emplace(pipe.first, pipe.second);
}
LOG(INFO) << fmt::format("{} tables plan complete, planned table cnt={}, returned plan cnt={}",
- _unplanned_pipes.size(), _planned_pipes.size(),
- _ctx->multi_table_put_result.params.size());
+ _unplanned_pipes.size(), _planned_pipes.size(), params.size());
_unplanned_pipes.clear();
- _inflight_plan_cnt += _ctx->multi_table_put_result.params.size();
- for (auto& plan : _ctx->multi_table_put_result.params) {
- // TODO: use pipeline in the future (currently is buggy for load)
- DCHECK_EQ(plan.__isset.table_name, true);
- DCHECK(_planned_pipes.find(plan.table_name) != _planned_pipes.end());
- putPipe(plan.params.fragment_instance_id, _planned_pipes[plan.table_name]);
- LOG(INFO) << "fragment_instance_id=" << plan.params.fragment_instance_id
- << " table=" << plan.table_name;
+ _inflight_plan_cnt += params.size();
+ for (auto& plan : params) {
+ if (!plan.__isset.table_name ||
+ _planned_pipes.find(plan.table_name) == _planned_pipes.end()) {
+ return Status::Aborted("Missing vital param: table_name");
+ }
+
+ if constexpr (std::is_same_v<ExecParam, TExecPlanFragmentParams>) {
+ putPipe(plan.params.fragment_instance_id, _planned_pipes[plan.table_name]);
+ LOG(INFO) << "fragment_instance_id=" << plan.params.fragment_instance_id
+ << " table=" << plan.table_name;
+ } else if constexpr (std::is_same_v<ExecParam, TPipelineFragmentParams>) {
+ auto pipe_id = calculate_pipe_id(plan.query_id, plan.fragment_id);
+ putPipe(pipe_id, _planned_pipes[plan.table_name]);
+ LOG(INFO) << "pipe_id=" << pipe_id << "table=" << plan.table_name;
+ } else {
+ LOG(WARNING) << "illegal exec param type, need `TExecPlanFragmentParams` or "
+ "`TPipelineFragmentParams`, will crash";
+ CHECK(false);
+ }
+
exec_env->fragment_mgr()->exec_plan_fragment(plan, [this](RuntimeState* state,
Status* status) {
{
@@ -243,6 +277,7 @@ Status MultiTablePipe::request_and_exec_plans() {
return Status::OK();
}
+
#else
Status MultiTablePipe::request_and_exec_plans() {
// put unplanned pipes into planned pipes
@@ -254,36 +289,46 @@ Status MultiTablePipe::request_and_exec_plans() {
_unplanned_pipes.clear();
return Status::OK();
}
+
+template <typename ExecParam>
+Status MultiTablePipe::exec_plans(ExecEnv* exec_env, std::vector<ExecParam> params) {
+ return Status::OK();
+}
+
#endif
-Status MultiTablePipe::putPipe(const TUniqueId& fragment_instance_id,
- std::shared_ptr<io::StreamLoadPipe> pipe) {
+Status MultiTablePipe::putPipe(const TUniqueId& pipe_id, std::shared_ptr<io::StreamLoadPipe> pipe) {
std::lock_guard<std::mutex> l(_pipe_map_lock);
- auto it = _pipe_map.find(fragment_instance_id);
+ auto it = _pipe_map.find(pipe_id);
if (it != std::end(_pipe_map)) {
return Status::InternalError("id already exist");
}
- _pipe_map.emplace(fragment_instance_id, pipe);
+ _pipe_map.emplace(pipe_id, pipe);
return Status::OK();
}
-std::shared_ptr<io::StreamLoadPipe> MultiTablePipe::getPipe(const TUniqueId& fragment_instance_id) {
+std::shared_ptr<io::StreamLoadPipe> MultiTablePipe::getPipe(const TUniqueId& pipe_id) {
std::lock_guard<std::mutex> l(_pipe_map_lock);
- auto it = _pipe_map.find(fragment_instance_id);
+ auto it = _pipe_map.find(pipe_id);
if (it == std::end(_pipe_map)) {
return std::shared_ptr<io::StreamLoadPipe>(nullptr);
}
return it->second;
}
-void MultiTablePipe::removePipe(const TUniqueId& fragment_instance_id) {
+void MultiTablePipe::removePipe(const TUniqueId& pipe_id) {
std::lock_guard<std::mutex> l(_pipe_map_lock);
- auto it = _pipe_map.find(fragment_instance_id);
+ auto it = _pipe_map.find(pipe_id);
if (it != std::end(_pipe_map)) {
_pipe_map.erase(it);
- VLOG_NOTICE << "remove stream load pipe: " << fragment_instance_id;
+ VLOG_NOTICE << "remove stream load pipe: " << pipe_id;
}
}
+template Status MultiTablePipe::exec_plans(ExecEnv* exec_env,
+ std::vector<TExecPlanFragmentParams> params);
+template Status MultiTablePipe::exec_plans(ExecEnv* exec_env,
+ std::vector<TPipelineFragmentParams> params);
+
} // namespace io
} // namespace doris
\ No newline at end of file
diff --git a/be/src/io/fs/multi_table_pipe.h b/be/src/io/fs/multi_table_pipe.h
index b225ea3061..fd30722256 100644
--- a/be/src/io/fs/multi_table_pipe.h
+++ b/be/src/io/fs/multi_table_pipe.h
@@ -68,6 +68,9 @@ private:
// [thread-unsafe] dispatch data to corresponding KafkaConsumerPipe
Status dispatch(const std::string& table, const char* data, size_t size, AppendFunc cb);
+ template <typename ExecParam>
+ Status exec_plans(ExecEnv* exec_env, std::vector<ExecParam> params);
+
private:
std::unordered_map<std::string /*table*/, KafkaConsumerPipePtr> _planned_pipes;
std::unordered_map<std::string /*table*/, KafkaConsumerPipePtr> _unplanned_pipes;
diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp
index d79cfd028d..23a4e3c604 100644
--- a/be/src/io/fs/stream_load_pipe.cpp
+++ b/be/src/io/fs/stream_load_pipe.cpp
@@ -250,5 +250,12 @@ void StreamLoadPipe::cancel(const std::string& reason) {
_put_cond.notify_all();
}
+TUniqueId StreamLoadPipe::calculate_pipe_id(const UniqueId& query_id, int32_t fragment_id) {
+ TUniqueId pipe_id;
+ pipe_id.lo = query_id.lo + fragment_id;
+ pipe_id.hi = query_id.hi;
+ return pipe_id;
+}
+
} // namespace io
} // namespace doris
diff --git a/be/src/io/fs/stream_load_pipe.h b/be/src/io/fs/stream_load_pipe.h
index 848175ce9a..a184cf9e78 100644
--- a/be/src/io/fs/stream_load_pipe.h
+++ b/be/src/io/fs/stream_load_pipe.h
@@ -78,6 +78,9 @@ public:
size_t get_queue_size() { return _buf_queue.size(); }
+ // used for pipeline load, which use TUniqueId(lo: query_id.lo + fragment_id, hi: query_id.hi) as pipe_id
+ static TUniqueId calculate_pipe_id(const UniqueId& query_id, int32_t fragment_id);
+
protected:
Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
const IOContext* io_ctx) override;
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 9fc8c4ce24..ea0a6c9240 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -214,9 +214,9 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re
.tag("pthread_id", (uintptr_t)pthread_self());
// 1. init _runtime_state
- _runtime_state =
- RuntimeState::create_unique(local_params, request.query_id, request.query_options,
- _query_ctx->query_globals, _exec_env);
+ _runtime_state = RuntimeState::create_unique(local_params, request.query_id,
+ request.fragment_id, request.query_options,
+ _query_ctx->query_globals, _exec_env);
_runtime_state->set_query_ctx(_query_ctx.get());
_runtime_state->set_query_mem_tracker(_query_ctx->query_mem_tracker);
_runtime_state->set_tracer(std::move(tracer));
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 66bc4a5743..28de6c752e 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -99,8 +99,9 @@ RuntimeState::RuntimeState(const TPlanFragmentExecParams& fragment_exec_params,
}
RuntimeState::RuntimeState(const TPipelineInstanceParams& pipeline_params,
- const TUniqueId& query_id, const TQueryOptions& query_options,
- const TQueryGlobals& query_globals, ExecEnv* exec_env)
+ const TUniqueId& query_id, int32_t fragment_id,
+ const TQueryOptions& query_options, const TQueryGlobals& query_globals,
+ ExecEnv* exec_env)
: _profile("Fragment " + print_id(pipeline_params.fragment_instance_id)),
_load_channel_profile("<unnamed>"),
_obj_pool(new ObjectPool()),
@@ -108,6 +109,7 @@ RuntimeState::RuntimeState(const TPipelineInstanceParams& pipeline_params,
_data_stream_recvrs_pool(new ObjectPool()),
_unreported_error_idx(0),
_query_id(query_id),
+ _fragment_id(fragment_id),
_is_cancelled(false),
_per_fragment_instance_idx(0),
_num_rows_load_total(0),
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index d1b90579e2..89543e492b 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -39,6 +39,7 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/factory_creator.h"
#include "common/status.h"
+#include "gutil/integral_types.h"
#include "util/runtime_profile.h"
#include "util/telemetry/telemetry.h"
@@ -66,8 +67,8 @@ public:
ExecEnv* exec_env);
RuntimeState(const TPipelineInstanceParams& pipeline_params, const TUniqueId& query_id,
- const TQueryOptions& query_options, const TQueryGlobals& query_globals,
- ExecEnv* exec_env);
+ int32 fragment_id, const TQueryOptions& query_options,
+ const TQueryGlobals& query_globals, ExecEnv* exec_env);
// RuntimeState for executing expr in fe-support.
RuntimeState(const TQueryGlobals& query_globals);
@@ -115,6 +116,8 @@ public:
const std::string& user() const { return _user; }
const TUniqueId& query_id() const { return _query_id; }
const TUniqueId& fragment_instance_id() const { return _fragment_instance_id; }
+ // should only be called in pipeline engine
+ int32_t fragment_id() const { return _fragment_id; }
ExecEnv* exec_env() { return _exec_env; }
std::shared_ptr<MemTrackerLimiter> query_mem_tracker() const;
@@ -460,6 +463,8 @@ private:
cctz::time_zone _timezone_obj;
TUniqueId _query_id;
+ // fragment id for each TPipelineFragmentParams
+ int32_t _fragment_id;
TUniqueId _fragment_instance_id;
TQueryOptions _query_options;
ExecEnv* _exec_env = nullptr;
diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp
index 7390d9103a..9da29ed6dc 100644
--- a/be/src/vec/exec/format/csv/csv_reader.cpp
+++ b/be/src/vec/exec/format/csv/csv_reader.cpp
@@ -166,8 +166,7 @@ Status CsvReader::init_reader(bool is_load) {
_file_description.start_offset = start_offset;
if (_params.file_type == TFileType::FILE_STREAM) {
- RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader,
- _state->fragment_instance_id()));
+ RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader, _state));
} else {
io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state);
_file_description.mtime = _range.__isset.modification_time ? _range.modification_time : 0;
diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
index f6eabaa7cd..7d17f650c6 100644
--- a/be/src/vec/exec/format/json/new_json_reader.cpp
+++ b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -378,8 +378,7 @@ Status NewJsonReader::_open_file_reader() {
_file_description.start_offset = start_offset;
if (_params.file_type == TFileType::FILE_STREAM) {
- RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader,
- _state->fragment_instance_id()));
+ RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader, _state));
} else {
io::FileReaderOptions reader_options = FileFactory::get_reader_options(_state);
_file_description.mtime = _range.__isset.modification_time ? _range.modification_time : 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/18: [FIX](map) fix map key-column nullable for arrow serde #21762
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9b61fcbf22c3cd5e57410b6da009b7474cd9afe2
Author: amory <wa...@selectdb.com>
AuthorDate: Fri Jul 14 00:30:07 2023 +0800
[FIX](map) fix map key-column nullable for arrow serde #21762
arrow is not support key column has null element , but doris default map key column is nullable , so need to deal with if doris map row if key column has null element , we put null to arrow
---
.../vec/data_types/serde/data_type_map_serde.cpp | 15 +++-
.../serde/data_type_serde_arrow_test.cpp | 84 +++++++++++++++++++++-
2 files changed, 96 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp b/be/src/vec/data_types/serde/data_type_map_serde.cpp
index fb3eddc4ff..edb21a60ef 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_map_serde.cpp
@@ -19,6 +19,7 @@
#include "arrow/array/builder_nested.h"
#include "util/jsonb_document.h"
+#include "util/simd/bits.h"
#include "vec/columns/column.h"
#include "vec/columns/column_const.h"
#include "vec/columns/column_map.h"
@@ -50,13 +51,23 @@ void DataTypeMapSerDe::write_column_to_arrow(const IColumn& column, const NullMa
auto& builder = assert_cast<arrow::MapBuilder&>(*array_builder);
auto& map_column = assert_cast<const ColumnMap&>(column);
const IColumn& nested_keys_column = map_column.get_keys();
- CHECK(!nested_keys_column.is_nullable());
const IColumn& nested_values_column = map_column.get_values();
+ // now we default set key value in map is nullable
+ DCHECK(nested_keys_column.is_nullable());
+ DCHECK(nested_values_column.is_nullable());
+ auto keys_nullmap_data =
+ check_and_get_column<ColumnNullable>(nested_keys_column)->get_null_map_data().data();
auto& offsets = map_column.get_offsets();
auto key_builder = builder.key_builder();
auto value_builder = builder.item_builder();
+
for (size_t r = start; r < end; ++r) {
- if (null_map && (*null_map)[r]) {
+ if ((null_map && (*null_map)[r])) {
+ checkArrowStatus(builder.AppendNull(), column.get_name(),
+ array_builder->type()->name());
+ } else if (simd::contain_byte(keys_nullmap_data + offsets[r - 1],
+ offsets[r] - offsets[r - 1], 1)) {
+ // arrow do not support key is null so we just put null with this row
checkArrowStatus(builder.AppendNull(), column.get_name(),
array_builder->type()->name());
} else {
diff --git a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
index 7792d40839..c1913e6d86 100644
--- a/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_arrow_test.cpp
@@ -356,7 +356,9 @@ void serialize_and_deserialize_arrow_test() {
type_desc.add_sub_type(TYPE_STRING, true);
tslot.__set_slotType(type_desc.to_thrift());
{
- DataTypePtr s = std::make_shared<DataTypeString>();
+ DataTypePtr s =
+ std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
+ ;
DataTypePtr d =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
DataTypePtr m = std::make_shared<DataTypeMap>(s, d);
@@ -503,4 +505,84 @@ TEST(DataTypeSerDeArrowTest, DataTypeCollectionSerDeTest) {
serialize_and_deserialize_arrow_test<false>();
}
+TEST(DataTypeSerDeArrowTest, DataTypeMapNullKeySerDeTest) {
+ TupleDescriptor tuple_desc(PTupleDescriptor(), true);
+ TSlotDescriptor tslot;
+ std::string col_name = "map_null_key";
+ tslot.__set_colName(col_name);
+ TypeDescriptor type_desc(TYPE_MAP);
+ type_desc.add_sub_type(TYPE_STRING, true);
+ type_desc.add_sub_type(TYPE_INT, true);
+ tslot.__set_slotType(type_desc.to_thrift());
+ vectorized::Block block;
+ {
+ DataTypePtr s = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
+ ;
+ DataTypePtr d = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>());
+ DataTypePtr m = std::make_shared<DataTypeMap>(s, d);
+ Array k1, k2, v1, v2, k3, v3;
+ k1.push_back(Null());
+ k1.push_back("doris");
+ k1.push_back("clever amory");
+ v1.push_back(11);
+ v1.push_back(Null());
+ v1.push_back(30);
+ k2.push_back("hello amory");
+ k2.push_back("NULL");
+ k2.push_back("cute amory");
+ k2.push_back("doris");
+ v2.push_back(26);
+ v2.push_back(Null());
+ v2.push_back(6);
+ v2.push_back(7);
+ k3.push_back("test");
+ k3.push_back(Null());
+ v3.push_back(11);
+ v3.push_back(30);
+ Map m1, m2, m3;
+ m1.push_back(k1);
+ m1.push_back(v1);
+ m2.push_back(k2);
+ m2.push_back(v2);
+ m3.push_back(k3);
+ m3.push_back(v3);
+ MutableColumnPtr map_column = m->create_column();
+ map_column->reserve(3);
+ map_column->insert(m1);
+ map_column->insert(m2);
+ map_column->insert(m3);
+ vectorized::ColumnWithTypeAndName type_and_name(map_column->get_ptr(), m, col_name);
+ block.insert(type_and_name);
+ }
+
+ tslot.__set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(tslot);
+ tuple_desc.add_slot(slot);
+ RowDescriptor row_desc(&tuple_desc, true);
+ // arrow schema
+ std::shared_ptr<arrow::Schema> _arrow_schema;
+ EXPECT_EQ(convert_to_arrow_schema(row_desc, &_arrow_schema), Status::OK());
+
+ // serialize
+ std::shared_ptr<arrow::RecordBatch> result;
+ std::cout << "block structure: " << block.dump_structure() << std::endl;
+ std::cout << "_arrow_schema: " << _arrow_schema->ToString(true) << std::endl;
+
+ convert_to_arrow_batch(block, _arrow_schema, arrow::default_memory_pool(), &result);
+ Block new_block = block.clone_empty();
+ EXPECT_TRUE(result != nullptr);
+ std::cout << "result: " << result->ToString() << std::endl;
+ // deserialize
+ auto* array = result->GetColumnByName(col_name).get();
+ auto& column_with_type_and_name = new_block.get_by_name(col_name);
+ arrow_column_to_doris_column(array, 0, column_with_type_and_name.column,
+ column_with_type_and_name.type, block.rows(), "UTC");
+ std::cout << block.dump_data() << std::endl;
+ std::cout << new_block.dump_data() << std::endl;
+ // new block row_index 0, 2 is should be empty
+ EXPECT_EQ(new_block.dump_one_line(0, 1), "{}");
+ EXPECT_EQ(new_block.dump_one_line(2, 1), "{}");
+ EXPECT_EQ(block.dump_data(1, 1), new_block.dump_data(1, 1));
+}
+
} // namespace doris::vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org