You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/03/01 03:57:27 UTC
[incubator-doris] branch array-type updated (e1d75ee -> d256206)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch array-type
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
omit e1d75ee [feature-wip][array-type] Support ArrayLiteral in SQL. (#8089)
omit 0d14205 [feature-wip](array-type) Create table with nested array type. (#8003)
add b1e7343 [Vectorized] [HashJoin] Opt HashJoin Performance (#8119)
add f4c6de5 [docs] update backup and restore docs (#8161)
add 9120de2 [refactor] fix some typos (#8159)
add 31ab569 [Vectorized][Feature] support some bitmap functions (#8138)
add d17ed5e [vectorization](storage)support seq column in storage layer (#8186)
add e3f1efc [Vec][Storage] Support delete condition;ut (#8091)
add 01fb25a [UT] Fix the UT of column_nullable_test (#8180)
add 83543c6 [improvement](storage)Using Be config to switch storage layer vectorization #8166
add 0726a43 [fix](be-ut) Fix unused-but-set-variable errors. (#8211)
add 9a7931c [fix](mem-pool) fix bug that mem pool failed to allocate in ASAN mode (#8216)
add 90a8ca8 [Bug][Vectorized] fix bitmap_min(empty) not return null (#8190)
add a6bc9cb [Function] Refactor the function code of log (#8199)
add 0dcbfbd [fix](load) Fix InsertStmt prepareExpressions (#8112)
add dccb3cf [git] Ignore all the .flattened-pom.xml files (#8197)
add b93936c [Feature][array-type]add proto for complex data type ARRAY (#8212)
add df7e848 [chore](dependency) upgrade-grpc-version (#8218)
add 4c5d7c2 [Bug] group_concat(value,null) not return null
add 8642fa3 [Bug] Double/Float % 0 should be NULL (#8230)
add 6f4cf57 [Bug][Vectorized] Fix bug of decimal cast to double loss precision (#8221)
add ddf08cc [refactor](fe) Remove version hash on FE side (#8099)
add cce721a [improvement](olap) using placement-new to avoid dynamic mallocing for ParsedPage (#8172)
add f7c18d3 [Improvement] Add minimum fe meta version check (#8203)
add 40c1fa2 [refactor] change mysql server version to avoid some cve issues (#8223)
add a630e03 [Enhancement](routine_load) Support show routine load statement with like predicate (#8188)
add 83521a8 [Feature](create_table) Support create table with random distribution to avoid data skew (#8041)
add 668188b [improvement][vectorized] support es node predicate peel (#8174)
add 87b96cf [feature](iceberg) Step3: Support query iceberg external table (#8179)
add 385ccf7 [fix](routine-load) fix show routine load task error (#8195)
add 30fbf1d [typo](doc)fix some confusing doc content (#8239)
add 351fbbc [fix](olap) add check statement to protect get_dict_word_info() from crash (#8241)
add 7f3564c [chore] Support aarch64 target with ldb_toolchain (#8249)
add c66a9bf [fix](be-ut) fix unit test bug for tablet_info_test (#8253)
add 27d2e3e [refactor](fe) Remove old fe meta version (#8246)
add 01365f1 [community] add more collaborators in .asf.yaml (#8029) (#8252)
add e77e2b0 [improvement](lateral-view) Add number rows filtered in profile (#8251)
add 13f007b [doc] Modify document of compilation on ARM64 (#8254)
add 0fce094 [typo] fix listdb description error (#8257)
add 7d0e36a [fix](be-ut) fix bitmap_ut result wrong && fix schema_change compile error (#8261)
add 757e357 [refactor] remove unused new_in_predicate code (#8263)
add 93c638f [fix][chore](insert)(fe) Fix analysis error of insert stmt and modify grpc-netty dependency (#8265)
add ada39dd [improvement][vec] better memequal impl to speed up string compare (#8229)
new 37becad [feature-wip](array-type) Create table with nested array type. (#8003)
new d256206 [feature-wip][array-type] Support ArrayLiteral in SQL. (#8089)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e1d75ee)
\
N -- N -- N refs/heads/array-type (d256206)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 2 +
.gitignore | 8 +-
be/CMakeLists.txt | 1 -
be/src/common/config.h | 2 +
be/src/common/daemon.cpp | 2 -
be/src/exec/es/es_predicate.cpp | 2 +-
be/src/exec/es/es_predicate.h | 4 +-
be/src/exec/es/es_query_builder.cpp | 121 ++--
be/src/exec/es/es_query_builder.h | 1 +
be/src/exec/es_http_scan_node.cpp | 24 +-
be/src/exec/es_http_scan_node.h | 10 +-
be/src/exec/olap_scan_node.cpp | 58 +-
be/src/exec/olap_scan_node.h | 14 +-
be/src/exec/scan_node.cpp | 28 +
be/src/exec/scan_node.h | 9 +-
be/src/exec/table_function_node.cpp | 8 +-
be/src/exec/table_function_node.h | 3 +
be/src/exec/tablet_info.cpp | 113 ++--
be/src/exec/tablet_info.h | 16 +-
be/src/exec/tablet_sink.cpp | 29 +-
be/src/exec/tablet_sink.h | 13 +
be/src/exprs/CMakeLists.txt | 1 -
be/src/exprs/aggregate_functions.cpp | 4 +-
be/src/exprs/arithmetic_expr.cpp | 4 +-
be/src/exprs/math_functions.cpp | 12 +-
be/src/exprs/new_in_predicate.cpp | 185 ------
be/src/exprs/new_in_predicate.h | 348 -----------
be/src/exprs/table_function/explode_bitmap.cpp | 4 +-
be/src/exprs/table_function/explode_json_array.cpp | 14 +
be/src/exprs/table_function/explode_split.cpp | 1 +
be/src/exprs/table_function/table_function.h | 1 +
be/src/glibc-compatibility/CMakeLists.txt | 9 +-
be/src/glibc-compatibility/FastMemcpy.c | 220 -------
be/src/glibc-compatibility/FastMemcpy.h | 694 ---------------------
be/src/glibc-compatibility/LICENSE_FastMemcpy | 22 -
.../glibc-compatibility/memcpy/memcpy_aarch64.cpp | 245 ++++++++
.../glibc-compatibility/memcpy/memcpy_x86_64.cpp | 220 +++++++
be/src/glibc-compatibility/memcpy_wrapper.c | 6 -
be/src/http/action/stream_load.cpp | 8 +
be/src/http/http_common.h | 5 +-
be/src/olap/comparison_predicate.cpp | 62 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 105 ++--
be/src/olap/rowset/column_reader.h | 1 -
be/src/olap/rowset/segment_v2/binary_plain_page.h | 2 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 49 +-
be/src/olap/rowset/segment_v2/column_reader.h | 5 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 28 +-
.../olap/rowset/segment_v2/indexed_column_reader.h | 2 +-
be/src/olap/rowset/segment_v2/parsed_page.h | 19 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 18 +-
be/src/runtime/descriptors.cpp | 28 +
be/src/runtime/descriptors.h | 18 +
be/src/runtime/mem_pool.h | 4 +-
be/src/runtime/string_search.hpp | 68 +-
be/src/runtime/string_value.hpp | 4 +
be/src/util/bitmap_value.h | 101 ++-
be/src/vec/CMakeLists.txt | 2 +
be/src/vec/columns/column_complex.h | 12 +-
be/src/vec/columns/column_decimal.h | 10 +-
be/src/vec/columns/column_string.cpp | 6 +-
be/src/vec/columns/column_vector.cpp | 17 +-
be/src/vec/common/cow.h | 5 -
be/src/vec/common/string_ref.h | 107 +---
be/src/vec/core/block.h | 25 +-
be/src/vec/data_types/data_type_decimal.h | 2 +-
be/src/vec/exec/join/join_op.h | 12 +-
be/src/vec/exec/join/vhash_join_node.cpp | 286 +++++----
be/src/vec/exec/join/vhash_join_node.h | 10 +-
be/src/vec/exec/ves_http_scanner.cpp | 8 +-
be/src/vec/exec/ves_http_scanner.h | 24 +-
be/src/vec/exec/vset_operation_node.cpp | 42 +-
be/src/vec/exec/vset_operation_node.h | 10 +-
.../vec/functions/function_always_not_nullable.h | 3 -
be/src/vec/functions/function_binary_arithmetic.h | 7 +
be/src/vec/functions/function_bitmap.cpp | 336 +++++-----
be/src/vec/functions/function_bitmap_min_or_max.h | 100 +++
be/src/vec/functions/function_bitmap_variadic.cpp | 142 +++++
be/src/vec/functions/function_const.h | 2 +-
be/src/vec/functions/function_convert_tz.h | 5 +-
.../function_date_or_datetime_computation.cpp | 6 +-
.../function_date_or_datetime_computation.h | 2 +
.../function_date_or_datetime_to_something.h | 2 +
.../function_date_or_datetime_to_string.h | 2 +
.../functions/function_datetime_string_to_string.h | 2 +
.../{function_convert_tz.cpp => function_fake.cpp} | 6 +-
...{function_multi_same_args.h => function_fake.h} | 36 +-
be/src/vec/functions/function_grouping.h | 10 +-
be/src/vec/functions/function_hash.h | 1 +
be/src/vec/functions/function_ifnull.h | 49 +-
.../vec/functions/function_math_binary_float64.h | 248 --------
be/src/vec/functions/function_math_unary.h | 1 +
..._unary.h => function_math_unary_to_null_type.h} | 87 +--
be/src/vec/functions/function_multi_same_args.h | 4 +-
be/src/vec/functions/function_rpc.h | 2 +
be/src/vec/functions/function_string_to_string.h | 2 +
be/src/vec/functions/math.cpp | 131 ++--
be/src/vec/functions/modulo.cpp | 2 +-
be/src/vec/functions/simple_function_factory.h | 5 +
be/src/vec/io/io_helper.h | 1 +
be/src/vec/olap/vgeneric_iterators.cpp | 23 +-
be/src/vec/olap/vgeneric_iterators.h | 2 +-
be/src/vec/sink/vdata_stream_sender.cpp | 8 +-
be/src/vec/sink/vtablet_sink.cpp | 18 +-
be/src/vec/utils/template_helpers.hpp | 3 -
be/src/vec/utils/util.hpp | 26 +-
be/test/common/status_test.cpp | 2 -
be/test/exec/es_predicate_test.cpp | 1 +
be/test/exec/tablet_info_test.cpp | 39 +-
be/test/olap/block_column_predicate_test.cpp | 202 ++++++
.../rowset/segment_v2/binary_dict_page_test.cpp | 2 +
.../rowset/segment_v2/binary_plain_page_test.cpp | 1 +
be/test/olap/schema_change_test.cpp | 306 ++++++---
be/test/plugin/CMakeLists.txt | 6 +-
be/test/tools/benchmark_tool.cpp | 5 +-
be/test/vec/core/column_nullable_test.cpp | 7 +-
be/test/vec/exec/vgeneric_iterators_test.cpp | 118 +++-
be/test/vec/function/function_bitmap_test.cpp | 97 ++-
be/test/vec/function/function_math_test.cpp | 11 +-
be/test/vec/function/function_test_util.h | 1 -
docker/Dockerfile | 2 +-
docs/en/extending-doris/hive-of-doris.md | 2 +-
docs/en/extending-doris/iceberg-of-doris.md | 2 +-
docs/en/extending-doris/odbc-of-doris.md | 2 +-
docs/en/installing/compilation-arm.md | 29 +-
docs/en/installing/install-deploy.md | 14 +-
.../aggregate-functions/group_concat.md | 7 +
.../sql-statements/Data Definition/BACKUP.md | 10 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 7 +-
.../sql-statements/Data Definition/RESTORE.md | 19 +-
.../Data Manipulation/BROKER LOAD.md | 2 +
.../Data Manipulation/ROUTINE LOAD.md | 3 +
.../Data Manipulation/SHOW ROUTINE LOAD.md | 9 +-
.../Data Manipulation/STREAM LOAD.md | 2 +
docs/zh-CN/extending-doris/hive-of-doris.md | 2 +-
docs/zh-CN/extending-doris/iceberg-of-doris.md | 2 +-
docs/zh-CN/extending-doris/odbc-of-doris.md | 2 +-
docs/zh-CN/installing/compilation-arm.md | 30 +-
docs/zh-CN/installing/install-deploy.md | 6 +-
.../aggregate-functions/group_concat.md | 7 +
.../sql-statements/Data Definition/BACKUP.md | 10 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 8 +-
.../sql-statements/Data Definition/RESTORE.md | 20 +-
.../Data Manipulation/BROKER LOAD.md | 1 +
.../Data Manipulation/ROUTINE LOAD.md | 4 +
.../Data Manipulation/SHOW ROUTINE LOAD.md | 7 +-
.../Data Manipulation/STREAM LOAD.md | 2 +
.../org/apache/doris/common/FeMetaVersion.java | 5 +
.../java/org/apache/doris/meta/MetaContext.java | 7 +
fe/fe-core/pom.xml | 10 +-
fe/fe-core/src/main/cup/sql_parser.cup | 12 +-
.../src/main/java/org/apache/doris/PaloFe.java | 2 +-
.../java/org/apache/doris/alter/AlterHandler.java | 11 +-
.../java/org/apache/doris/alter/AlterJobV2.java | 20 +-
.../doris/alter/MaterializedViewHandler.java | 4 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 12 +-
.../apache/doris/alter/SchemaChangeHandler.java | 3 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 47 +-
.../apache/doris/analysis/CreateFunctionStmt.java | 2 +-
.../doris/analysis/CreateRoutineLoadStmt.java | 10 +
.../org/apache/doris/analysis/DateLiteral.java | 28 +-
.../doris/analysis/HashDistributionDesc.java | 4 +-
.../java/org/apache/doris/analysis/InsertStmt.java | 9 +-
.../java/org/apache/doris/analysis/LabelName.java | 9 +-
.../java/org/apache/doris/analysis/LoadStmt.java | 7 +
.../analysis/ModifyTablePropertiesClause.java | 5 +-
.../doris/analysis/RandomDistributionDesc.java | 5 +-
.../apache/doris/analysis/ShowRoutineLoadStmt.java | 21 +-
.../apache/doris/analysis/SinglePartitionDesc.java | 5 +-
.../java/org/apache/doris/analysis/TableRef.java | 13 +-
.../java/org/apache/doris/backup/BackupJob.java | 21 +-
.../org/apache/doris/backup/BackupJobInfo.java | 3 -
.../java/org/apache/doris/backup/BackupMeta.java | 5 -
.../java/org/apache/doris/backup/RestoreJob.java | 42 +-
.../java/org/apache/doris/catalog/Catalog.java | 448 ++++++-------
.../apache/doris/catalog/CatalogRecycleBin.java | 11 +-
.../apache/doris/catalog/ColocateTableIndex.java | 89 ++-
.../main/java/org/apache/doris/catalog/Column.java | 40 +-
.../java/org/apache/doris/catalog/ColumnType.java | 7 +-
.../java/org/apache/doris/catalog/Database.java | 45 +-
.../java/org/apache/doris/catalog/DiskInfo.java | 25 +-
.../org/apache/doris/catalog/DistributionInfo.java | 1 -
.../java/org/apache/doris/catalog/EsTable.java | 125 ++--
.../java/org/apache/doris/catalog/FsBroker.java | 11 +-
.../java/org/apache/doris/catalog/HiveTable.java | 2 +-
.../org/apache/doris/catalog/IcebergTable.java | 159 ++++-
.../java/org/apache/doris/catalog/MetaObject.java | 10 +-
.../java/org/apache/doris/catalog/MysqlTable.java | 40 +-
.../java/org/apache/doris/catalog/OlapTable.java | 110 +---
.../java/org/apache/doris/catalog/Partition.java | 113 +---
.../org/apache/doris/catalog/PartitionInfo.java | 7 +-
.../apache/doris/catalog/RangePartitionInfo.java | 13 +-
.../java/org/apache/doris/catalog/Replica.java | 120 ++--
.../main/java/org/apache/doris/catalog/Table.java | 13 +-
.../main/java/org/apache/doris/catalog/Tablet.java | 25 +-
.../apache/doris/catalog/TabletInvertedIndex.java | 27 +-
.../org/apache/doris/catalog/TabletStatMgr.java | 3 +-
.../org/apache/doris/catalog/TempPartitions.java | 11 +-
.../java/org/apache/doris/clone/TabletChecker.java | 1 -
.../org/apache/doris/clone/TabletSchedCtx.java | 39 +-
.../org/apache/doris/clone/TabletScheduler.java | 6 +-
.../org/apache/doris/common/CaseSensibility.java | 3 +-
.../apache/doris/common/proc/LoadJobProcNode.java | 2 +-
.../doris/common/proc/PartitionsProcDir.java | 3 +-
.../apache/doris/common/proc/ReplicasProcNode.java | 8 +-
.../doris/common/proc/RoutineLoadsProcDir.java | 2 +-
.../apache/doris/common/proc/StatisticProcDir.java | 2 +-
.../apache/doris/common/proc/TabletsProcDir.java | 11 +-
.../doris/common/proc/TransPartitionProcNode.java | 1 -
.../doris/common/profile/PlanTreeBuilder.java | 7 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 26 +-
.../java/org/apache/doris/common/util/Util.java | 4 -
.../doris/consistency/CheckConsistencyJob.java | 11 +-
.../doris/consistency/ConsistencyChecker.java | 10 +-
.../org/apache/doris/http/rest/RowCountAction.java | 3 +-
.../doris/http/rest/TableQueryPlanAction.java | 9 +-
.../apache/doris/httpv2/rest/RowCountAction.java | 3 +-
.../doris/httpv2/rest/TableQueryPlanAction.java | 9 +-
.../org/apache/doris/load/BrokerFileGroup.java | 28 +-
.../java/org/apache/doris/load/DeleteHandler.java | 4 +-
.../java/org/apache/doris/load/DeleteInfo.java | 48 +-
.../main/java/org/apache/doris/load/DppConfig.java | 19 +-
.../main/java/org/apache/doris/load/ExportJob.java | 40 +-
.../org/apache/doris/load/HadoopEtlJobInfo.java | 11 +-
.../src/main/java/org/apache/doris/load/Load.java | 29 +-
.../java/org/apache/doris/load/LoadChecker.java | 4 +-
.../main/java/org/apache/doris/load/LoadJob.java | 96 ++-
.../org/apache/doris/load/PartitionLoadInfo.java | 25 +-
.../main/java/org/apache/doris/load/Source.java | 8 +-
.../java/org/apache/doris/load/TabletLoadInfo.java | 6 +-
.../apache/doris/load/loadv2/BrokerLoadJob.java | 2 +-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 41 +-
.../java/org/apache/doris/load/loadv2/LoadJob.java | 63 +-
.../apache/doris/load/loadv2/LoadLoadingTask.java | 4 +-
.../doris/load/loadv2/LoadingTaskPlanner.java | 2 +-
.../org/apache/doris/load/loadv2/SparkLoadJob.java | 5 +-
.../load/routineload/KafkaRoutineLoadJob.java | 15 +-
.../doris/load/routineload/RoutineLoadJob.java | 64 +-
.../doris/load/routineload/RoutineLoadManager.java | 10 +-
.../apache/doris/load/update/UpdatePlanner.java | 2 +-
.../java/org/apache/doris/master/MasterImpl.java | 18 +-
.../org/apache/doris/master/ReportHandler.java | 84 +--
.../apache/doris/mysql/MysqlHandshakePacket.java | 4 +-
.../org/apache/doris/mysql/privilege/PaloAuth.java | 4 +-
.../org/apache/doris/mysql/privilege/PaloRole.java | 14 +-
.../apache/doris/mysql/privilege/PrivEntry.java | 8 +-
.../apache/doris/mysql/privilege/UserProperty.java | 80 +--
.../apache/doris/mysql/privilege/WhiteList.java | 72 +--
.../apache/doris/persist/ColocatePersistInfo.java | 20 +-
.../apache/doris/persist/ConsistencyCheckInfo.java | 9 +-
.../org/apache/doris/persist/CreateTableInfo.java | 11 +-
.../org/apache/doris/persist/DatabaseInfo.java | 16 +-
.../java/org/apache/doris/persist/DropDbInfo.java | 12 +-
.../java/org/apache/doris/persist/DropInfo.java | 6 +-
.../apache/doris/persist/DropPartitionInfo.java | 12 +-
.../apache/doris/persist/ModifyPartitionInfo.java | 4 +-
.../apache/doris/persist/PartitionPersistInfo.java | 16 +-
.../java/org/apache/doris/persist/PrivInfo.java | 8 +-
.../apache/doris/persist/ReplicaPersistInfo.java | 122 ++--
.../apache/doris/persist/TablePropertyInfo.java | 14 +-
.../apache/doris/persist/TruncateTableInfo.java | 12 +-
.../org/apache/doris/planner/BrokerScanNode.java | 28 +-
.../apache/doris/planner/DistributedPlanner.java | 2 +-
.../org/apache/doris/planner/HiveScanNode.java | 2 +-
.../org/apache/doris/planner/IcebergScanNode.java | 109 ++++
.../org/apache/doris/planner/OlapScanNode.java | 20 +-
.../org/apache/doris/planner/OlapTableSink.java | 20 +-
.../org/apache/doris/planner/PlanFragment.java | 2 +-
.../apache/doris/planner/SingleNodePlanner.java | 4 +
.../apache/doris/planner/StreamLoadPlanner.java | 3 +-
.../java/org/apache/doris/qe/AuditLogBuilder.java | 5 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 10 +-
.../java/org/apache/doris/qe/SessionVariable.java | 55 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 9 +-
.../main/java/org/apache/doris/qe/VariableMgr.java | 7 +-
.../org/apache/doris/rpc/BackendServiceClient.java | 2 +-
.../main/java/org/apache/doris/system/Backend.java | 59 +-
.../java/org/apache/doris/system/Frontend.java | 8 +-
.../org/apache/doris/system/SystemInfoService.java | 5 -
.../java/org/apache/doris/task/AgentTaskQueue.java | 7 +-
.../org/apache/doris/task/AlterReplicaTask.java | 13 +-
.../apache/doris/task/CheckConsistencyTask.java | 10 +-
.../main/java/org/apache/doris/task/CloneTask.java | 11 +-
.../org/apache/doris/task/CreateReplicaTask.java | 8 +-
.../org/apache/doris/task/ExportPendingTask.java | 1 -
.../java/org/apache/doris/task/LoadTaskInfo.java | 1 +
.../main/java/org/apache/doris/task/PushTask.java | 16 +-
.../java/org/apache/doris/task/SnapshotTask.java | 9 +-
.../java/org/apache/doris/task/StreamLoadTask.java | 9 +
.../doris/transaction/DatabaseTransactionMgr.java | 50 +-
.../doris/transaction/PartitionCommitInfo.java | 27 +-
.../doris/transaction/PublishVersionDaemon.java | 8 +-
.../apache/doris/transaction/TransactionState.java | 44 +-
.../org/apache/doris/alter/RollupJobV2Test.java | 62 +-
.../apache/doris/alter/SchemaChangeJobV2Test.java | 8 +-
.../apache/doris/analysis/CreateTableStmtTest.java | 15 +-
.../org/apache/doris/backup/BackupHandlerTest.java | 4 +-
.../org/apache/doris/backup/BackupJobTest.java | 3 +-
.../java/org/apache/doris/bdb/BDBToolTest.java | 3 +-
.../org/apache/doris/catalog/CatalogTestUtil.java | 39 +-
.../apache/doris/catalog/CreateTableLikeTest.java | 7 +-
.../java/org/apache/doris/catalog/FakeCatalog.java | 5 -
.../doris/catalog/OdbcCatalogResourceTest.java | 2 +-
.../org/apache/doris/catalog/OlapTableTest.java | 2 +-
.../java/org/apache/doris/catalog/ReplicaTest.java | 86 +--
.../java/org/apache/doris/catalog/TabletTest.java | 18 +-
.../apache/doris/catalog/TempPartitionTest.java | 2 +-
.../java/org/apache/doris/clone/RebalanceTest.java | 2 +-
.../org/apache/doris/clone/RebalancerTestUtil.java | 2 +-
.../org/apache/doris/common/util/UnitTestUtil.java | 8 +-
.../doris/external/elasticsearch/EsTestCase.java | 4 +-
.../org/apache/doris/http/DorisHttpTestCase.java | 18 +-
.../doris/http/TableQueryPlanActionTest.java | 1 -
.../org/apache/doris/load/LoadCheckerTest.java | 10 +-
.../java/org/apache/doris/load/LoadJobTest.java | 6 +-
.../apache/doris/load/PartitionLoadInfoTest.java | 2 -
.../doris/load/loadv2/BrokerLoadJobTest.java | 2 +-
.../load/routineload/RoutineLoadManagerTest.java | 22 +-
.../org/apache/doris/persist/DropDbInfoTest.java | 2 +-
.../org/apache/doris/persist/DropInfoTest.java | 2 +-
.../doris/persist/DropPartitionInfoTest.java | 2 +-
.../org/apache/doris/persist/FsBrokerTest.java | 2 +-
.../doris/persist/ReplicaPersistInfoTest.java | 5 +-
.../apache/doris/planner/OlapTableSinkTest.java | 8 +-
.../org/apache/doris/planner/QueryPlanTest.java | 86 ++-
.../org/apache/doris/plugin/PluginMgrTest.java | 3 +-
.../org/apache/doris/qe/PartitionCacheTest.java | 24 +-
.../java/org/apache/doris/task/AgentTaskTest.java | 7 +-
.../org/apache/doris/task/LoadEtlTaskTest.java | 2 +-
.../org/apache/doris/task/LoadPendingTaskTest.java | 2 +-
.../transaction/DatabaseTransactionMgrTest.java | 5 +-
.../transaction/GlobalTransactionMgrTest.java | 2 +-
.../doris/transaction/TransactionStateTest.java | 2 +-
fe/pom.xml | 16 +-
gensrc/proto/data.proto | 5 +-
gensrc/script/doris_builtins_functions.py | 36 +-
gensrc/thrift/DataSinks.thrift | 1 +
gensrc/thrift/FrontendService.thrift | 1 +
gensrc/thrift/Types.thrift | 4 +-
thirdparty/patches/libhdfs3-master.patch | 24 +-
339 files changed, 4446 insertions(+), 5547 deletions(-)
delete mode 100644 be/src/exprs/new_in_predicate.cpp
delete mode 100644 be/src/exprs/new_in_predicate.h
delete mode 100644 be/src/glibc-compatibility/FastMemcpy.c
delete mode 100644 be/src/glibc-compatibility/FastMemcpy.h
delete mode 100644 be/src/glibc-compatibility/LICENSE_FastMemcpy
create mode 100644 be/src/glibc-compatibility/memcpy/memcpy_aarch64.cpp
create mode 100644 be/src/glibc-compatibility/memcpy/memcpy_x86_64.cpp
delete mode 100644 be/src/glibc-compatibility/memcpy_wrapper.c
create mode 100644 be/src/vec/functions/function_bitmap_min_or_max.h
create mode 100644 be/src/vec/functions/function_bitmap_variadic.cpp
copy be/src/vec/functions/{function_convert_tz.cpp => function_fake.cpp} (83%)
copy be/src/vec/functions/{function_multi_same_args.h => function_fake.h} (72%)
delete mode 100644 be/src/vec/functions/function_math_binary_float64.h
copy be/src/vec/functions/{function_math_unary.h => function_math_unary_to_null_type.h} (57%)
rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/FeMetaVersion.java (96%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/IcebergScanNode.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/02: [feature-wip](array-type) Create table with nested array type. (#8003)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch array-type
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 37becad1ff454d2c1f2a78a4fd95de71dbe1a572
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Wed Feb 16 12:44:30 2022 +0800
[feature-wip](array-type) Create table with nested array type. (#8003)
```
create table array_type_table(k1 INT, k2 Array<Array<int>>) duplicate key (k1)
distributed by hash(k1) buckets 1 properties('replication_num' = '1');
```
---
be/src/olap/tablet_meta.cpp | 9 ++++-----
.../org/apache/doris/analysis/CreateTableStmt.java | 16 ++++------------
.../main/java/org/apache/doris/analysis/TypeDef.java | 10 ++++++++--
.../java/org/apache/doris/catalog/ArrayType.java | 9 +--------
.../main/java/org/apache/doris/catalog/Column.java | 10 ++++------
.../src/main/java/org/apache/doris/catalog/Type.java | 20 +++++++-------------
.../org/apache/doris/catalog/CreateTableTest.java | 13 +++++++++++++
7 files changed, 41 insertions(+), 46 deletions(-)
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 6e6195d..73632a1 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -128,11 +128,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
}
}
}
-
- if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
- ColumnPB* children_column = column->add_children_columns();
- _init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
- }
}
schema->set_next_column_unique_id(next_unique_id);
@@ -210,6 +205,10 @@ void TabletMeta::_init_column_from_tcolumn(uint32_t unique_id, const TColumn& tc
if (tcolumn.__isset.is_bloom_filter_column) {
column->set_is_bf_column(tcolumn.is_bloom_filter_column);
}
+ if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
+ ColumnPB* children_column = column->add_children_columns();
+ _init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
+ }
}
OLAPStatus TabletMeta::create_from_file(const string& file_path) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index ed2b689..5fb937e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,8 +17,11 @@
package org.apache.doris.analysis;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.AggregateType;
-import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Index;
@@ -35,12 +38,6 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -372,11 +369,6 @@ public class CreateTableStmt extends DdlStmt {
columnDef.analyze(engineName.equals("olap"));
if (columnDef.getType().isArrayType()) {
- ArrayType tp = (ArrayType) columnDef.getType();
- if (!tp.getItemType().getPrimitiveType().isIntegerType() &&
- !tp.getItemType().getPrimitiveType().isCharFamily()) {
- throw new AnalysisException("Array column just support INT/VARCHAR sub-type");
- }
if (columnDef.getAggregateType() != null && columnDef.getAggregateType() != AggregateType.NONE) {
throw new AnalysisException("Array column can't support aggregation " + columnDef.getAggregateType());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index 6bce9b5..d81d633 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -92,8 +92,10 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
if (type.isArrayType()) {
- ScalarType itemType = (ScalarType) ((ArrayType) type).getItemType();
- analyzeNestedType(itemType);
+ Type itemType = ((ArrayType) type).getItemType();
+ if (itemType instanceof ScalarType) {
+ analyzeNestedType((ScalarType) itemType);
+ }
}
if (type.isMapType()) {
ScalarType keyType = (ScalarType) ((MapType) type).getKeyType();
@@ -115,6 +117,10 @@ public class TypeDef implements ParseNode {
if (type.isNull()) {
throw new AnalysisException("Unsupported data type: " + type.toSql());
}
+ if (!type.getPrimitiveType().isIntegerType() &&
+ !type.getPrimitiveType().isCharFamily()) {
+ throw new AnalysisException("Array column just support INT/VARCHAR sub-type");
+ }
if (type.getPrimitiveType().isStringType()
&& !type.isAssignedStrLenInColDefinition()) {
type.setLength(1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
index e4c16a1..fff4f05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -83,9 +83,6 @@ public class ArrayType extends Type {
@Override
public String toSql(int depth) {
- if (depth >= MAX_NESTING_DEPTH) {
- return "ARRAY<...>";
- }
return String.format("ARRAY<%s>", itemType.toSql(depth + 1));
}
@@ -125,11 +122,7 @@ public class ArrayType extends Type {
if (!Config.enable_complex_type_support) {
return false;
}
-
- if (itemType.isNull()) {
- return false;
- }
- return true;
+ return !itemType.isNull();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 522d6c0..278055f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
@@ -30,11 +33,6 @@ import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.annotations.SerializedName;
-
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -341,7 +339,6 @@ public class Column implements Writable {
tColumn.setIsAllowNull(this.isAllowNull);
tColumn.setDefaultValue(this.defaultValue);
tColumn.setVisible(visible);
- tColumn.setChildrenColumn(new ArrayList<>());
toChildrenThrift(this, tColumn);
// ATTN:
@@ -370,6 +367,7 @@ public class Column implements Writable {
childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
childrenTColumn.setColumnType(childrenTColumnType);
+ tColumn.setChildrenColumn(new ArrayList<>());
tColumn.children_column.add(childrenTColumn);
toChildrenThrift(children, childrenTColumn);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index efee561..8d51a1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -46,9 +46,10 @@ import java.util.List;
public abstract class Type {
private static final Logger LOG = LogManager.getLogger(Type.class);
- // Maximum nesting depth of a type. This limit was determined experimentally byorg.apache.doris.rewrite.FoldConstantsRule.apply
- // generating and scanning deeply nested Parquet and Avro files. In those experiments,
- // we exceeded the stack space in the scanner (which uses recursion for dealing with
+ // Maximum nesting depth of a type. This limit was determined experimentally by
+ // org.apache.doris.rewrite.FoldConstantsRule.apply generating and scanning
+ // deeply nested Parquet and Avro files. In those experiments, we exceeded
+ // the stack space in the scanner (which uses recursion for dealing with
// nested types) at a nesting depth between 200 and 300 (200 worked, 300 crashed).
public static int MAX_NESTING_DEPTH = 2;
@@ -469,20 +470,13 @@ public abstract class Type {
}
}
} else if (isArrayType()) {
- ArrayType arrayType = (ArrayType) this;
- if (arrayType.getItemType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return false;
} else if (isMultiRowType()) {
MultiRowType multiRowType = (MultiRowType) this;
- if (multiRowType.getItemType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return multiRowType.getItemType().exceedsMaxNestingDepth(d + 1);
} else if (isMapType()) {
MapType mapType = (MapType) this;
- if (mapType.getValueType().exceedsMaxNestingDepth(d + 1)) {
- return true;
- }
+ return mapType.getValueType().exceedsMaxNestingDepth(d + 1);
} else {
Preconditions.checkState(isScalarType());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 24b3c89..67686a7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -519,4 +519,17 @@ public class CreateTableTest {
" 'data_sort.sort_type' = 'zorder'," +
" 'data_sort.col_num' = '');"));
}
+
+ @Test
+ public void testCreateTableWithArrayType() throws Exception {
+ Config.enable_complex_type_support = true;
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table1(k1 INT, k2 Array<int>) duplicate key (k1) " +
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
+ });
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table2(k1 INT, k2 Array<Array<int>>) duplicate key (k1) " +
+ "distributed by hash(k1) buckets 1 properties('replication_num' = '1');");
+ });
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/02: [feature-wip][array-type] Support ArrayLiteral in SQL. (#8089)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch array-type
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit d256206049df5a74740375a118cc693045656c7d
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Fri Feb 18 10:59:57 2022 +0800
[feature-wip][array-type] Support ArrayLiteral in SQL. (#8089)
Please refer to #8074
---
be/src/olap/column_vector.cpp | 21 ++--
be/src/olap/rowset/segment_v2/column_reader.cpp | 8 +-
be/src/olap/rowset/segment_v2/column_reader.h | 13 +++
be/src/olap/rowset/segment_v2/column_writer.cpp | 10 +-
be/src/olap/rowset/segment_v2/column_writer.h | 5 +-
be/src/runtime/collection_value.cpp | 16 ++-
be/src/runtime/mysql_result_writer.cpp | 6 +-
be/src/runtime/raw_value.cpp | 1 +
be/src/runtime/row_batch.cpp | 10 +-
be/src/runtime/tuple.cpp | 29 +++---
.../segment_v2/column_reader_writer_test.cpp | 24 +++++
fe/fe-core/src/main/cup/sql_parser.cup | 14 +++
.../org/apache/doris/analysis/ArrayLiteral.java | 28 +++--
.../org/apache/doris/analysis/CreateTableStmt.java | 10 +-
.../main/java/org/apache/doris/analysis/Expr.java | 3 +-
.../java/org/apache/doris/catalog/ArrayType.java | 4 +
.../main/java/org/apache/doris/catalog/Column.java | 13 ++-
.../main/java/org/apache/doris/catalog/Type.java | 4 +-
.../org/apache/doris/analysis/ColumnDefTest.java | 18 +++-
.../apache/doris/analysis/InsertArrayStmtTest.java | 114 +++++++++++++++++++++
.../org/apache/doris/utframe/UtFrameUtils.java | 9 ++
21 files changed, 306 insertions(+), 54 deletions(-)
diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp
index 8672339..949c802 100644
--- a/be/src/olap/column_vector.cpp
+++ b/be/src/olap/column_vector.cpp
@@ -210,14 +210,19 @@ void ArrayColumnVectorBatch::prepare_for_read(size_t start_idx, size_t size, boo
DCHECK(start_idx + size <= capacity());
for (size_t i = 0; i < size; ++i) {
if (!is_null_at(start_idx + i)) {
- _data[start_idx + i] = CollectionValue(
- _elements->mutable_cell_ptr(*(_offsets->scalar_cell_ptr(start_idx + i))),
- *(_offsets->scalar_cell_ptr(start_idx + i + 1)) -
- *(_offsets->scalar_cell_ptr(start_idx + i)),
- item_has_null,
- _elements->is_nullable() ? const_cast<bool*>(&_elements->null_signs()[*(
- _offsets->scalar_cell_ptr(start_idx + i))])
- : nullptr);
+ auto next_offset = *(_offsets->scalar_cell_ptr(start_idx + i + 1));
+ auto offset = *(_offsets->scalar_cell_ptr(start_idx + i));
+ uint32_t length = next_offset - offset;
+ if (length == 0) {
+ _data[start_idx + i] = CollectionValue(length);
+ } else {
+ _data[start_idx + i] = CollectionValue(
+ _elements->mutable_cell_ptr(offset),
+ length,
+ item_has_null,
+ _elements->is_nullable() ? const_cast<bool*>(&_elements->null_signs()[offset])
+ : nullptr);
+ }
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index dbd5097..cbb4529 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -127,7 +127,7 @@ Status ColumnReader::init() {
"Bad file $0: invalid column index type $1", _path_desc.filepath, index_meta.type()));
}
}
- if (_ordinal_index_meta == nullptr) {
+ if (!is_empty() && _ordinal_index_meta == nullptr) {
return Status::Corruption(strings::Substitute(
"Bad file $0: missing ordinal index for column $1", _path_desc.filepath, _meta.column_id()));
}
@@ -339,6 +339,10 @@ Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterat
}
Status ColumnReader::new_iterator(ColumnIterator** iterator) {
+ if (is_empty()) {
+ *iterator = new EmptyFileColumnIterator();
+ return Status::OK();
+ }
if (is_scalar_type((FieldType)_meta.type())) {
*iterator = new FileColumnIterator(this);
return Status::OK();
@@ -427,7 +431,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool
// read item
size_t item_size = array_batch->get_item_size(dst->current_offset(), *n);
- if (item_size > 0) {
+ if (item_size >= 0) {
bool item_has_null = false;
ColumnVectorBatch* item_vector_batch = array_batch->elements();
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 2765b75..c1dd109 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -131,6 +131,8 @@ public:
PagePointer get_dict_page_pointer() const { return _meta.dict_page(); }
+ inline bool is_empty() const { return _num_rows == 0; }
+
private:
ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows,
FilePathDesc path_desc);
@@ -306,6 +308,17 @@ private:
std::unique_ptr<StringRef[]> _dict_word_info;
};
+class EmptyFileColumnIterator final : public ColumnIterator {
+public:
+ Status seek_to_first() override { return Status::OK(); }
+ Status seek_to_ordinal(ordinal_t ord) override { return Status::OK(); }
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override {
+ *n = 0;
+ return Status::OK();
+ }
+ ordinal_t get_current_ordinal() const override { return 0; }
+};
+
class ArrayFileColumnIterator final : public ColumnIterator {
public:
explicit ArrayFileColumnIterator(ColumnReader* reader, FileColumnIterator* length_reader,
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index ee26ce3..5f98208 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -524,8 +524,8 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
if (num_written <
1) { // page is full, write first item offset and update current length page's start ordinal
RETURN_IF_ERROR(_length_writer->finish_current_page());
- _current_length_page_first_ordinal += _lengh_sum_in_cur_page;
- _lengh_sum_in_cur_page = 0;
+ _current_length_page_first_ordinal += _length_sum_in_cur_page;
+ _length_sum_in_cur_page = 0;
} else {
// write child item.
if (_item_writer->is_nullable()) {
@@ -539,7 +539,7 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
RETURN_IF_ERROR(_item_writer->append_data(reinterpret_cast<const uint8_t**>(&data),
col_cursor->length()));
}
- _lengh_sum_in_cur_page += col_cursor->length();
+ _length_sum_in_cur_page += col_cursor->length();
}
remaining -= num_written;
col_cursor += num_written;
@@ -579,7 +579,9 @@ Status ArrayColumnWriter::write_ordinal_index() {
if (is_nullable()) {
RETURN_IF_ERROR(_null_writer->write_ordinal_index());
}
- RETURN_IF_ERROR(_item_writer->write_ordinal_index());
+ if (!has_empty_items()) {
+ RETURN_IF_ERROR(_item_writer->write_ordinal_index());
+ }
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index b98f488..26c9731 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -307,6 +307,9 @@ public:
private:
Status put_extra_info_in_page(DataPageFooterPB* header) override;
inline Status write_null_column(size_t num_rows, bool is_null); // 写入num_rows个null标记
+ inline bool has_empty_items() const {
+ return _item_writer->get_next_rowid() == 0;
+ }
private:
std::unique_ptr<ScalarColumnWriter> _length_writer;
@@ -314,7 +317,7 @@ private:
std::unique_ptr<ColumnWriter> _item_writer;
ColumnWriterOptions _opts;
ordinal_t _current_length_page_first_ordinal = 0;
- ordinal_t _lengh_sum_in_cur_page = 0;
+ ordinal_t _length_sum_in_cur_page = 0;
};
} // namespace segment_v2
diff --git a/be/src/runtime/collection_value.cpp b/be/src/runtime/collection_value.cpp
index 9568f9b..9b7ea7d 100644
--- a/be/src/runtime/collection_value.cpp
+++ b/be/src/runtime/collection_value.cpp
@@ -23,6 +23,10 @@
namespace doris {
int sizeof_type(PrimitiveType type) {
switch (type) {
+ case TYPE_TINYINT:
+ return sizeof(int8_t);
+ case TYPE_SMALLINT:
+ return sizeof(int16_t);
case TYPE_INT:
return sizeof(int32_t);
case TYPE_CHAR:
@@ -40,6 +44,8 @@ int sizeof_type(PrimitiveType type) {
Status type_check(PrimitiveType type) {
switch (type) {
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_CHAR:
case TYPE_VARCHAR:
@@ -140,7 +146,7 @@ Status CollectionValue::init_collection(FunctionContext* context, uint32_t size,
}
CollectionValue CollectionValue::from_collection_val(const CollectionVal& val) {
- return CollectionValue(val.data, val.length, val.null_signs);
+ return CollectionValue(val.data, val.length, val.has_null, val.null_signs);
}
Status CollectionValue::set(uint32_t i, PrimitiveType type, const AnyVal* value) {
@@ -160,6 +166,12 @@ Status CollectionValue::set(uint32_t i, PrimitiveType type, const AnyVal* value)
}
switch (type) {
+ case TYPE_TINYINT:
+ *reinterpret_cast<int8_t*>(iter.value()) = reinterpret_cast<const TinyIntVal*>(value)->val;
+ break;
+ case TYPE_SMALLINT:
+ *reinterpret_cast<int16_t*>(iter.value()) = reinterpret_cast<const SmallIntVal*>(value)->val;
+ break;
case TYPE_INT:
*reinterpret_cast<int32_t*>(iter.value()) = reinterpret_cast<const IntVal*>(value)->val;
break;
@@ -214,7 +226,7 @@ void ArrayIterator::value(AnyVal* dest) {
break;
case TYPE_SMALLINT:
- reinterpret_cast<TinyIntVal*>(dest)->val = *reinterpret_cast<const int16_t*>(value());
+ reinterpret_cast<SmallIntVal*>(dest)->val = *reinterpret_cast<const int16_t*>(value());
break;
case TYPE_INT:
diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp
index eaf1bd7..229af18 100644
--- a/be/src/runtime/mysql_result_writer.cpp
+++ b/be/src/runtime/mysql_result_writer.cpp
@@ -185,7 +185,11 @@ int MysqlResultWriter::_add_row_value(int index, const TypeDescriptor& type, voi
buf_ret = _add_row_value(index, children_type, iter.value());
buf_ret = _row_buffer->push_string("'", 1);
} else {
- buf_ret = _add_row_value(index, children_type, iter.value());
+ if (!iter.value()) {
+ buf_ret = _row_buffer->push_string("NULL", 4);
+ } else {
+ buf_ret = _add_row_value(index, children_type, iter.value());
+ }
}
iter.next();
diff --git a/be/src/runtime/raw_value.cpp b/be/src/runtime/raw_value.cpp
index f101233..ba51616 100644
--- a/be/src/runtime/raw_value.cpp
+++ b/be/src/runtime/raw_value.cpp
@@ -324,6 +324,7 @@ void RawValue::write(const void* value, void* dst, const TypeDescriptor& type, M
ArrayIterator src_iter = src->iterator(children_type);
ArrayIterator val_iter = val->iterator(children_type);
+ val->set_has_null(src->has_null());
val->copy_null_signs(src);
while (src_iter.has_next() && val_iter.has_next()) {
diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp
index cdbbebb..73fc3aa 100644
--- a/be/src/runtime/row_batch.cpp
+++ b/be/src/runtime/row_batch.cpp
@@ -179,8 +179,10 @@ RowBatch::RowBatch(const RowDescriptor& row_desc, const PRowBatch& input_batch,
// assgin data and null_sign pointer position in tuple_data
int data_offset = convert_to<int>(array_val->data());
array_val->set_data(tuple_data + data_offset);
- int null_offset = convert_to<int>(array_val->null_signs());
- array_val->set_null_signs(convert_to<bool*>(tuple_data + null_offset));
+ if (array_val->has_null()) {
+ int null_offset = convert_to<int>(array_val->null_signs());
+ array_val->set_null_signs(convert_to<bool*>(tuple_data + null_offset));
+ }
const TypeDescriptor& item_type = slot_collection->type().children.at(0);
if (!item_type.is_string_type()) {
@@ -597,7 +599,9 @@ size_t RowBatch::total_byte_size() const {
// compute data null_signs size
CollectionValue* array_val =
tuple->get_collection_slot(slot_collection->tuple_offset());
- result += array_val->length() * sizeof(bool);
+ if (array_val->has_null()) {
+ result += array_val->length() * sizeof(bool);
+ }
const TypeDescriptor& item_type = slot_collection->type().children.at(0);
result += array_val->length() * item_type.get_slot_size();
diff --git a/be/src/runtime/tuple.cpp b/be/src/runtime/tuple.cpp
index af55261..75ae9ce 100644
--- a/be/src/runtime/tuple.cpp
+++ b/be/src/runtime/tuple.cpp
@@ -99,25 +99,20 @@ void Tuple::deep_copy(Tuple* dst, const TupleDescriptor& desc, MemPool* pool, bo
const TypeDescriptor& item_type = slot_desc->type().children.at(0);
int coll_byte_size = cv->length() * item_type.get_slot_size();
- int nulls_size = cv->length() * sizeof(bool);
+ int nulls_size = cv->has_null() ? cv->length() * sizeof(bool) : 0;
int64_t offset = pool->total_allocated_bytes();
char* coll_data = (char*)(pool->allocate(coll_byte_size + nulls_size));
// copy data and null_signs
- if (nulls_size > 0) {
- cv->set_has_null(true);
- cv->set_null_signs(convert_to<bool*>(coll_data) + coll_byte_size);
- memory_copy(coll_data, cv->null_signs(), nulls_size);
- } else {
- cv->set_has_null(false);
- }
+ memory_copy(convert_to<bool*>(coll_data), cv->null_signs(), nulls_size);
memory_copy(coll_data + nulls_size, cv->data(), coll_byte_size);
// assgin new null_sign and data location
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(offset) : convert_to<bool*>(coll_data));
- cv->set_data(convert_ptrs ? convert_to<char*>(offset + nulls_size)
- : coll_data + nulls_size);
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(offset) : convert_to<bool*>(coll_data));
+ }
+ cv->set_data(convert_ptrs ? convert_to<char*>(offset + nulls_size) : coll_data + nulls_size);
if (!item_type.is_string_type()) {
continue;
@@ -212,7 +207,7 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char** data, int64_t* offset,
const TypeDescriptor& item_type = slot_desc->type().children.at(0);
int coll_byte_size = cv->length() * item_type.get_slot_size();
- int nulls_size = cv->length() * sizeof(bool);
+ int nulls_size = cv->has_null() ? cv->length() * sizeof(bool) : 0;
// copy null_sign
memory_copy(*data, cv->null_signs(), nulls_size);
@@ -220,8 +215,9 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char** data, int64_t* offset,
memory_copy(*data + nulls_size, cv->data(), coll_byte_size);
if (!item_type.is_string_type()) {
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(*offset)
- : convert_to<bool*>(*data));
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(*offset) : convert_to<bool*>(*data));
+ }
cv->set_data(convert_ptrs ? convert_to<char*>(*offset + nulls_size)
: *data + nulls_size);
*data += coll_byte_size + nulls_size;
@@ -250,8 +246,9 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char** data, int64_t* offset,
}
}
// assgin new null_sign and data location
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(base_offset)
- : convert_to<bool*>(base_data));
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(base_offset) : convert_to<bool*>(base_data));
+ }
cv->set_data(convert_ptrs ? convert_to<char*>(base_offset + nulls_size)
: base_data + nulls_size);
}
diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
index 7045708..bdd36a0 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
@@ -821,6 +821,30 @@ TEST_F(ColumnReaderWriterTest, test_v_default_value) {
test_v_read_default_value<OLAP_FIELD_TYPE_DECIMAL>(v_decimal, &decimal);
}
+TEST_F(ColumnReaderWriterTest, test_single_empty_array) {
+ size_t num_array = 1;
+ std::unique_ptr<uint8_t[]> array_is_null(new uint8_t[BitmapSize(num_array)]());
+ CollectionValue array(0);
+ test_array_nullable_data<OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE, BIT_SHUFFLE>(
+ &array, array_is_null.get(), num_array, "test_single_empty_array");
+}
+
+TEST_F(ColumnReaderWriterTest, test_mixed_empty_arrays) {
+ size_t num_array = 3;
+ std::unique_ptr<uint8_t[]> array_is_null(new uint8_t[BitmapSize(num_array)]());
+ std::unique_ptr<CollectionValue[]> collection_values(new CollectionValue[num_array]);
+ int data[] = {1, 2, 3};
+ for (int i = 0; i < num_array; ++ i) {
+ if (i % 2 == 1) {
+ new (&collection_values[i]) CollectionValue(0);
+ } else {
+ new (&collection_values[i]) CollectionValue(&data, 3, false, nullptr);
+ }
+ }
+ test_array_nullable_data<OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, BIT_SHUFFLE>(
+ collection_values.get(), array_is_null.get(), num_array, "test_mixed_empty_arrays");
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index a29946f..dc68d98 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -379,6 +379,7 @@ nonterminal CaseExpr case_expr;
nonterminal ArrayList<CaseWhenClause> case_when_clause_list;
nonterminal FunctionParams function_params;
nonterminal Expr function_call_expr, array_expr;
+nonterminal ArrayLiteral array_literal;
nonterminal StructField struct_field;
nonterminal ArrayList<StructField> struct_field_list;
nonterminal AnalyticWindow opt_window_clause;
@@ -4745,6 +4746,17 @@ function_call_expr ::=
:}
;
+array_literal ::=
+ LBRACKET RBRACKET
+ {:
+ RESULT = new ArrayLiteral();
+ :}
+ | LBRACKET expr_list:list RBRACKET
+ {:
+ RESULT = new ArrayLiteral(list.toArray(new LiteralExpr[0]));
+ :}
+ ;
+
array_expr ::=
KW_ARRAY LPAREN function_params:params RPAREN
{:
@@ -4793,6 +4805,8 @@ non_pred_expr ::=
{: RESULT = l; :}
| array_expr:a
{: RESULT = a; :}
+ | array_literal:a
+ {: RESULT = a; :}
| function_call_expr:e
{: RESULT = e; :}
| KW_DATE STRING_LITERAL:l
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
index 74cb323..8a9ed1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -17,6 +17,14 @@
package org.apache.doris.analysis;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+
+import org.apache.commons.lang.StringUtils;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -24,12 +32,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
-import org.apache.doris.catalog.ArrayType;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.thrift.TExprNode;
-import org.apache.doris.thrift.TExprNodeType;
-
public class ArrayLiteral extends LiteralExpr {
public ArrayLiteral() {
@@ -113,4 +115,18 @@ public class ArrayLiteral extends LiteralExpr {
public Expr clone() {
return new ArrayLiteral(this);
}
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isArrayType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ ArrayLiteral literal = new ArrayLiteral(this);
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ literal.children.set(i, child.uncheckedCastTo(((ArrayType)targetType).getItemType()));
+ }
+ literal.setType(targetType);
+ return literal;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 5fb937e..16e0609 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,10 +17,6 @@
package org.apache.doris.analysis;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
@@ -38,6 +34,12 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index c393329..a4a9b0d 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1253,7 +1253,8 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
public Expr checkTypeCompatibility(Type targetType) throws AnalysisException {
- if (targetType.getPrimitiveType().equals(type.getPrimitiveType())) {
+ if (targetType.getPrimitiveType() != PrimitiveType.ARRAY &&
+ targetType.getPrimitiveType() == type.getPrimitiveType()) {
return this;
}
// bitmap must match exactly
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
index fff4f05..172bb9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -95,6 +95,10 @@ public class ArrayType extends Type {
return otherArrayType.itemType.equals(itemType);
}
+ public static boolean canCastTo(ArrayType type, ArrayType targetType) {
+ return Type.canCastTo(type.getItemType(), targetType.getItemType());
+ }
+
@Override
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 278055f..eb45de6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -17,9 +17,6 @@
package org.apache.doris.catalog;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.annotations.SerializedName;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
@@ -33,6 +30,11 @@ import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -158,6 +160,10 @@ public class Column implements Writable {
public void createChildrenColumn(Type type, Column column) {
if (type.isArrayType()) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
+ // TODO We always set the item type in array nullable.
+ // We may provide an alternative to configure this property of
+ // the item type in array in future.
+ c.setIsAllowNull(true);
column.addChildrenColumn(c);
}
}
@@ -366,6 +372,7 @@ public class Column implements Writable {
childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
tColumn.setChildrenColumn(new ArrayList<>());
tColumn.children_column.add(childrenTColumn);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index 8d51a1f..bcb72d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -384,8 +384,10 @@ public abstract class Type {
public static boolean canCastTo(Type t1, Type t2) {
if (t1.isScalarType() && t2.isScalarType()) {
return ScalarType.canCastTo((ScalarType) t1, (ScalarType) t2);
+ } else if (t1.isArrayType() && t2.isArrayType()) {
+ return ArrayType.canCastTo((ArrayType)t1, (ArrayType)t2);
}
- return false;
+ return t1.isNull();
}
/**
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
index 6aa19f9..adf50b8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
@@ -19,9 +19,13 @@ package org.apache.doris.analysis;
import org.apache.doris.analysis.ColumnDef.DefaultValue;
import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.junit.Assert;
import org.junit.Before;
@@ -39,7 +43,6 @@ public class ColumnDefTest {
stringCol = new TypeDef(ScalarType.createChar(10));
floatCol = new TypeDef(ScalarType.createType(PrimitiveType.FLOAT));
booleanCol = new TypeDef(ScalarType.createType(PrimitiveType.BOOLEAN));
-
}
@Test
@@ -117,5 +120,16 @@ public class ColumnDefTest {
}
}
-
+ @Test
+ public void testArray() throws AnalysisException {
+ Config.enable_complex_type_support = true;
+ TypeDef typeDef = new TypeDef(new ArrayType(Type.INT));
+ ColumnDef columnDef = new ColumnDef("array", typeDef, false, null, true, DefaultValue.NOT_SET, "");
+ Column column = columnDef.toColumn();
+ Assert.assertEquals(1, column.getChildren().size());
+ Column childColumn = column.getChildren().get(0);
+ Assert.assertEquals("item", childColumn.getName());
+ Assert.assertEquals(Type.INT, childColumn.getType());
+ Assert.assertTrue(childColumn.isAllowNull());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java
new file mode 100644
index 0000000..ad0d832
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.ExceptionChecker;
+import org.apache.doris.common.util.SqlParserUtils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TUniqueId;
+import org.apache.doris.utframe.UtFrameUtils;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+
+public class InsertArrayStmtTest {
+ private static final String RUNNING_DIR = UtFrameUtils.generateRandomFeRunningDir(InsertArrayStmtTest.class);
+ private static ConnectContext connectContext;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Config.enable_complex_type_support = true;
+ UtFrameUtils.createDorisCluster(RUNNING_DIR);
+ connectContext = UtFrameUtils.createDefaultCtx();
+ createDatabase("create database test;");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ UtFrameUtils.cleanDorisFeDir(RUNNING_DIR);
+ }
+
+ private static void createDatabase(String sql) throws Exception {
+ CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
+ }
+
+ private static void createTable(String sql) throws Exception {
+ CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
+ }
+
+ private static InsertStmt parseAndAnalyze(String sql) throws Exception {
+ SqlParser parser = new SqlParser(new SqlScanner(
+ new StringReader(sql), connectContext.getSessionVariable().getSqlMode()
+ ));
+ InsertStmt insertStmt = (InsertStmt) SqlParserUtils.getFirstStmt(parser);
+ Analyzer analyzer = new Analyzer(connectContext.getCatalog(), connectContext);
+ insertStmt.analyze(analyzer);
+ return insertStmt;
+ };
+
+ @Test
+ public void testInsertArrayStmt() throws Exception {
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table1 (k1 INT, v1 Array<int>) duplicate key (k1) " +
+ " distributed by hash (k1) buckets 1 properties ('replication_num' = '1');");
+ });
+
+ connectContext.setQueryId(new TUniqueId(1, 0));
+ InsertStmt insertStmt = parseAndAnalyze("insert into test.table1 values (1, [1, 2, 3]);");
+ ArrayList<Expr> row = ((SelectStmt) insertStmt.getQueryStmt()).getValueList().getFirstRow();
+ Assert.assertEquals(2, row.size());
+ Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
+ ArrayLiteral arrayLiteral = (ArrayLiteral) row.get(1);
+ Assert.assertEquals(3, arrayLiteral.getChildren().size());
+ Assert.assertTrue(arrayLiteral.isAnalyzed);
+ for (int i = 1; i <= 3; ++ i) {
+ Expr child = arrayLiteral.getChild(i - 1);
+ Assert.assertTrue(child.isAnalyzed);
+ Assert.assertSame(PrimitiveType.INT, child.getType().getPrimitiveType());
+ Assert.assertTrue(child instanceof IntLiteral);
+ Assert.assertEquals(i, ((IntLiteral) child).getValue());
+ }
+
+ connectContext.setQueryId(new TUniqueId(2, 0));
+ insertStmt = parseAndAnalyze("insert into test.table1 values (1, []);");
+ row = ((SelectStmt) insertStmt.getQueryStmt()).getValueList().getFirstRow();
+ Assert.assertEquals(2, row.size());
+ Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
+ arrayLiteral = (ArrayLiteral) row.get(1);
+ Assert.assertTrue(arrayLiteral.isAnalyzed);
+ Assert.assertTrue(arrayLiteral.getChildren().isEmpty());
+ Assert.assertSame(PrimitiveType.INT, ((ArrayType) arrayLiteral.getType()).getItemType().getPrimitiveType());
+
+ connectContext.setQueryId(new TUniqueId(3, 0));
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "type not match", ()-> {
+ parseAndAnalyze("insert into test.table1 values (1, [[1, 2], [3, 4]]);");
+ });
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index fc20d57..9635130 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -61,6 +61,7 @@ import java.nio.channels.SocketChannel;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
public class UtFrameUtils {
@@ -125,6 +126,14 @@ public class UtFrameUtils {
return statementBases;
}
+ public static String generateRandomFeRunningDir(Class testSuiteClass) {
+ return generateRandomFeRunningDir(testSuiteClass.getSimpleName());
+ }
+
+ public static String generateRandomFeRunningDir(String testSuiteName) {
+ return "fe" + "/mocked/" + testSuiteName + "/" + UUID.randomUUID().toString() + "/";
+ }
+
public static int startFEServer(String runningDir) throws EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException, InterruptedException {
// get DORIS_HOME
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org