You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ch...@apache.org on 2025/01/15 07:37:54 UTC
(iotdb) 04/06: merge master
This is an automated email from the ASF dual-hosted git repository.
chenyz pushed a commit to branch udtf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6aa45462f90d318265ee00b57e5881114cac6908
Merge: ba6de95a58e cfc15dec90a
Author: Chen YZ <ch...@gmail.com>
AuthorDate: Tue Jan 7 16:11:37 2025 +0800
merge master
.github/workflows/cluster-it-1c1d.yml | 1 +
.github/workflows/cluster-it-1c1d1a.yml | 2 +
.github/workflows/cluster-it-1c3d.yml | 2 +
.github/workflows/compile-check.yml | 2 +
.github/workflows/dependency-check.yml | 2 +
.github/workflows/multi-language-client.yml | 2 +
.github/workflows/pipe-it-2cluster.yml | 344 +++++++-
.github/workflows/sonar-codecov.yml | 2 +
.github/workflows/table-cluster-it-1c1d.yml | 4 +-
.github/workflows/table-cluster-it-1c3d.yml | 2 +
.github/workflows/todos-check.yml | 1 +
.github/workflows/unit-test.yml | 2 +
example/client-cpp-example/pom.xml | 4 +
example/client-cpp-example/src/CMakeLists.txt | 3 +
.../src/TableModelSessionExample.cpp | 129 +--
.../java/org/apache/iotdb/TableHttpExample.java | 4 +-
.../java/org/apache/iotdb/TableHttpsExample.java | 4 +-
.../org/apache/iotdb/TableModelSessionExample.java | 14 +-
.../apache/iotdb/TableModelSessionPoolExample.java | 28 +-
example/udf/pom.xml | 4 +
.../apache/iotdb/udf/AggregateFunctionExample.java | 2 +-
.../apache/iotdb/udf/ScalarFunctionExample.java | 2 +-
integration-test/pom.xml | 44 +-
.../org/apache/iotdb/itbase/category/ManualIT.java | 27 +-
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 +-
.../it/autocreate/IoTDBPipeAutoConflictIT.java | 51 ++
.../manual/IoTDBPipeTypeConversionISessionIT.java | 2 +-
.../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +-
.../pipe/it/tablemodel/IoTDBPipeClusterIT.java | 290 +++---
.../IoTDBPipeTypeConversionISessionIT.java | 11 +-
.../it/tablemodel/IoTDBPipeTypeConversionIT.java | 2 +-
.../pipe/it/tablemodel/IoTDBPipeWithLoadIT.java | 58 +-
.../iotdb/pipe/it/tablemodel/TableModelUtils.java | 20 +-
.../it/db/it/IoTDBCaseWhenThenTableIT.java | 6 +-
.../relational/it/db/it/IoTDBDeletionTableIT.java | 968 +++++++++++++++++++--
.../it/db/it/IoTDBDisableDeletionTableIT.java | 120 ---
.../it/db/it/IoTDBExecuteBatchTableIT.java | 26 +-
.../it/db/it/IoTDBFlushQueryTableIT.java | 30 +-
.../it/db/it/IoTDBInsertAlignedValuesTableIT.java | 122 ++-
.../relational/it/db/it/IoTDBInsertTableIT.java | 267 +++---
.../it/db/it/IoTDBMultiDeviceTableIT.java | 90 +-
...va => IoTDBMultiTAGsWithAttributesTableIT.java} | 135 ++-
.../relational/it/db/it/IoTDBRecoverTableIT.java | 16 +-
.../it/db/it/IoTDBRecoverUnclosedTableIT.java | 16 +-
.../relational/it/db/it/IoTDBRestartTableIT.java | 51 +-
.../udf/IoTDBUserDefinedAggregateFunctionIT.java | 2 +-
.../it/udf/IoTDBUserDefinedScalarFunctionIT.java | 4 +-
.../it/query/old/IoTDBDatetimeFormatTableIT.java | 2 +-
.../it/query/old/IoTDBFilterBetweenTableIT.java | 2 +-
.../it/query/old/IoTDBFilterNullTableIT.java | 2 +-
.../it/query/old/IoTDBFilterTableIT.java | 10 +-
.../it/query/old/IoTDBNestedQueryTableIT.java | 6 +-
.../it/query/old/IoTDBSimpleQueryTableIT.java | 39 +-
.../alignbydevice/IoTDBAlignByDeviceTableIT.java | 2 +-
.../IoTDBAlignByDeviceWithTemplateTableIT.java | 2 +-
...oTDBOrderByLimitOffsetAlignByDeviceTableIT.java | 4 +-
.../IoTDBOrderByWithAlignByDeviceTableIT.java | 6 +-
.../IoTDBAlignedOffsetLimitPushDownTableIT.java | 2 +-
.../it/query/old/aligned/TableUtils.java | 2 +-
.../scalar/IoTDBCastFunctionTableIT.java | 6 +-
.../scalar/IoTDBCastFunctionTableSpecialIT.java | 2 +-
.../scalar/IoTDBDiffFunctionTableIT.java | 2 +-
.../scalar/IoTDBReplaceFunctionTableIT.java | 2 +-
.../scalar/IoTDBRoundFunctionTableIT.java | 2 +-
.../scalar/IoTDBScalarFunctionTableIT.java | 62 +-
.../scalar/IoTDBSubStringFunctionTableIT.java | 2 +-
.../it/query/old/orderBy/IoTDBOrderByTableIT.java | 36 +-
.../it/query/old/query/IoTDBArithmeticTableIT.java | 4 +-
.../it/query/old/query/IoTDBFuzzyQueryTableIT.java | 2 +-
.../it/query/old/query/IoTDBInTableIT.java | 4 +-
...oTDBNoSelectExpressionAfterAnalyzedTableIT.java | 2 +-
.../query/old/query/IoTDBNullOperandTableIT.java | 2 +-
.../it/query/old/query/IoTDBPaginationTableIT.java | 4 +-
.../it/query/old/query/IoTDBQueryDemoTableIT.java | 2 +-
.../IoTDBQueryWithComplexValueFilterTableIT.java | 2 +-
.../it/query/old/query/IoTDBResultSetTableIT.java | 6 +-
.../query/IoTDBSelectCompareExpressionTableIT.java | 2 +-
.../it/query/recent/IoTDBFillTableIT.java | 6 +-
.../it/query/recent/IoTDBGapFillTableIT.java | 2 +-
.../it/query/recent/IoTDBNullIdQueryIT.java | 4 +-
.../it/query/recent/IoTDBTableAggregationIT.java | 363 +++++++-
.../recent/subquery/SubqueryDataSetUtils.java | 4 +-
.../it/rest/it/IoTDBRestServiceCaseWhenThenIT.java | 6 +-
.../it/rest/it/IoTDBRestServiceFlushQueryIT.java | 34 +-
.../relational/it/rest/it/IoTDBRestServiceIT.java | 54 +-
.../it/IoTDBRestServiceInsertAlignedValuesIT.java | 67 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 18 +-
.../iotdb/relational/it/schema/IoTDBDeviceIT.java | 14 +-
.../iotdb/relational/it/schema/IoTDBTableIT.java | 56 +-
.../it/session/IoTDBSessionRelationalIT.java | 428 +++++----
.../it/session/IoTDBTableModelSessionIT.java | 8 +-
.../pool/IoTDBInsertTableSessionPoolIT.java | 57 +-
.../session/pool/IoTDBTableModelSessionPoolIT.java | 4 +-
.../udf/api/relational/AggregateFunction.java | 3 +-
.../cli/src/assembly/resources/sbin/start-cli.bat | 2 +-
iotdb-client/client-cpp/src/main/Session.h | 37 +-
iotdb-client/client-cpp/src/main/TableSession.cpp | 2 +-
iotdb-client/client-cpp/src/main/TableSession.h | 2 +-
.../src/test/cpp/sessionRelationalIT.cpp | 26 +-
.../client-py/table_model_session_example.py | 12 +-
.../client-py/table_model_session_pool_example.py | 8 +-
.../tests/integration/test_relational_session.py | 12 +-
.../tests/integration/test_tablemodel_insert.py | 252 +++---
.../apache/iotdb/rpc/TElasticFramedTransport.java | 15 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +
.../iotdb/rpc/TElasticFramedTransportTest.java | 71 ++
.../iotdb/session/SessionCacheLeaderTest.java | 12 +-
.../ainode/iotdb/ainode/model/model_factory.py | 2 -
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../apache/iotdb/confignode/manager/IManager.java | 2 +-
.../iotdb/confignode/manager/load/LoadManager.java | 11 +
.../confignode/manager/load/cache/LoadCache.java | 17 +
.../load/cache/node/DataNodeHeartbeatCache.java | 3 +-
.../iotdb/confignode/manager/node/NodeManager.java | 13 +
.../runtime/PipeRuntimeCoordinator.java | 4 +-
.../runtime/heartbeat/PipeHeartbeat.java | 8 +-
.../persistence/schema/ClusterSchemaInfo.java | 6 +-
.../confignode/persistence/schema/ConfigMTree.java | 2 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 4 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 10 +-
.../persistence/schema/ConfigMTreeTest.java | 8 +-
.../schema/table/AddTableColumnProcedureTest.java | 4 +-
.../schema/table/CreateTableProcedureTest.java | 8 +-
.../consensus/iot/IoTConsensusServerImpl.java | 28 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 23 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 17 +-
.../schemaregion/SchemaExecutionVisitor.java | 5 +-
.../db/exception/VerifyMetadataException.java | 12 +-
.../VerifyMetadataTypeMismatchException.java} | 26 +-
.../agent/runtime/SimpleProgressIndexAssigner.java | 36 +-
.../connector/protocol/opcua/OpcUaNameSpace.java | 4 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 2 +-
.../tablet/parser/TabletInsertionEventParser.java | 8 +-
...ileInsertionEventTableParserTabletIterator.java | 2 +-
.../PipeDataNodeRemainingEventAndTimeOperator.java | 20 +-
.../protocol/legacy/loader/TsFileLoader.java | 1 +
.../protocol/thrift/IoTDBDataNodeReceiver.java | 1 +
.../resource/memory/InsertNodeMemoryEstimator.java | 2 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 4 +-
.../table/v1/handler/RequestValidationHandler.java | 20 +-
.../v1/handler/StatementConstructionHandler.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 12 +-
.../execution/fragment/QueryContext.java | 18 +-
.../execution/function/table/HOPTableFunction.java | 9 +-
.../process/function/PartitionRecognizer.java | 5 -
.../process/function/TableFunctionOperator.java | 9 +-
.../schema/source/TableDeviceFetchSource.java | 2 +-
.../schema/source/TableDeviceQuerySource.java | 2 +-
...ator.java => AbstractAggTableScanOperator.java} | 430 +++++----
.../AbstractDefaultAggTableScanOperator.java | 109 +++
...perator.java => AbstractTableScanOperator.java} | 98 ++-
...lator.java => DefaultAggTableScanOperator.java} | 29 +-
.../relational/LastQueryAggTableScanOperator.java | 424 +++++++++
.../source/relational/TableScanOperator.java | 332 +------
...eAlignedDeviceViewAggregationScanOperator.java} | 33 +-
.../TreeAlignedDeviceViewScanOperator.java} | 32 +-
.../relational/aggregation/AccumulatorFactory.java | 15 +-
.../aggregation/AggregationOperator.java | 1 +
.../relational/aggregation/CountIfAccumulator.java | 101 +++
.../relational/aggregation/LastAccumulator.java | 22 +-
.../relational/aggregation/LastByAccumulator.java | 28 +-
.../aggregation/LastByDescAccumulator.java | 5 +
.../aggregation/LastDescAccumulator.java | 9 +-
.../relational/aggregation/TableAccumulator.java | 4 +
.../relational/aggregation/TableAggregator.java | 16 +
.../UserDefinedAggregateFunctionAccumulator.java | 6 +
.../aggregation/grouped/GroupedAccumulator.java | 8 +
.../aggregation/grouped/GroupedAggregator.java | 4 +
.../grouped/GroupedCountIfAccumulator.java | 79 ++
.../GroupedUserDefinedAggregateAccumulator.java | 6 +
.../grouped/HashAggregationOperator.java | 1 +
.../grouped/StreamingAggregationOperator.java | 1 +
.../db/queryengine/plan/analyze/Analysis.java | 10 +-
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 24 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 35 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 77 +-
.../analyze/load/LoadTsFileTableSchemaCache.java | 7 +-
.../load/LoadTsFileToTreeModelAnalyzer.java | 7 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 32 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +-
.../plan/planner/LogicalPlanBuilder.java | 2 +-
.../plan/planner/OperatorTreeGenerator.java | 4 +-
.../plan/planner/TableOperatorGenerator.java | 459 ++++++++--
.../plan/planner/plan/node/PlanGraphPrinter.java | 24 +-
.../plan/planner/plan/node/PlanNode.java | 2 +
.../plan/planner/plan/node/PlanNodeType.java | 12 +
.../plan/planner/plan/node/PlanVisitor.java | 20 +
.../plan/planner/plan/node/source/SourceNode.java | 1 +
.../plan/planner/plan/node/write/InsertNode.java | 8 +-
.../plan/node/write/RelationalDeleteDataNode.java | 47 +-
.../planner/plan/parameter/SeriesScanOptions.java | 12 +
.../relational/analyzer/StatementAnalyzer.java | 9 +-
.../predicate/ConvertPredicateToFilterVisitor.java | 2 +-
.../schema/CheckSchemaPredicateVisitor.java | 4 +-
.../ConvertSchemaPredicateToFilterVisitor.java | 4 +-
.../relational/metadata/AlignedDeviceEntry.java | 56 ++
.../plan/relational/metadata/DeviceEntry.java | 42 +-
.../metadata/NonAlignedAlignedDeviceEntry.java | 56 ++
.../relational/metadata/TableMetadataImpl.java | 9 +
.../plan/relational/metadata/TableSchema.java | 2 +-
.../relational/metadata/TreeDeviceViewSchema.java} | 32 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 14 +-
.../fetcher/TableHeaderSchemaValidator.java | 24 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 2 +-
.../plan/relational/planner/RelationPlanner.java | 22 +-
.../distribute/TableDistributedPlanGenerator.java | 240 ++++-
.../iterative/rule/PruneTableScanColumns.java | 65 +-
.../planner/node/AggregationTableScanNode.java | 387 ++++----
.../node/AggregationTreeDeviceViewScanNode.java | 205 +++++
.../planner/node/DeviceTableScanNode.java | 197 ++---
.../node/InformationSchemaTableScanNode.java | 105 +--
.../relational/planner/node/TableScanNode.java | 125 ++-
.../node/TreeAlignedDeviceViewScanNode.java | 126 +++
.../planner/node/TreeDeviceViewScanNode.java | 208 +++++
.../node/TreeNonAlignedDeviceViewScanNode.java | 126 +++
.../DataNodeLocationSupplierFactory.java | 40 +-
.../PushAggregationIntoTableScan.java | 4 +
.../PushLimitOffsetIntoTableScan.java | 5 +-
.../optimizations/PushPredicateIntoTableScan.java | 18 +-
.../TransformAggregationToStreamable.java | 4 +-
.../optimizations/TransformSortToStreamSort.java | 5 +-
.../optimizations/UnaliasSymbolReferences.java | 40 +-
.../relational/planner/optimizations/Util.java | 73 +-
.../sql/ast/AbstractQueryDeviceWithCache.java | 2 +-
.../plan/relational/sql/ast/ColumnDefinition.java | 2 +-
.../plan/relational/sql/ast/LoadTsFile.java | 21 +-
.../relational/sql/ast/WrappedInsertStatement.java | 7 +-
.../plan/relational/sql/parser/AstBuilder.java | 16 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 80 +-
.../plan/statement/crud/InsertBaseStatement.java | 2 +-
.../plan/statement/crud/LoadTsFileStatement.java | 70 +-
.../db/service/metrics/CompactionMetrics.java | 24 +
.../db/service/metrics/DataNodeMetricsHelper.java | 82 +-
.../metrics/memory/ThresholdMemoryMetrics.java | 194 +++++
.../db/storageengine/buffer/BloomFilterCache.java | 28 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 27 +-
.../buffer/TimeSeriesMetadataCache.java | 56 +-
.../db/storageengine/dataregion/DataRegion.java | 43 +-
.../CompactionSourceFileDeletedException.java} | 25 +-
.../execute/task/CrossSpaceCompactionTask.java | 2 +
.../execute/task/InnerSpaceCompactionTask.java | 38 +-
.../task/InsertionCrossSpaceCompactionTask.java | 2 +
.../task/RepairUnsortedFileCompactionTask.java | 2 +
.../execute/utils/CompactionPathUtils.java | 15 +-
.../execute/utils/CompactionTableSchema.java | 4 +-
.../compaction/execute/utils/CompactionUtils.java | 46 +
.../executor/fast/SeriesCompactionExecutor.java | 10 +-
.../compaction/io/CompactionTsFileReader.java | 14 +-
.../compaction/io/CompactionTsFileWriter.java | 2 +-
.../schedule/CompactionScheduleContext.java | 21 +-
.../estimator/AbstractCrossSpaceEstimator.java | 7 +-
.../estimator/AbstractInnerSpaceEstimator.java | 8 +-
.../estimator/CompactionEstimateUtils.java | 35 +-
.../FastCompactionInnerCompactionEstimator.java | 16 +-
.../FastCrossSpaceCompactionEstimator.java | 17 +-
.../selector/estimator/MetadataInfo.java | 42 +
.../ReadChunkInnerCompactionEstimator.java | 15 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 12 +-
.../utils/CrossSpaceCompactionCandidate.java | 4 +-
.../selector/utils/TsFileResourceCandidate.java | 76 +-
.../dataregion/memtable/AbstractMemTable.java | 4 +-
.../dataregion/memtable/TsFileProcessor.java | 27 +-
.../dataregion/modification/ModificationFile.java | 31 +-
.../dataregion/tsfile/TsFileResource.java | 70 +-
.../db/storageengine/load/LoadTsFileManager.java | 1 -
.../load/active/ActiveLoadTsFileLoader.java | 1 +
.../load/config/LoadTsFileConfigurator.java | 29 +-
.../LoadConvertedInsertTabletStatement.java | 52 ++
...ertedInsertTabletStatementExceptionVisitor.java | 51 ++
...vertedInsertTabletStatementTSStatusVisitor.java | 65 ++
...leStatementDataTypeConvertExecutionVisitor.java | 143 +++
...eeStatementDataTypeConvertExecutionVisitor.java | 130 +++
.../converter/LoadTsFileDataTypeConverter.java | 114 +++
.../load/metrics/LoadTsFileCostMetricsSet.java | 13 +-
.../rescon/memory/TsFileResourceManager.java | 2 +-
.../db/tools/schema/SRStatementGenerator.java | 4 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 6 -
.../java/org/apache/iotdb/db/utils/MemUtils.java | 2 +-
.../db/utils/datastructure/AlignedTVList.java | 3 +-
.../SchemaRegionSimpleRecoverTest.java | 4 +-
.../db/pipe/consensus/DeletionRecoverTest.java | 3 +-
.../db/pipe/consensus/DeletionResourceTest.java | 9 +-
.../plan/parser/StatementGeneratorTest.java | 11 +-
.../node/write/InsertRowsNodeSerdeTest.java | 22 +-
.../node/write/InsertTabletNodeSerdeTest.java | 16 +-
.../planner/node/write/WritePlanNodeSplitTest.java | 2 +-
.../node/write/RelationalDeleteDataNodeTest.java | 3 +-
.../analyzer/MockTableModelDataPartition.java | 10 +-
.../plan/relational/analyzer/ShowQueriesTest.java | 21 +
.../plan/relational/analyzer/TSBSMetadata.java | 61 +-
.../plan/relational/analyzer/TestMatadata.java | 156 +++-
.../plan/relational/analyzer/TestPlanBuilder.java | 6 +-
.../plan/relational/analyzer/TreeViewTest.java | 198 +++++
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 32 +-
.../relational/planner/assertions/PlanAssert.java | 2 +-
.../planner/assertions/PlanMatchPattern.java | 72 ++
.../plan/relational/sql/ast/InsertTabletTest.java | 4 +-
.../plan/statement/InsertStatementTest.java | 45 +-
.../plan/statement/StatementTestUtils.java | 14 +-
.../storageengine/buffer/BloomFilterCacheTest.java | 29 +-
.../cross/CrossSpaceCompactionSelectorTest.java | 102 +--
.../inner/InnerSequenceCompactionSpeedTest.java | 43 +
.../repair/RepairUnsortedFileCompactionTest.java | 53 ++
.../CompactionTableModelTestFileWriter.java | 2 +-
.../CompactionTableSchemaCollectorTest.java | 44 +-
.../TableModelCompactionWithTTLTest.java | 9 +-
.../compaction/utils/CompactionCheckerUtils.java | 30 +-
.../utils/CompactionTaskMemCostEstimatorTest.java | 48 +
.../modification/ModificationFileTest.java | 242 +++++-
.../dataregion/tsfile/TsFileResourceTest.java | 144 +++
.../dataregion/wal/io/WALFileTest.java | 2 +-
.../file/UnsealedTsFileRecoverPerformerTest.java | 4 +-
.../conf/iotdb-system.properties.template | 11 +-
.../apache/iotdb/commons/client/ThriftClient.java | 17 +-
.../apache/iotdb/commons/cluster/NodeStatus.java | 13 +
.../commons/schema/table/InformationSchema.java | 16 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 10 +-
...entColumnSchema.java => FieldColumnSchema.java} | 18 +-
.../{IdColumnSchema.java => TagColumnSchema.java} | 16 +-
.../schema/table/column/TsTableColumnCategory.java | 24 +-
.../table/column/TsTableColumnSchemaUtil.java | 16 +-
.../service/AbstractThriftServiceThread.java | 7 +-
.../iotdb/commons/service/metric/enums/Metric.java | 3 +
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
.../TableBuiltinAggregationFunction.java | 2 +
.../db/relational/grammar/sql/RelationalSql.g4 | 16 +-
.../src/main/openapi3/iotdb_rest_table_v1.yaml | 2 +-
.../src/main/thrift/confignode.thrift | 4 +-
.../iotdb/library/match/PatternExecutor.java | 2 +-
.../iotdb/library/match/UDAFPatternMatch.java | 62 +-
.../org/apache/iotdb/library/UDAFPatternTest.java | 70 ++
pom.xml | 2 +-
333 files changed, 9562 insertions(+), 4035 deletions(-)
diff --cc example/udf/pom.xml
index 5ec19a501dd,5e801f359b9..3449e7258b7
--- a/example/udf/pom.xml
+++ b/example/udf/pom.xml
@@@ -29,9 -29,9 +29,13 @@@
<artifactId>udf-example</artifactId>
<name>IoTDB: Example: UDF</name>
<dependencies>
++ <dependency>
++ <groupId>org.slf4j</groupId>
++ <artifactId>slf4j-api</artifactId>
++ </dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-server</artifactId>
+ <artifactId>udf-api</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/function/table/HOPTableFunction.java
index 33151d56a1d,00000000000..9dfa044084d
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/function/table/HOPTableFunction.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/function/table/HOPTableFunction.java
@@@ -1,165 -1,0 +1,164 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.function.table;
+
+import org.apache.iotdb.udf.api.exception.UDFException;
+import org.apache.iotdb.udf.api.relational.access.Record;
+import org.apache.iotdb.udf.api.relational.table.TableFunction;
+import org.apache.iotdb.udf.api.relational.table.TableFunctionAnalysis;
+import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider;
+import org.apache.iotdb.udf.api.relational.table.argument.Argument;
+import org.apache.iotdb.udf.api.relational.table.argument.Descriptor;
+import org.apache.iotdb.udf.api.relational.table.argument.DescriptorArgument;
+import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument;
+import org.apache.iotdb.udf.api.relational.table.argument.TableArgument;
+import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor;
+import org.apache.iotdb.udf.api.relational.table.specification.DescriptorParameterSpecification;
+import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification;
+import org.apache.iotdb.udf.api.relational.table.specification.ReturnTypeSpecification;
+import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification;
+import org.apache.iotdb.udf.api.relational.table.specification.TableParameterSpecification;
++import org.apache.iotdb.udf.api.type.Type;
+
+import org.apache.tsfile.block.column.ColumnBuilder;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.iotdb.udf.api.relational.table.specification.ReturnTypeSpecification.GenericTable.GENERIC_TABLE;
+
+public class HOPTableFunction extends TableFunction {
+
+ private static final String DATA_PARAMETER_NAME = "DATA";
+ private static final String TIMECOL_PARAMETER_NAME = "TIMECOL";
+ private static final String SLIDE_PARAMETER_NAME = "SLIDE";
+ private static final String SIZE_PARAMETER_NAME = "SIZE";
+
+ @Override
+ public List<ParameterSpecification> getArgumentsSpecification() {
+ return Arrays.asList(
+ TableParameterSpecification.builder()
+ .name(DATA_PARAMETER_NAME)
+ .passThroughColumns()
+ .keepWhenEmpty()
+ .build(),
+ DescriptorParameterSpecification.builder().name(TIMECOL_PARAMETER_NAME).build(),
+ ScalarParameterSpecification.builder()
+ .name(SLIDE_PARAMETER_NAME)
- .type(org.apache.iotdb.udf.api.type.Type.INT64)
++ .type(Type.INT64)
+ .build(),
+ ScalarParameterSpecification.builder()
+ .name(SIZE_PARAMETER_NAME)
- .type(org.apache.iotdb.udf.api.type.Type.INT64)
++ .type(Type.INT64)
+ .build());
+ }
+
+ @Override
+ public ReturnTypeSpecification getReturnTypeSpecification() {
+ return GENERIC_TABLE;
+ }
+
+ @Override
+ public Optional<Descriptor> getReturnProperColumns() {
+ return Optional.of(
+ Descriptor.descriptor(
+ Arrays.asList("window_start", "window_end"),
- Arrays.asList(
- org.apache.iotdb.udf.api.type.Type.TIMESTAMP,
- org.apache.iotdb.udf.api.type.Type.TIMESTAMP)));
++ Arrays.asList(Type.TIMESTAMP, Type.TIMESTAMP)));
+ }
+
+ @Override
+ public TableFunctionAnalysis analyze(Map<String, Argument> arguments) {
+ TableArgument tableArgument = (TableArgument) arguments.get(DATA_PARAMETER_NAME);
+ DescriptorArgument descriptorArgument =
+ (DescriptorArgument) arguments.get(TIMECOL_PARAMETER_NAME);
+ String expectedFieldName =
+ descriptorArgument
+ .getDescriptor()
+ .orElseThrow(() -> new UDFException("The descriptor of the argument is empty"))
+ .getFields()
+ .get(0)
+ .getName();
+ int requiredIndex = -1;
+ for (int i = 0; i < tableArgument.getFieldTypes().size(); i++) {
+ Optional<String> fieldName = tableArgument.getFieldNames().get(i);
+ if (fieldName.isPresent() && expectedFieldName.equalsIgnoreCase(fieldName.get())) {
+ requiredIndex = i;
+ break;
+ }
+ }
+ if (requiredIndex == -1) {
+ throw new UDFException("The required field is not found in the input table");
+ }
+ return TableFunctionAnalysis.builder()
+ .requiredColumns(DATA_PARAMETER_NAME, Collections.singletonList(requiredIndex))
+ .build();
+ }
+
+ @Override
+ public TableFunctionProcessorProvider getProcessorProvider(Map<String, Argument> arguments) {
+ return new TableFunctionProcessorProvider() {
+ @Override
+ public TableFunctionDataProcessor getDataProcessor() {
+ return new HOPDataProcessor(
+ (Long) ((ScalarArgument) arguments.get(SLIDE_PARAMETER_NAME)).getValue() * 1000,
+ (Long) ((ScalarArgument) arguments.get(SLIDE_PARAMETER_NAME)).getValue() * 1000);
+ }
+ };
+ }
+
+ private static class HOPDataProcessor implements TableFunctionDataProcessor {
+
+ private final long slide;
+ private final long size;
+ private long startTime = Long.MIN_VALUE;
+
+ public HOPDataProcessor(long slide, long size) {
+ this.slide = slide;
+ this.size = size;
+ }
+
+ @Override
+ public void process(Record input, List<ColumnBuilder> columnBuilders) {
+ long curTime = input.getLong(0);
+ if (startTime == Long.MIN_VALUE) {
+ startTime = curTime;
+ }
+ while (curTime - startTime >= size) {
+ startTime += slide;
+ }
+
+ for (int i = 0; i < input.size(); i++) {
+ if (input.isNull(i)) {
+ columnBuilders.get(i + 2).appendNull();
+ } else {
+ columnBuilders.get(i + 2).writeObject(input.getObject(i));
+ }
+ }
+ columnBuilders.get(0).writeLong(startTime);
+ columnBuilders.get(1).writeLong(startTime + size);
+ }
+
+ @Override
+ public void finish(List<ColumnBuilder> columnBuilders) {}
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/PartitionRecognizer.java
index d8272926206,00000000000..f2d5cd83691
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/PartitionRecognizer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/PartitionRecognizer.java
@@@ -1,258 -1,0 +1,253 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.operator.process.function;
+
+import org.apache.iotdb.db.queryengine.execution.operator.process.function.partition.PartitionState;
+import org.apache.iotdb.udf.api.relational.access.Record;
+import org.apache.iotdb.udf.api.type.Type;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.DateUtils;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Objects;
+
+public class PartitionRecognizer {
+
+ private final List<Integer> partitionChannels;
+ private final List<Object> partitionValues;
+ private TsBlock currentTsBlock = null;
+ private boolean noMoreData = false;
+ private int currentIndex = 0;
+ private PartitionState state = PartitionState.INIT_STATE;
+
+ public PartitionRecognizer(List<Integer> partitionChannels) {
+ this.partitionChannels = partitionChannels;
+ this.partitionValues = new ArrayList<>(partitionChannels.size());
+ for (int i = 0; i < partitionChannels.size(); i++) {
+ partitionValues.add(null);
+ }
+ }
+
- public boolean needInput() {
- return PartitionState.NEED_MORE_DATA_STATE.equals(state)
- || PartitionState.INIT_STATE.equals(state);
- }
-
+ // TsBlock is sorted by partition columns already
+ public void addTsBlock(TsBlock tsBlock) {
+ if (noMoreData) {
+ throw new IllegalArgumentException(
+ "The partition handler is finished, cannot add more data.");
+ }
+ currentTsBlock = tsBlock;
+ }
+
+ /** Marks the handler as finished. */
+ public void noMoreData() {
+ noMoreData = true;
+ }
+
+ public PartitionState getState() {
+ updateState();
+ return state;
+ }
+
+ private void updateState() {
+ switch (state.getStateType()) {
+ case INIT:
+ state = handleInitState();
+ break;
+ case NEW_PARTITION:
+ state = handleNewPartitionState();
+ break;
+ case ITERATING:
+ state = handleIteratingState();
+ break;
+ case NEED_MORE_DATA:
+ state = handleNeedMoreDataState();
+ break;
+ case FINISHED:
+ // do nothing
+ return;
+ }
+ if (PartitionState.NEED_MORE_DATA_STATE.equals(state)) {
+ currentIndex = 0;
+ }
+ }
+
+ private PartitionState handleInitState() {
+ if (currentTsBlock == null || currentTsBlock.isEmpty()) {
+ return PartitionState.INIT_STATE;
+ }
+ int endPartitionIndex = findEndPartitionIndex();
+ Iterator<Record> recordIterator = getRecordIterator(currentIndex, endPartitionIndex);
+ currentIndex = endPartitionIndex;
+ return PartitionState.newPartitionState(recordIterator);
+ }
+
+ private PartitionState handleNewPartitionState() {
+ if (currentIndex >= currentTsBlock.getPositionCount()) {
+ return PartitionState.NEED_MORE_DATA_STATE;
+ } else {
+ int endPartitionIndex = findEndPartitionIndex();
+ Iterator<Record> recordIterator = getRecordIterator(currentIndex, endPartitionIndex);
+ currentIndex = endPartitionIndex;
+ return PartitionState.newPartitionState(recordIterator);
+ }
+ }
+
+ private PartitionState handleNeedMoreDataState() {
+ if (noMoreData) {
+ return PartitionState.FINISHED_STATE;
+ } else if (currentTsBlock == null || currentTsBlock.isEmpty()) {
+ return PartitionState.NEED_MORE_DATA_STATE;
+ }
+ int endPartitionIndex = findEndPartitionIndex();
+ if (endPartitionIndex != 0) {
+ Iterator<Record> recordIterator = getRecordIterator(currentIndex, endPartitionIndex);
+ currentIndex = endPartitionIndex;
+ return PartitionState.iteratingState(recordIterator);
+ } else {
+ currentIndex = endPartitionIndex;
+ endPartitionIndex = findEndPartitionIndex();
+ Iterator<Record> recordIterator = getRecordIterator(currentIndex, endPartitionIndex);
+ currentIndex = endPartitionIndex;
+ return PartitionState.newPartitionState(recordIterator);
+ }
+ }
+
+ private PartitionState handleIteratingState() {
+ if (currentIndex >= currentTsBlock.getPositionCount()) {
+ return PartitionState.NEED_MORE_DATA_STATE;
+ } else {
+ int endPartitionIndex = findEndPartitionIndex();
+ Iterator<Record> recordIterator = getRecordIterator(currentIndex, endPartitionIndex);
+ currentIndex = endPartitionIndex;
+ return PartitionState.newPartitionState(recordIterator);
+ }
+ }
+
+ private int findEndPartitionIndex() {
+ int i = currentIndex;
+ while (i < currentTsBlock.getPositionCount()) {
+ for (int j = 0; j < partitionChannels.size(); j++) {
+ if (!Objects.equals(
+ partitionValues.get(j),
+ currentTsBlock.getColumn(partitionChannels.get(j)).getObject(i))) {
+ // update partition values
+ for (int k = 0; k < partitionChannels.size(); k++) {
+ partitionValues.set(k, currentTsBlock.getColumn(partitionChannels.get(k)).getObject(i));
+ }
+ return i;
+ }
+ }
+ i++;
+ }
+ return i;
+ }
+
+ private Iterator<Record> getRecordIterator(int startPartitionIndex, int endPartitionIndex) {
+ return new Iterator<Record>() {
+ private int curIndex = startPartitionIndex;
+ private final int endIndex = endPartitionIndex;
+ private final TsBlock tsBlock = currentTsBlock;
+
+ @Override
+ public boolean hasNext() {
+ return curIndex < endIndex;
+ }
+
+ @Override
+ public Record next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ final int idx = curIndex++;
+ return new Record() {
+ @Override
+ public int getInt(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getInt(idx);
+ }
+
+ @Override
+ public long getLong(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getLong(idx);
+ }
+
+ @Override
+ public float getFloat(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getFloat(idx);
+ }
+
+ @Override
+ public double getDouble(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getDouble(idx);
+ }
+
+ @Override
+ public boolean getBoolean(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getBoolean(idx);
+ }
+
+ @Override
+ public Binary getBinary(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getBinary(idx);
+ }
+
+ @Override
+ public String getString(int columnIndex) {
+ return tsBlock
+ .getColumn(columnIndex)
+ .getBinary(idx)
+ .getStringValue(TSFileConfig.STRING_CHARSET);
+ }
+
+ @Override
+ public LocalDate getLocalDate(int columnIndex) {
+ return DateUtils.parseIntToLocalDate(tsBlock.getColumn(columnIndex).getInt(idx));
+ }
+
+ @Override
+ public Object getObject(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).getObject(idx);
+ }
+
+ @Override
+ public Type getDataType(int columnIndex) {
+ // TODO(UDF)
+ return null;
+ }
+
+ @Override
+ public boolean isNull(int columnIndex) {
+ return tsBlock.getColumn(columnIndex).isNull(idx);
+ }
+
+ @Override
+ public int size() {
+ return tsBlock.getValueColumns().length;
+ }
+ };
+ }
+ };
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/TableFunctionOperator.java
index 4609eca0c77,00000000000..6e70cbd0ebf
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/TableFunctionOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/TableFunctionOperator.java
@@@ -1,192 -1,0 +1,191 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.operator.process.function;
+
+import org.apache.iotdb.db.queryengine.execution.operator.Operator;
+import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
+import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator;
+import org.apache.iotdb.db.queryengine.execution.operator.process.function.partition.PartitionState;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionNode;
+import org.apache.iotdb.udf.api.relational.access.Record;
+import org.apache.iotdb.udf.api.relational.table.TableFunctionProcessorProvider;
+import org.apache.iotdb.udf.api.relational.table.processor.TableFunctionDataProcessor;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE;
+
+// only one input source is supported now
+public class TableFunctionOperator implements ProcessOperator {
+
+ private final OperatorContext operatorContext;
+ private final Operator inputOperator;
+ private final TableFunctionProcessorProvider processorProvider;
+ private final List<TSDataType> outputDataTypes;
+ private final int properChannelCount;
+ private final List<Integer> requiredChannels;
+ private final TableFunctionNode.PassThroughSpecification passThroughSpecifications;
+ private final List<Integer> partitionChannels;
+ private final PartitionRecognizer partitionRecognizer;
+ private final TsBlockBuilder blockBuilder;
+
+ private TableFunctionDataProcessor processor;
+ private PartitionState partitionState;
+ private ListenableFuture<?> isBlocked;
+ private boolean finished = false;
+
+ public TableFunctionOperator(
+ OperatorContext operatorContext,
+ TableFunctionProcessorProvider processorProvider,
+ Operator inputOperator,
+ List<TSDataType> outputDataTypes,
+ int properChannelCount,
+ List<Integer> requiredChannels,
+ TableFunctionNode.PassThroughSpecification passThroughSpecifications,
+ List<Integer> partitionChannels) {
+ this.operatorContext = operatorContext;
+ this.inputOperator = inputOperator;
+ this.processorProvider = processorProvider;
+ this.outputDataTypes = outputDataTypes;
+ this.properChannelCount = properChannelCount;
+ this.requiredChannels = requiredChannels;
+ this.passThroughSpecifications = passThroughSpecifications;
+ this.partitionChannels = partitionChannels;
+ this.partitionRecognizer = new PartitionRecognizer(partitionChannels);
+ this.partitionState = PartitionState.INIT_STATE;
+ this.blockBuilder = new TsBlockBuilder(outputDataTypes);
+ }
+
+ @Override
+ public OperatorContext getOperatorContext() {
+ return this.operatorContext;
+ }
+
+ @Override
+ public ListenableFuture<?> isBlocked() {
+ if (isBlocked == null) {
+ isBlocked = tryGetNextTsBlock();
+ }
+ return isBlocked;
+ }
+
+ private ListenableFuture<?> tryGetNextTsBlock() {
+ try {
+ if (inputOperator.isFinished()) {
+ partitionRecognizer.noMoreData();
+ return NOT_BLOCKED;
+ }
+ if (!inputOperator.isBlocked().isDone()) {
+ return inputOperator.isBlocked();
+ }
+ if (inputOperator.hasNextWithTimer()) {
+ partitionRecognizer.addTsBlock(inputOperator.nextWithTimer());
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return NOT_BLOCKED;
+ }
+
+ @Override
+ public TsBlock next() throws Exception {
+ PartitionState.StateType stateType = partitionState.getStateType();
+ if (stateType == PartitionState.StateType.INIT
+ || stateType == PartitionState.StateType.NEED_MORE_DATA) {
+ isBlocked = null;
+ return null;
+ } else {
+ List<ColumnBuilder> columnBuilders = getOutputColumnBuilders();
+ if (stateType == PartitionState.StateType.FINISHED) {
+ if (processor != null) {
+ processor.finish(columnBuilders);
+ }
+ finished = true;
- return buildTsBlock();
++ return buildTsBlock(columnBuilders);
+ }
+ if (stateType == PartitionState.StateType.NEW_PARTITION) {
+ if (processor != null) {
+ processor.finish(columnBuilders);
+ }
+ processor = processorProvider.getDataProcessor();
+ }
+ Iterator<Record> recordIterator = partitionState.getRecordIterator();
+ while (recordIterator.hasNext()) {
+ processor.process(recordIterator.next(), columnBuilders);
+ }
- return buildTsBlock();
++ return buildTsBlock(columnBuilders);
+ }
+ }
+
+ private List<ColumnBuilder> getOutputColumnBuilders() {
+ blockBuilder.reset();
+ return Arrays.asList(blockBuilder.getValueColumnBuilders());
+ }
+
- private TsBlock buildTsBlock() {
- // TODO(UDF): it should be implemented in the other way
- int positionCount = getOutputColumnBuilders().get(0).build().getPositionCount();
++ private TsBlock buildTsBlock(List<ColumnBuilder> columnBuilders) {
++ int positionCount = columnBuilders.get(0).getPositionCount();
+ blockBuilder.declarePositions(positionCount);
+ return blockBuilder.build(new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, positionCount));
+ }
+
+ @Override
+ public boolean hasNext() throws Exception {
+ isBlocked().get(); // wait for the next TsBlock
+ partitionState = partitionRecognizer.getState();
+ return !finished;
+ }
+
+ @Override
+ public void close() throws Exception {}
+
+ @Override
+ public boolean isFinished() throws Exception {
+ return finished;
+ }
+
+ @Override
+ public long calculateMaxPeekMemory() {
+ return 0;
+ }
+
+ @Override
+ public long calculateMaxReturnSize() {
+ return 0;
+ }
+
+ @Override
+ public long calculateRetainedSizeAfterCallingNext() {
+ return 0;
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index 9f8d5f61a6d,d1cf042ef52..c3e92548c5b
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@@ -130,9 -145,9 +146,11 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode;
@@@ -194,9 -215,8 +220,9 @@@ import java.util.stream.Collectors
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.Iterables.getOnlyElement;
import static java.util.Objects.requireNonNull;
- import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.MEASUREMENT;
+ import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.TIME;
import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction.getAggregationTypeByFuncName;
import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
index b950b16e2c2,1811d7f46ae..e7c3c1de1bf
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
@@@ -71,15 -72,10 +72,16 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode;
+import org.apache.iotdb.udf.api.relational.table.argument.Argument;
+import org.apache.iotdb.udf.api.relational.table.argument.DescriptorArgument;
+import org.apache.iotdb.udf.api.relational.table.argument.ScalarArgument;
+import org.apache.iotdb.udf.api.relational.table.argument.TableArgument;
+import com.google.common.base.Joiner;
import org.apache.commons.lang3.Validate;
import org.apache.tsfile.utils.Pair;
import org.eclipse.jetty.util.StringUtil;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index b5b69e29131,348104f95f9..99db38ae149
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@@ -126,9 -127,10 +127,12 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
@@@ -773,11 -775,19 +777,27 @@@ public abstract class PlanVisitor<R, C
return visitDeviceTableScan(node, context);
}
+ public R visitTreeDeviceViewScan(TreeDeviceViewScanNode node, C context) {
+ return visitDeviceTableScan(node, context);
+ }
+
+ public R visitAggregationTreeDeviceViewScan(AggregationTreeDeviceViewScanNode node, C context) {
+ return visitAggregationTableScan(node, context);
+ }
+
+ public R visitTreeAlignedDeviceViewScan(TreeAlignedDeviceViewScanNode node, C context) {
+ return visitTreeDeviceViewScan(node, context);
+ }
+
+ public R visitTreeNonAlignedDeviceViewScan(TreeNonAlignedDeviceViewScanNode node, C context) {
+ return visitTreeDeviceViewScan(node, context);
+ }
++
+ public R visitTableFunction(TableFunctionNode node, C context) {
+ return visitPlan(node, context);
+ }
+
+ public R visitTableFunctionProcessor(TableFunctionProcessorNode node, C context) {
+ return visitPlan(node, context);
+ }
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index ecee5791d52,2f11adf246e..b5661dc5481
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@@ -3107,585 -3066,6 +3104,585 @@@ public class StatementAnalyzer
protected Scope visitShowSubscriptions(ShowSubscriptions node, Optional<Scope> context) {
return createAndAssignScope(node, context);
}
+
+ @Override
+ public Scope visitTableFunctionInvocation(TableFunctionInvocation node, Optional<Scope> scope) {
+ // TableFunctionMetadata tableFunctionMetadata = resolveTableFunction(node)
+ // .orElseThrow(() -> semanticException(FUNCTION_NOT_FOUND, node, "Table function
+ // '%s' not registered", node.getName()));
+ //
+ // ConnectorTableFunction function = tableFunctionMetadata.function();
+ // CatalogHandle catalogHandle = tableFunctionMetadata.catalogHandle();
+
+ TableFunction function = metadata.getTableFunction(node.getName().toString());
+ Node errorLocation = node;
+ if (!node.getArguments().isEmpty()) {
+ errorLocation = node.getArguments().get(0);
+ }
+
+ ArgumentsAnalysis argumentsAnalysis =
+ analyzeArguments(
+ function.getArgumentsSpecification(), node.getArguments(), scope, errorLocation);
+ //
+ // ConnectorTransactionHandle transactionHandle =
+ // transactionManager.getConnectorTransaction(session.getRequiredTransactionId(),
+ // catalogHandle);
+ TableFunctionAnalysis functionAnalysis =
+ function.analyze(argumentsAnalysis.getPassedArguments());
+
+ if (node.getCopartitioning().size() != 0) {
+ // TODO(UDF): support copartition in future
+ throw new SemanticException("Copartitioning is not supported now.");
+ }
+
+ // determine the result relation type per SQL standard ISO/IEC 9075-2, 4.33 SQL-invoked
+ // routines, p. 123, 413, 414
+ ReturnTypeSpecification returnTypeSpecification = function.getReturnTypeSpecification();
+ if (returnTypeSpecification == GENERIC_TABLE
+ || !argumentsAnalysis.getTableArgumentAnalyses().isEmpty()) {
+ analysis.addPolymorphicTableFunction(node);
+ }
+ Optional<Descriptor> properColumns = function.getReturnProperColumns();
+ Descriptor properColumnsDescriptor;
+ if (returnTypeSpecification == ONLY_PASS_THROUGH) {
+ if (properColumns.isPresent()) {
+ // If a table function has ONLY PASS THROUGH returned type, it does not produce any proper
+ // columns,
+ // so the function's analyze() method should not return the proper columns descriptor.
+ throw new SemanticException(
+ "A table function with ONLY_PASS_THROUGH return type must not return proper columns.");
+ }
+ if (function.getArgumentsSpecification().stream()
+ .filter(TableParameterSpecification.class::isInstance)
+ .map(TableParameterSpecification.class::cast)
+ .noneMatch(TableParameterSpecification::isPassThroughColumns)) {
+ // According to SQL standard ISO/IEC 9075-2, 10.4 <routine invocation>, p. 764,
+ // if there is no generic table parameter that specifies PASS THROUGH, then number of
+ // proper columns shall be positive.
+ // For GENERIC_TABLE and DescribedTable returned types, this is enforced by the Descriptor
+ // constructor, which requires positive number of fields.
+ // Here we enforce it for the remaining returned type specification: ONLY_PASS_THROUGH.
+ throw new SemanticException(
+ "A table function with ONLY_PASS_THROUGH return type must have a table argument with pass-through columns.");
+ }
+ properColumnsDescriptor = null;
+ } else if (returnTypeSpecification == GENERIC_TABLE) {
+ properColumnsDescriptor =
+ properColumns.orElseThrow(
+ () ->
+ new SemanticException(
+ String.format(
+ "Cannot determine returned relation type for generic table function %s",
+ node.getName())));
+ } else { // returned type is statically declared at function declaration
+ // TODO(UDF): 如果是DescribedTable,结果在声明的时候就已经知道了。这边或许可以考虑和GENERIC_TABLE合并?
+ properColumnsDescriptor =
+ ((ReturnTypeSpecification.DescribedTable) returnTypeSpecification).getDescriptor();
+ }
+
+ // validate the required input columns
+ // <TableArgumentName, TableColumnIndexes>
+ // TODO(UDF): mock required function here
+ Map<String, List<Integer>> requiredColumns = functionAnalysis.getRequiredColumns();
+ Map<String, TableArgumentAnalysis> tableArgumentsByName =
+ argumentsAnalysis.getTableArgumentAnalyses().stream()
+ .collect(toImmutableMap(TableArgumentAnalysis::getArgumentName, Function.identity()));
+ Set<String> tableArgumentNameSet = ImmutableSet.copyOf(tableArgumentsByName.keySet());
+ requiredColumns.forEach(
+ (name, columns) -> {
+ if (!tableArgumentNameSet.contains(name)) {
+ throw new SemanticException(
+ String.format(
+ "Table function %s specifies required columns from table argument %s which cannot be found",
+ node.getName(), name));
+ }
+ // TODO(UDF): 定义的时候需要确保 columns 不是empty且不为负数,不在这里二次确认了
+ // if (columns.isEmpty()) {
+ // throw new TrinoException(
+ // FUNCTION_IMPLEMENTATION_ERROR,
+ // format(
+ // "Table function %s specifies empty list of required columns from
+ // table argument %s",
+ // node.getName(), name));
+ // }
+ // the scope is recorded, because table arguments are already analyzed
+ Scope inputScope = analysis.getScope(tableArgumentsByName.get(name).getRelation());
+ columns.stream()
+ .filter(column -> column >= inputScope.getRelationType().getVisibleFieldCount())
+ .findFirst()
+ .ifPresent(
+ column -> {
+ throw new SemanticException(
+ String.format(
+ "Index %s of required column from table argument %s is out of bounds for table with %s columns",
+ column, name, inputScope.getRelationType().getAllFieldCount()));
+ });
+ // record the required columns for access control
+ columns.stream()
+ .map(inputScope.getRelationType()::getFieldByIndex)
+ .forEach(this::recordColumnAccess);
+ });
+ // TODO(UDF):我觉得requiredInputs应该是一定等于tableArgumentNameSet的。可以考虑让用户在定义的时候,一定要为每个table
+ // argument指定required columns?这个应该在注册的时候做检查。
+ Set<String> requiredInputs = ImmutableSet.copyOf(requiredColumns.keySet());
+ tableArgumentNameSet.stream()
+ .filter(input -> !requiredInputs.contains(input))
+ .findFirst()
+ .ifPresent(
+ input -> {
+ throw new SemanticException(
+ String.format(
+ "Table function %s does not specify required input columns from table argument %s",
+ node.getName(), input));
+ });
+
+ // The result relation type of a table function consists of:
+ // 1. columns created by the table function, called the proper columns.
+ // 2. passed columns from input tables:
+ // - for tables with the "pass through columns" option, these are all columns of the table,
+ // - for tables without the "pass through columns" option, these are the partitioning columns
+ // of the table, if any.
+ ImmutableList.Builder<Field> fields = ImmutableList.builder();
+
+ // proper columns first
+ if (properColumnsDescriptor != null) {
+ properColumnsDescriptor.getFields().stream()
+ // per spec, field names are mandatory. We support anonymous fields.
+ .map(
+ field ->
+ Field.newUnqualified(
+ field.getName(),
+ UDFDataTypeTransformer.transformUDFDataTypeToReadType(
+ field
+ .getType()
+ .orElseThrow(
+ () -> new SemanticException("Field type is not present."))),
- TsTableColumnCategory.MEASUREMENT))
++ TsTableColumnCategory.FIELD))
+ .forEach(fields::add);
+ }
+
+ // next, columns derived from table arguments, in order of argument declarations
+ List<String> tableArgumentNames =
+ function.getArgumentsSpecification().stream()
+ .filter(TableParameterSpecification.class::isInstance)
+ .map(ParameterSpecification::getName)
+ .collect(toImmutableList());
+
+ // table arguments in order of argument declarations
+ ImmutableList.Builder<TableArgumentAnalysis> orderedTableArguments = ImmutableList.builder();
+
+ for (String name : tableArgumentNames) {
+ TableArgumentAnalysis argument = tableArgumentsByName.get(name);
+ // analyze arguments will make sure that all table arguments are present
+ checkArgument(argument != null, "Missing table argument: %s", name);
+ orderedTableArguments.add(argument);
+ Scope argumentScope = analysis.getScope(argument.getRelation());
+ if (argument.isPassThroughColumns()) {
+ argumentScope.getRelationType().getAllFields().forEach(fields::add);
+ } else if (argument.getPartitionBy().isPresent()) {
+ argument.getPartitionBy().get().stream()
+ .map(expression -> validateAndGetInputField(expression, argumentScope))
+ .forEach(fields::add);
+ }
+ }
+
+ analysis.setTableFunctionAnalysis(
+ node,
+ new TableFunctionInvocationAnalysis(
+ node.getName().toString(),
+ argumentsAnalysis.getPassedArguments(),
+ orderedTableArguments.build(),
+ requiredColumns,
+ properColumnsDescriptor == null ? 0 : properColumnsDescriptor.getFields().size()));
+ // TODO(UDF): 这里也许需要多一个TableFunctionConfiguration
+
+ return createAndAssignScope(node, scope, fields.build());
+ }
+
+ private ArgumentsAnalysis analyzeArguments(
+ List<ParameterSpecification> parameterSpecifications,
+ List<TableFunctionArgument> arguments,
+ Optional<Scope> scope,
+ Node errorLocation) {
+ if (parameterSpecifications.size() < arguments.size()) {
+ throw new SemanticException(
+ String.format(
+ "Too many arguments. Expected at most %s arguments, got %s arguments",
+ parameterSpecifications.size(), arguments.size()));
+ }
+
+ if (parameterSpecifications.isEmpty()) {
+ return new ArgumentsAnalysis(ImmutableMap.of(), ImmutableList.of());
+ }
+
+ boolean argumentsPassedByName =
+ !arguments.isEmpty()
+ && arguments.stream().allMatch(argument -> argument.getName().isPresent());
+ boolean argumentsPassedByPosition =
+ arguments.stream().allMatch(argument -> argument.getName().isPresent());
+ if (!argumentsPassedByName && !argumentsPassedByPosition) {
+ throw new SemanticException(
+ "All arguments must be passed by name or all must be passed positionally");
+ }
+
+ ImmutableMap.Builder<String, Argument> passedArguments = ImmutableMap.builder();
+ ImmutableList.Builder<TableArgumentAnalysis> tableArgumentAnalyses = ImmutableList.builder();
+ // TODO(UDF): args passed positionally - can one only pass some prefix of args?
+ // TODO(UDF): check at most one table argument passed
+ if (argumentsPassedByName) {
+ Map<String, ParameterSpecification> argumentSpecificationsByName = new HashMap<>();
+ for (ParameterSpecification parameterSpecification : parameterSpecifications) {
+ if (argumentSpecificationsByName.put(
+ parameterSpecification.getName(), parameterSpecification)
+ != null) {
+ // this should never happen, because the argument names are validated at function
+ // registration time
+ throw new IllegalStateException(
+ "Duplicate argument specification for name: " + parameterSpecification.getName());
+ }
+ }
+ Set<String> uniqueArgumentNames = new HashSet<>();
+ for (TableFunctionArgument argument : arguments) {
+ // it has been checked that all arguments have different names
+ String argumentName = argument.getName().get().getCanonicalValue();
+ if (!uniqueArgumentNames.add(argumentName)) {
+ throw new SemanticException(String.format("Duplicate argument name: %s", argumentName));
+ }
+ ParameterSpecification parameterSpecification =
+ argumentSpecificationsByName.remove(argumentName);
+ if (parameterSpecification == null) {
+ throw new SemanticException(
+ String.format("Unexpected argument name: %s", argumentName));
+ }
+ ArgumentAnalysis argumentAnalysis =
+ analyzeArgument(parameterSpecification, argument, scope);
+ passedArguments.put(argumentName, argumentAnalysis.getArgument());
+ argumentAnalysis.getTableArgumentAnalysis().ifPresent(tableArgumentAnalyses::add);
+ }
+ // apply defaults for not specified arguments
+ for (Map.Entry<String, ParameterSpecification> entry :
+ argumentSpecificationsByName.entrySet()) {
+ ParameterSpecification parameterSpecification = entry.getValue();
+ passedArguments.put(
+ parameterSpecification.getName(),
+ analyzeDefault(parameterSpecification, errorLocation));
+ }
+ } else {
+ for (int i = 0; i < arguments.size(); i++) {
+ TableFunctionArgument argument = arguments.get(i);
+ ParameterSpecification parameterSpecification = parameterSpecifications.get(i);
+ ArgumentAnalysis argumentAnalysis =
+ analyzeArgument(parameterSpecification, argument, scope);
+ passedArguments.put(parameterSpecification.getName(), argumentAnalysis.getArgument());
+ argumentAnalysis.getTableArgumentAnalysis().ifPresent(tableArgumentAnalyses::add);
+ }
+ // apply defaults for not specified arguments
+ for (int i = arguments.size(); i < parameterSpecifications.size(); i++) {
+ ParameterSpecification parameterSpecification = parameterSpecifications.get(i);
+ passedArguments.put(
+ parameterSpecification.getName(),
+ analyzeDefault(parameterSpecification, errorLocation));
+ }
+ }
+
+ return new ArgumentsAnalysis(passedArguments.buildOrThrow(), tableArgumentAnalyses.build());
+ }
+
+ private ArgumentAnalysis analyzeArgument(
+ ParameterSpecification parameterSpecification,
+ TableFunctionArgument argument,
+ Optional<Scope> scope) {
+ String actualType;
+ if (argument.getValue() instanceof TableFunctionTableArgument) {
+ actualType = "table";
+ } else if (argument.getValue() instanceof TableFunctionDescriptorArgument) {
+ actualType = "descriptor";
+ } else if (argument.getValue() instanceof Expression) {
+ actualType = "expression";
+ } else {
+ throw new SemanticException(
+ String.format(
+ "Unexpected table function argument type: %s",
+ argument.getClass().getSimpleName()));
+ }
+
+ if (parameterSpecification instanceof TableParameterSpecification) {
+ if (!(argument.getValue() instanceof TableFunctionTableArgument)) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Expected table argument, got %s",
+ parameterSpecification.getName(), actualType));
+ }
+ return analyzeTableArgument(
+ (TableFunctionTableArgument) argument.getValue(),
+ (TableParameterSpecification) parameterSpecification,
+ scope);
+ } else if (parameterSpecification instanceof DescriptorParameterSpecification) {
+ if (!(argument.getValue() instanceof TableFunctionDescriptorArgument)) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Expected descriptor argument, got %s",
+ parameterSpecification.getName(), actualType));
+ }
+ return analyzeDescriptorArgument((TableFunctionDescriptorArgument) argument.getValue());
+ } else if (parameterSpecification instanceof ScalarParameterSpecification) {
+ if (!(argument.getValue() instanceof Expression)) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Expected scalar argument, got %s",
+ parameterSpecification.getName(), actualType));
+ }
+ // TODO(UDF):
+ // 如果functionName是descriptor的话,会被解析成FunctionCall还是TableFunctionDescriptorArgument?
+ // // 'descriptor' as a function name is not allowed in this context
+ // if (expression instanceof FunctionCall && ((FunctionCall)
+ // expression).getName().hasSuffix(QualifiedName.of("descriptor"))) { // function name is
+ // always compared case-insensitive
+ // throw semanticException(INVALID_FUNCTION_ARGUMENT, argument, "'descriptor'
+ // function is not allowed as a table function argument");
+ // }
+ return analyzeScalarArgument(
+ (Expression) argument.getValue(),
+ (ScalarParameterSpecification) parameterSpecification);
+ } else {
+ throw new IllegalStateException(
+ "Unexpected argument specification: "
+ + parameterSpecification.getClass().getSimpleName());
+ }
+ }
+
+ private ArgumentAnalysis analyzeTableArgument(
+ TableFunctionTableArgument tableArgument,
+ TableParameterSpecification argumentSpecification,
+ Optional<Scope> scope) {
+ List<Optional<String>> fieldNames;
+ List<org.apache.iotdb.udf.api.type.Type> fieldTypes;
+ List<String> partitionBy = Collections.emptyList();
+ List<String> orderBy = Collections.emptyList();
+
+ TableArgumentAnalysis.Builder analysisBuilder = TableArgumentAnalysis.builder();
+ analysisBuilder.withArgumentName(argumentSpecification.getName());
+
+ // process the relation
+ Relation relation = tableArgument.getTable();
+ analysisBuilder.withRelation(relation);
+ Scope argumentScope = process(relation, scope);
+ QualifiedName relationName = analysis.getRelationName(relation);
+ if (relationName != null) {
+ analysisBuilder.withName(relationName);
+ }
+
+ // analyze field
+ Collection<Field> fields = argumentScope.getRelationType().getVisibleFields();
+ fieldNames = fields.stream().map(Field::getName).collect(toImmutableList());
+ fieldTypes =
+ fields.stream()
+ .map(Field::getType)
+ .map(UDFDataTypeTransformer::transformReadTypeToUDFDataType)
+ .collect(toImmutableList());
+
+ // analyze PARTITION BY
+ if (tableArgument.getPartitionBy().isPresent()) {
+ if (argumentSpecification.isRowSemantics()) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Partitioning can not be specified for table argument with row semantics",
+ argumentSpecification.getName()));
+ }
+ List<Expression> partitionByExpression = tableArgument.getPartitionBy().get();
+ analysisBuilder.withPartitionBy(partitionByExpression);
+ partitionByExpression.forEach(
+ partitioningColumn -> {
+ validateAndGetInputField(partitioningColumn, argumentScope);
+ Type type =
+ analyzeExpression(partitioningColumn, argumentScope).getType(partitioningColumn);
+ if (!type.isComparable()) {
+ throw new SemanticException(
+ String.format(
+ "%s is not comparable, and therefore cannot be used in PARTITION BY",
+ type));
+ }
+ });
+ partitionBy =
+ partitionByExpression.stream()
+ .map(
+ expression -> {
+ if (expression instanceof Identifier) {
+ return ((Identifier) expression).getValue();
+ } else if (expression instanceof DereferenceExpression) {
+ return expression.toString();
+ } else {
+ throw new IllegalStateException(
+ "Unexpected partitionBy expression: " + expression);
+ }
+ })
+ .collect(toImmutableList());
+ }
+
+ // analyze ORDER BY
+ if (tableArgument.getOrderBy().isPresent()) {
+ if (argumentSpecification.isRowSemantics()) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Ordering can not be specified for table argument with row semantics",
+ argumentSpecification.getName()));
+ }
+ OrderBy orderByExpression = tableArgument.getOrderBy().get();
+ analysisBuilder.withOrderBy(orderByExpression);
+ orderByExpression.getSortItems().stream()
+ .map(SortItem::getSortKey)
+ .forEach(
+ orderingColumn -> {
+ validateAndGetInputField(orderingColumn, argumentScope);
+ Type type =
+ analyzeExpression(orderingColumn, argumentScope).getType(orderingColumn);
+ if (!type.isOrderable()) {
+ throw new SemanticException(
+ String.format(
+ "%s is not orderable, and therefore cannot be used in ORDER BY", type));
+ }
+ });
+ orderBy =
+ orderByExpression.getSortItems().stream()
+ .map(SortItem::getSortKey)
+ .map(
+ expression -> {
+ if (expression instanceof Identifier) {
+ return ((Identifier) expression).getValue();
+ } else if (expression instanceof DereferenceExpression) {
+ return expression.toString();
+ } else {
+ throw new IllegalStateException(
+ "Unexpected orderBy expression: " + expression);
+ }
+ })
+ .collect(toImmutableList());
+ }
+
+ // analyze the PRUNE/KEEP WHEN EMPTY property
+ boolean pruneWhenEmpty = argumentSpecification.isPruneWhenEmpty();
+ if (tableArgument.getEmptyTableTreatment().isPresent()) {
+ if (argumentSpecification.isRowSemantics()) {
+ throw new SemanticException(
+ String.format(
+ "Invalid argument %s. Empty behavior specified for table argument with row semantics",
+ argumentSpecification.getName()));
+ }
+ pruneWhenEmpty =
+ tableArgument.getEmptyTableTreatment().get().getTreatment()
+ == EmptyTableTreatment.Treatment.PRUNE;
+ }
+ analysisBuilder.withPruneWhenEmpty(pruneWhenEmpty);
+
+ // record remaining properties
+ analysisBuilder.withRowSemantics(argumentSpecification.isRowSemantics());
+ analysisBuilder.withPassThroughColumns(argumentSpecification.isPassThroughColumns());
+
+ return new ArgumentAnalysis(
+ new TableArgument(fieldNames, fieldTypes, partitionBy, orderBy),
+ Optional.of(analysisBuilder.build()));
+ }
+
+ private ArgumentAnalysis analyzeDescriptorArgument(TableFunctionDescriptorArgument argument) {
+ return new ArgumentAnalysis(
+ argument
+ .getDescriptor()
+ .map(
+ descriptor ->
+ new DescriptorArgument(
+ Optional.of(
+ new Descriptor(
+ descriptor.getFields().stream()
+ .map(
+ field ->
+ new Descriptor.Field(
+ field.getName().getCanonicalValue(),
+ field
+ .getType()
+ .map(
+ type -> {
+ try {
+ return UDFDataTypeTransformer
+ .transformReadTypeToUDFDataType(
+ typeManager.getType(
+ toTypeSignature(type)));
+ } catch (TypeNotFoundException e) {
+ throw new SemanticException(
+ String.format(
+ "Unknown type: %s", type));
+ }
+ })))
+ .collect(toImmutableList())))))
+ .orElse(NULL_DESCRIPTOR),
+ Optional.empty());
+ }
+
+ private ArgumentAnalysis analyzeScalarArgument(
+ Expression expression, ScalarParameterSpecification argumentSpecification) {
+ // currently, only constant arguments are supported
+ Object constantValue =
+ IrExpressionInterpreter.evaluateConstantExpression(
+ expression, new PlannerContext(metadata, typeManager), sessionContext);
+ if (!argumentSpecification.getType().checkObjectType(constantValue)) {
+ throw new SemanticException(
+ String.format(
+ "Invalid scalar argument value. Expected type %s, got %s",
+ argumentSpecification.getType(), constantValue.getClass().getSimpleName()));
+ }
+ return new ArgumentAnalysis(
+ new ScalarArgument(argumentSpecification.getType(), constantValue), Optional.empty());
+ }
+
+ private Argument analyzeDefault(
+ ParameterSpecification parameterSpecification, Node errorLocation) {
+ if (parameterSpecification.isRequired()) {
+ throw new SemanticException(
+ String.format("Missing required argument: %s", parameterSpecification.getName()));
+ }
+ checkArgument(
+ !(parameterSpecification instanceof TableParameterSpecification),
+ "Table argument specification cannot have a default value.");
+
+ if (parameterSpecification instanceof DescriptorParameterSpecification) {
+ if (parameterSpecification.getDefaultValue().isPresent()) {
+ return new DescriptorArgument(
+ Optional.of((Descriptor) parameterSpecification.getDefaultValue().get()));
+ } else {
+ return NULL_DESCRIPTOR;
+ }
+ } else if (parameterSpecification instanceof ScalarParameterSpecification) {
+ checkArgument(
+ parameterSpecification.getDefaultValue().isPresent(),
+ String.format(
+ "Missing default value for scalar argument: %s", parameterSpecification.getName()));
+ return new ScalarArgument(
+ ((ScalarParameterSpecification) parameterSpecification).getType(),
+ parameterSpecification.getDefaultValue().get());
+ } else {
+ throw new IllegalStateException(
+ "Unexpected argument specification: "
+ + parameterSpecification.getClass().getSimpleName());
+ }
+ }
+
+ private Field validateAndGetInputField(Expression expression, Scope inputScope) {
+ QualifiedName qualifiedName;
+ if (expression instanceof Identifier) {
+ qualifiedName = QualifiedName.of(ImmutableList.of((Identifier) expression));
+ } else if (expression instanceof DereferenceExpression) {
+ qualifiedName = getQualifiedName((DereferenceExpression) expression);
+ } else {
+ throw new SemanticException(
+ String.format("Expected column reference. Actual: %s", expression));
+ }
+ Optional<ResolvedField> field = inputScope.tryResolveField(expression, qualifiedName);
+ if (!field.isPresent() || !field.get().isLocal()) {
+ throw new SemanticException(
+ String.format("Column %s is not present in the input relation", expression));
+ }
+
+ return field.get().getField();
+ }
}
private static boolean hasScopeAsLocalParent(Scope root, Scope parent) {
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index f96f00327e2,5f08926fe70..97b17ff48a4
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@@ -48,8 -47,8 +49,9 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java
index 0cf4e80adb1,53d6516e2e3..9dbfc5cf82d
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java
@@@ -56,8 -59,10 +59,11 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.AbstractTableDeviceQueryNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java
index 6dbaeecdb2a,dda33ea1c2f..058bad7cb1b
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java
@@@ -47,9 -46,8 +47,10 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode;
+ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral;