You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/09/26 08:33:02 UTC
[iotdb] branch lmh/RefactorAnalyzer updated (5160df0b59 -> 7bed396603)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/RefactorAnalyzer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 5160df0b59 finish refactor analyzer
add de65da1da9 [IOTDB-4395] fix size statistic of disk. (#7342)
add b572da8dd6 fix default config. (#7347)
add 81967ee09f [IOTDB-4453] Fix the overflow of compareTo in TimeRange
add ccf7cc7f95 [IOTDB-4431] Add QuickSort in TVList (#7290)
add 935f530c00 [IOTDB-3656] mpp load with supporting modify MAX_PLAN_NODE_SIZE (#7367)
add 84515a2f8d [IOTDB-3216] Refactor Segments in SchemaFile to B+ Tree structure from list structure (#5976)
add 63864bf6a6 [IOTDB-4292] Make semantic of like and regexp expressions the same as 0.13
add 9ff80560ca [IOTDB-4469] Frame size larger than protect max size! (#7369)
add 151cc07aaf [IOTDB-4454] Show build info in cli and SHOW VERSION query (#7365)
add 99352611c8 [IOTDB-4435] Fix show child nodes failed (#7366)
add a363ff2d8a AlignedChunkReader should uncompress value page by the compress type in chunkHeader (#7368)
add 44238ce1c1 [IOTDB-4466]error code is incorrect in groupby time query when session has expired (#7370)
add b10f6bcf85 fix two issues in retry and multileader (#7363)
add c9ee123814 [IOTDB-4364]Reduce read amplication in compaction (#7312)
add 867eaf322f [IOTDB-4479] Add pattern overlap check when deleting timeseries (#7373)
add 84c0833dda [IOTDB-4430] Restrict multi_dir_strategy to MaxDiskUsableSpaceFirstStrategy when using cluster (#7358)
add ab0661c36c Bump flink.version from 1.14.0 to 1.14.5 (#7337)
add 0be43d6b6e Bump snakeyaml from 1.17 to 1.31 in /metrics/interface (#7336)
add 6d3c8c5725 Bump logback-classic from 1.2.10 to 1.2.11 (#7338)
add 479d8eec80 [IOTDB-4478] [Ratis] Add RatisConsensus config parameters in confignode.properties
add 9436a29446 Update Time zone description in UserGuide (#7391)
add e6a85b6352 print redundancy clean session log when no session exist (#7372)
add 5036ee2c61 [IOTDB-3862] Fix: Guarantee to add Ratis MetaFile
add 3c40544133 [IOTDB-4394] Build SessionIT environment (#7326)
add 265992dc21 [IOTDB-4251] Persist ChunkMetadata in TsFileIOWriter ahead of time to save memory (#7276)
add bbd0ee0b8f [IOTDB-4469]add data size while loading enire chunk (#7389)
add 31b1ea198a [IOTDB-4477] Support datatype cast while inserting with schema mismatch (#7378)
add 4eced0661c [IOTDB-4480]fix file name incorrect in compaction test
add 1363abf0a7 [IOTDB-4420] Monitor DataNode's remain disk space through cluster heartbeat (#7361)
add b0dac5ec59 [IOTDB-4399] Control text chunk size in memtable (#7320)
add 08829f5b9c [IOTDB-4449][IOTDB-4450] Optimize SchemaFetch (#7395)
add 22149b0f99 [IOTDB-4477] fix castDataType error (#7398)
add 5cc478c8e6 [IOTDB-4392] build separate cache queue for MultiLeader log sync
add f0ecd841aa add a section of delete data (#7311)
add 634a128f94 rename methods (#7396)
add 0d65058b35 [IOTDB-4492] Add necessary log and control total size for cross space compaction task (#7399)
add 7dc11bae56 [IOTDB-4493] Fix null message in TSStatus (#7402)
add c9fc325bd3 [IoTDB-4328] Complete add/remove replica for MultiLeaderConsensus (#7390)
add fb8400e161 reset ResultHandleContext before retyr (#7405)
add f99850c542 terminate workerMonitorExecutor (#7404)
add 46de3f32a5 [IOTDB-3586] ext-pipe FWK support Time-Aligned data (#7138)
add 0d34cda0a6 print absolute file path when flush error occured (#7415)
add aebafee38e [IOTDB-4495][PartitionCache] Add auto create schema judgement (#7411)
add e4250bb101 [IOTDB-4080] Change the port name of show cluster, show regions, show confignodes, show datandoes; make running result of `Remove-datanode.sh` clear (#7403)
add 09e358731c [IOTDB-4312] header parsing error in ImportCSV (#7214)
add f58133d4f6 [IOTDB-4324][IOTDB-4339] Antlr and Statement for Show Triggers and Implementation (#7234)
add 6b04945052 [IOTDB-4481] Use time_partition_interval_for_routing in PartitionFetcher (#7407)
add d8e310e1c6 Test reporter will never fail due to path (#7418)
add e2594a3a6d [MultiLeader] Fix the issue that the wal won't be deleted when leader transfer to follower (#7421)
add 630c68a01e [IOTDB-4518] Fix cannot delete .cmt -> .meta file in disk (#7423)
add 04d9a4a540 [IOTDB-4501]fix FileAlreadyExistsException when taking snapshot (#7408)
new 260edd222f Merge remote-tracking branch 'origin/master' into lmh/RefactorAnalyzer
new 2920ccb717 refactor Analysis
new b880760c64 refactor LogicalPlanner
new 7bed396603 add AnalyzeTest
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
cli/pom.xml | 5 +
.../java/org/apache/iotdb/cli/AbstractCli.java | 5 +-
cli/src/main/java/org/apache/iotdb/cli/Cli.java | 2 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 30 +-
.../resources/conf/iotdb-confignode.properties | 38 +-
.../confignode/client/DataNodeRequestType.java | 46 +-
.../async/handlers/ConfigNodeHeartbeatHandler.java | 4 +-
.../async/handlers/DataNodeHeartbeatHandler.java | 8 +-
.../sync/datanode/SyncDataNodeClientPool.java | 2 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 75 +-
.../confignode/conf/ConfigNodeDescriptor.java | 38 +-
.../confignode/manager/ClusterSchemaManager.java | 2 +
.../iotdb/confignode/manager/ConfigManager.java | 11 +-
.../iotdb/confignode/manager/ConsensusManager.java | 1 +
.../apache/iotdb/confignode/manager/IManager.java | 2 +
.../iotdb/confignode/manager/ProcedureManager.java | 1 +
.../iotdb/confignode/manager/load/LoadManager.java | 6 +-
.../manager/load/LoadManagerMetrics.java | 4 +-
.../manager/load/balancer/RegionBalancer.java | 4 +-
.../manager/load/balancer/RouteBalancer.java | 4 +-
.../partition/GreedyPartitionAllocator.java | 2 +-
.../{load/heartbeat => node}/BaseNodeCache.java | 28 +-
.../ConfigNodeHeartbeatCache.java | 9 +-
.../heartbeat => node}/DataNodeHeartbeatCache.java | 28 +-
.../heartbeat => node}/NodeHeartbeatSample.java | 37 +-
.../confignode/manager/{ => node}/NodeManager.java | 63 +-
.../manager/{ => partition}/PartitionManager.java | 15 +-
.../{load/heartbeat => partition}/RegionCache.java | 6 +-
.../heartbeat => partition}/RegionGroupCache.java | 2 +-
.../RegionHeartbeatSample.java | 2 +-
.../confignode/procedure/ProcedureExecutor.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../procedure/env/DataNodeRemoveHandler.java | 21 +-
.../iotdb/confignode/service/ConfigNode.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +
.../hash/DeviceGroupHashExecutorManualTest.java | 2 +-
.../load/balancer/router/LeaderRouterTest.java | 10 +-
.../balancer/router/LoadScoreGreedyRouterTest.java | 6 +-
.../confignode1conf/iotdb-confignode.properties | 18 +-
.../confignode2conf/iotdb-confignode.properties | 10 +-
.../confignode3conf/iotdb-confignode.properties | 10 +-
.../request/BatchIndexedConsensusRequest.java | 8 +-
.../exception/ConsensusGroupAddPeerException.java | 28 +-
.../multileader/MultiLeaderConsensus.java | 73 +-
.../multileader/MultiLeaderServerImpl.java | 331 ++++-
.../multileader/client/DispatchLogHandler.java | 10 +-
.../client/MultiLeaderConsensusClientPool.java | 24 +
.../client/SyncMultiLeaderServiceClient.java | 144 +++
.../multileader/logdispatcher/IndexController.java | 34 +-
.../multileader/logdispatcher/LogDispatcher.java | 101 +-
.../service/MultiLeaderRPCServiceProcessor.java | 179 ++-
.../multileader/snapshot/SnapshotFragment.java | 81 ++
.../snapshot/SnapshotFragmentReader.java | 69 ++
.../iotdb/consensus/ratis/SnapshotStorage.java | 13 +-
.../logdispatcher/IndexControllerTest.java | 8 +-
.../multileader/logdispatcher/SyncStatusTest.java | 8 +-
docs/UserGuide/Data-Concept/Time-Partition.md | 12 +-
docs/UserGuide/Data-Concept/Time-zone.md | 2 +-
.../Delete-Data.md | 0
.../{Operate-Metadata => Delete-Data}/TTL.md | 0
.../Maintenance-Tools/Maintenance-Command.md | 254 ++--
.../Reference/ConfigNode-Config-Manual.md | 12 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 30 +-
.../CSV-Tool.md | 0
.../Load-External-Tsfile.md | 0
.../TsFile-Tool.md | 0
.../Write-Data.md | 0
docs/zh/UserGuide/Data-Concept/Time-Partition.md | 16 +-
docs/zh/UserGuide/Data-Concept/Time-zone.md | 2 +-
.../Delete-Data.md | 0
.../{Operate-Metadata => Delete-Data}/TTL.md | 0
.../Maintenance-Tools/Maintenance-Command.md | 260 ++--
.../Reference/ConfigNode-Config-Manual.md | 16 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 12 +-
.../CSV-Tool.md | 0
.../Load-External-Tsfile.md | 0
.../TsFile-Tool.md | 0
.../Write-Data.md | 0
example/ext-pipe-plugin-example/pom.xml | 12 +-
.../iotdb/extpipe/ExtPipeSinkWriterFactory.java | 47 +-
.../iotdb/extpipe/ExtPipeSinkWriterImpl.java | 124 +-
example/pom.xml | 1 +
.../pipe/external/api/IExternalPipeSinkWriter.java | 30 +-
.../iotdb/influxdb/session/InfluxDBSession.java | 12 +-
integration-test/checkstyle.xml | 2 +-
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 14 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 6 +
.../org/apache/iotdb/it/env/RemoteServerEnv.java | 10 +
.../iotdb/it/framework/IoTDBTestReporter.java | 18 +-
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 65 +
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 48 +
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 23 +
.../iotdb/session/IoTDBSessionComplexIT.java | 407 ++++++
.../session/IoTDBSessionDisableMemControlIT.java | 220 ++++
.../java/org/apache/iotdb/session/SessionTest.java | 180 +++
.../apache/iotdb/session/pool/SessionPoolTest.java | 16 +-
metrics/interface/pom.xml | 2 +-
node-commons/pom.xml | 32 +
.../apache/iotdb/commons/cluster/NodeStatus.java | 1 +
.../apache/iotdb/commons/conf/CommonConfig.java | 49 +-
.../iotdb/commons/conf/CommonDescriptor.java | 11 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 23 +-
.../iotdb/commons/trigger/TriggerInformation.java | 17 +-
.../apache/iotdb/commons/trigger/TriggerTable.java | 17 +-
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 4 +-
pom.xml | 4 +-
.../resources/conf/iotdb-datanode.properties | 30 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 173 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 49 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 37 +-
.../statemachine/DataRegionStateMachine.java | 167 +--
.../apache/iotdb/db/engine/StorageEngineV2.java | 36 +-
.../compaction/cross/CrossSpaceCompactionTask.java | 17 +-
.../RewriteCrossSpaceCompactionSelector.java | 57 +-
.../rewrite/task/ReadPointPerformerSubTask.java | 10 +-
.../utils/AlignedSeriesCompactionExecutor.java | 1 +
.../inner/utils/MultiTsFileDeviceIterator.java | 76 +-
.../utils/SingleSeriesCompactionExecutor.java | 1 +
.../compaction/performer/ICompactionPerformer.java | 4 +-
.../impl/ReadChunkCompactionPerformer.java | 15 +-
.../impl/ReadPointCompactionPerformer.java | 174 +--
.../compaction/task/CompactionRecoverTask.java | 7 +
.../writer/AbstractCompactionWriter.java | 12 +-
.../writer/CrossSpaceCompactionWriter.java | 35 +-
.../writer/InnerSpaceCompactionWriter.java | 30 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 23 +-
.../iotdb/db/engine/load/AlignedChunkData.java | 10 +-
.../iotdb/db/engine/load/NonAlignedChunkData.java | 8 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 43 +-
.../engine/memtable/AlignedWritableMemChunk.java | 23 +-
.../memtable/AlignedWritableMemChunkGroup.java | 10 +-
.../db/engine/memtable/IWritableMemChunk.java | 16 +-
.../db/engine/memtable/IWritableMemChunkGroup.java | 5 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 29 +-
.../db/engine/memtable/WritableMemChunkGroup.java | 26 +-
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 6 +
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 13 +-
.../db/engine/storagegroup/TsFileProcessor.java | 23 +-
.../db/engine/storagegroup/TsFileResource.java | 20 +-
...Exception.java => ColossalRecordException.java} | 28 +-
.../schemafile/SchemaPageOverflowException.java | 3 +-
.../schemafile/SegmentNotFoundException.java | 11 +
.../schemafile/SegmentOverflowException.java | 7 +
.../store/disk/schemafile/AliasIndexPage.java | 518 ++++++++
.../mtree/store/disk/schemafile/ISchemaPage.java | 171 +--
.../mtree/store/disk/schemafile/ISegment.java | 71 +-
.../store/disk/schemafile/ISegmentedPage.java | 97 ++
.../mtree/store/disk/schemafile/InternalPage.java | 583 +++++++++
.../mtree/store/disk/schemafile/RecordUtils.java | 84 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 735 +----------
.../store/disk/schemafile/SchemaFileConfig.java | 88 ++
.../mtree/store/disk/schemafile/SchemaPage.java | 598 +--------
.../mtree/store/disk/schemafile/Segment.java | 737 ++++-------
.../mtree/store/disk/schemafile/SegmentedPage.java | 580 +++++++++
.../store/disk/schemafile/WrappedSegment.java | 479 +++++++
.../disk/schemafile/pagemgr/BTreePageManager.java | 496 ++++++++
.../disk/schemafile/pagemgr/IPageManager.java | 59 +
.../store/disk/schemafile/pagemgr/PageManager.java | 619 ++++++++++
.../iotdb/db/mpp/common/MPPQueryContext.java | 12 +-
.../db/mpp/common/header/ColumnHeaderConstant.java | 34 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../operator/schema/NodePathsConvertOperator.java | 2 +-
.../apache/iotdb/db/mpp/plan/TestRPCClient.java | 176 +++
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 93 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 7 +-
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 1 +
.../db/mpp/plan/analyze/cache/PartitionCache.java | 3 +-
.../iotdb/db/mpp/plan/constant/StatementType.java | 2 +
.../db/mpp/plan/execution/QueryExecution.java | 2 +
.../plan/execution/config/ConfigTaskVisitor.java | 10 +
.../config/executor/ClusterConfigTaskExecutor.java | 24 +
.../config/executor/IConfigTaskExecutor.java | 2 +
.../executor/StandaloneConfigTaskExecutor.java | 10 +
.../config/metadata/ShowTriggersTask.java | 93 ++
.../memory/StatementMemorySourceVisitor.java | 1 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 29 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 136 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 252 ++--
.../SimpleFragmentParallelPlanner.java | 2 +-
.../plan/node/load/LoadSingleTsFileNode.java | 30 +-
.../plan/node/load/LoadTsFilePieceNode.java | 12 +-
.../plan/node/write/InsertMultiTabletsNode.java | 5 +
.../plan/planner/plan/node/write/InsertNode.java | 5 +
.../planner/plan/node/write/InsertRowNode.java | 23 +-
.../planner/plan/node/write/InsertRowsNode.java | 9 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 9 +-
.../planner/plan/node/write/InsertTabletNode.java | 55 +-
.../db/mpp/plan/scheduler/ClusterScheduler.java | 1 +
.../db/mpp/plan/statement/StatementVisitor.java | 5 +
.../crud/InsertMultiTabletsStatement.java | 4 +-
.../plan/statement/crud/InsertRowStatement.java | 6 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 5 +-
.../plan/statement/crud/InsertRowsStatement.java | 4 +-
.../plan/statement/crud/InsertTabletStatement.java | 22 +-
.../statement/metadata/ShowTriggersStatement.java} | 48 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 9 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 31 +-
.../iotdb/db/query/control/SessionManager.java | 8 +-
.../db/query/control/SessionTimeoutManager.java | 12 +-
.../iotdb/db/rescon/PrimitiveArrayManager.java | 3 +
.../java/org/apache/iotdb/db/service/DataNode.java | 45 +-
.../db/service/DataNodeServerCommandLine.java | 7 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 -
.../service/metrics/predefined/SystemMetrics.java | 46 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 3 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 116 +-
.../db/service/thrift/impl/TSServiceImpl.java | 3 +-
.../iotdb/db/sync/datasource/TsFileOpBlock.java | 692 ++++++++---
.../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 29 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 4 +-
.../trigger/service/TriggerManagementService.java | 24 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 79 ++
.../apache/iotdb/db/utils/TimePartitionUtils.java | 41 +
.../db/utils/datastructure/AlignedTVList.java | 186 ++-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 85 ++
.../db/utils/datastructure/BooleanTVList.java | 45 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 39 +
.../iotdb/db/utils/datastructure/FloatTVList.java | 39 +
.../iotdb/db/utils/datastructure/IntTVList.java | 28 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 39 +
.../db/utils/datastructure/QuickAlignedTVList.java | 61 +
.../datastructure/QuickBinaryTVList.java} | 45 +-
.../datastructure/QuickBooleanTVList.java} | 44 +-
.../datastructure/QuickDoubleTVList.java} | 44 +-
.../datastructure/QuickFloatTVList.java} | 44 +-
.../datastructure/QuickIntTVList.java} | 44 +-
.../datastructure/QuickLongTVList.java} | 44 +-
.../iotdb/db/utils/datastructure/QuickSort.java | 73 ++
.../iotdb/db/utils/datastructure/TVList.java | 33 +-
.../utils/datastructure/TVListSortAlgorithm.java | 25 +-
.../db/utils/datastructure/TimAlignedTVList.java | 131 --
.../db/utils/datastructure/TimBinaryTVList.java | 33 -
.../db/utils/datastructure/TimBooleanTVList.java | 33 -
.../db/utils/datastructure/TimDoubleTVList.java | 32 -
.../db/utils/datastructure/TimFloatTVList.java | 32 -
.../iotdb/db/utils/datastructure/TimIntTVList.java | 16 -
.../db/utils/datastructure/TimLongTVList.java | 32 -
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 27 +-
.../file/AbstractTsFileRecoverPerformer.java | 18 +-
.../engine/compaction/AbstractCompactionTest.java | 121 +-
.../ReadPointCompactionPerformerTest.java | 1269 ++++++++++++++++++-
.../cross/RewriteCompactionFileSelectorTest.java | 30 +-
.../compaction/inner/InnerSeqCompactionTest.java | 7 +-
.../compaction/inner/InnerUnseqCompactionTest.java | 13 +-
.../inner/ReadChunkCompactionPerformerOldTest.java | 3 +-
.../db/engine/memtable/PrimitiveMemTableTest.java | 10 +-
.../mtree/schemafile/AliasIndexPageTest.java | 237 ++++
.../mtree/schemafile/InternalPageTest.java | 222 ++++
.../metadata/mtree/schemafile/RecordUtilTests.java | 21 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 290 ++++-
.../metadata/mtree/schemafile/SchemaPageTest.java | 71 +-
.../db/metadata/mtree/schemafile/SegmentTest.java | 151 ---
.../mtree/schemafile/WrappedSegmentTest.java | 248 ++++
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 90 +-
.../distribution/DistributionPlannerBasicTest.java | 2 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 27 +-
.../db/sync/datasource/PipeOpManagerTest.java | 171 ++-
.../db/sync/datasource/TsFileOpBlockTest.java | 619 +++++++++-
.../db/utils/datastructure/BinaryTVListTest.java | 38 +-
.../db/utils/datastructure/BooleanTVListTest.java | 8 +-
.../db/utils/datastructure/DoubleTVListTest.java | 12 +-
.../db/utils/datastructure/FloatTVListTest.java | 10 +-
.../db/utils/datastructure/IntTVListTest.java | 12 +-
.../db/utils/datastructure/LongTVListTest.java | 12 +-
.../db/utils/datastructure/VectorTVListTest.java | 77 +-
.../resources/datanode1conf}/iotdb-rest.properties | 0
.../resources/datanode2conf}/iotdb-rest.properties | 0
.../resources/datanode3conf}/iotdb-rest.properties | 0
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/session/ISession.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 136 +-
.../session/{Config.java => SessionConfig.java} | 2 +-
.../apache/iotdb/session/SessionConnection.java | 2 +-
.../org/apache/iotdb/session/SessionDataSet.java | 4 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 62 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 12 +-
site/src/main/.vuepress/config.js | 36 +-
.../src/main/thrift/confignode.thrift | 45 +-
.../src/main/thrift/mutlileader.thrift | 65 +-
thrift/src/main/thrift/client.thrift | 1 +
thrift/src/main/thrift/datanode.thrift | 15 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 8 +
.../file/metadata/MetadataIndexConstructor.java | 9 +-
.../tsfile/file/metadata/MetadataIndexNode.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 37 +
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 9 +-
.../iotdb/tsfile/read/TsFileDeviceIterator.java | 14 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 67 +
.../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +-
.../read/reader/chunk/AlignedChunkReader.java | 1 +
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 107 ++
.../write/writer/RestorableTsFileIOWriter.java | 12 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 344 ++++--
.../write/writer/tsmiterator/DiskTSMIterator.java | 132 ++
.../write/writer/tsmiterator/TSMIterator.java | 147 +++
.../iotdb/tsfile/read/common/TimeRangeTest.java | 19 +
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 15 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 251 ++++
.../writer/TsFileIOWriterMemoryControlTest.java | 1303 ++++++++++++++++++++
.../zeppelin/iotdb/IoTDBInterpreterTest.java | 4 +-
303 files changed, 16279 insertions(+), 4810 deletions(-)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => node}/BaseNodeCache.java (76%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => node}/ConfigNodeHeartbeatCache.java (91%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => node}/DataNodeHeartbeatCache.java (81%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => node}/NodeHeartbeatSample.java (67%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{ => node}/NodeManager.java (90%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{ => partition}/PartitionManager.java (98%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => partition}/RegionCache.java (91%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => partition}/RegionGroupCache.java (98%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{load/heartbeat => partition}/RegionHeartbeatSample.java (96%)
copy thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift => consensus/src/main/java/org/apache/iotdb/consensus/exception/ConsensusGroupAddPeerException.java (64%)
create mode 100644 consensus/src/main/java/org/apache/iotdb/consensus/multileader/client/SyncMultiLeaderServiceClient.java
create mode 100644 consensus/src/main/java/org/apache/iotdb/consensus/multileader/snapshot/SnapshotFragment.java
create mode 100644 consensus/src/main/java/org/apache/iotdb/consensus/multileader/snapshot/SnapshotFragmentReader.java
rename docs/UserGuide/{Write-And-Delete-Data => Delete-Data}/Delete-Data.md (100%)
rename docs/UserGuide/{Operate-Metadata => Delete-Data}/TTL.md (100%)
rename docs/UserGuide/{Write-And-Delete-Data => Write-Data}/CSV-Tool.md (100%)
rename docs/UserGuide/{Write-And-Delete-Data => Write-Data}/Load-External-Tsfile.md (100%)
rename docs/UserGuide/{Write-And-Delete-Data => Write-Data}/TsFile-Tool.md (100%)
rename docs/UserGuide/{Write-And-Delete-Data => Write-Data}/Write-Data.md (100%)
rename docs/zh/UserGuide/{Write-And-Delete-Data => Delete-Data}/Delete-Data.md (100%)
rename docs/zh/UserGuide/{Operate-Metadata => Delete-Data}/TTL.md (100%)
rename docs/zh/UserGuide/{Write-And-Delete-Data => Write-Data}/CSV-Tool.md (100%)
rename docs/zh/UserGuide/{Write-And-Delete-Data => Write-Data}/Load-External-Tsfile.md (100%)
rename docs/zh/UserGuide/{Write-And-Delete-Data => Write-Data}/TsFile-Tool.md (100%)
rename docs/zh/UserGuide/{Write-And-Delete-Data => Write-Data}/Write-Data.md (100%)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/session/IoTDBSessionComplexIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/session/IoTDBSessionDisableMemControlIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/session/SessionTest.java
copy server/src/main/java/org/apache/iotdb/db/exception/metadata/schemafile/{SegmentNotFoundException.java => ColossalRecordException.java} (53%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/AliasIndexPage.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISegmentedPage.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/InternalPage.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SchemaFileConfig.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SegmentedPage.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/WrappedSegment.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/BTreePageManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/IPageManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/PageManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/TestRPCClient.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/metadata/ShowTriggersTask.java
copy server/src/main/java/org/apache/iotdb/db/{engine/memtable/IWritableMemChunkGroup.java => mpp/plan/statement/metadata/ShowTriggersStatement.java} (51%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickAlignedTVList.java
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickBinaryTVList.java} (50%)
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickBooleanTVList.java} (51%)
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickDoubleTVList.java} (52%)
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickFloatTVList.java} (52%)
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickIntTVList.java} (52%)
copy server/src/main/java/org/apache/iotdb/db/{exception/metadata/schemafile/SegmentOverflowException.java => utils/datastructure/QuickLongTVList.java} (52%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/utils/datastructure/QuickSort.java
copy thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift => server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVListSortAlgorithm.java (64%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/AliasIndexPageTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/InternalPageTest.java
delete mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/SegmentTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/mtree/schemafile/WrappedSegmentTest.java
copy server/src/{assembly/resources/conf => test/resources/datanode1conf}/iotdb-rest.properties (100%)
copy server/src/{assembly/resources/conf => test/resources/datanode2conf}/iotdb-rest.properties (100%)
copy server/src/{assembly/resources/conf => test/resources/datanode3conf}/iotdb-rest.properties (100%)
rename session/src/main/java/org/apache/iotdb/session/{Config.java => SessionConfig.java} (98%)
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/DiskTSMIterator.java
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/tsmiterator/TSMIterator.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIntegrityCheckingTool.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriterMemoryControlTest.java
[iotdb] 01/04: Merge remote-tracking branch 'origin/master' into lmh/RefactorAnalyzer
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/RefactorAnalyzer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 260edd222fa94ad8cc864933140d8d4562fee89c
Merge: 5160df0b59 04d9a4a540
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Sep 26 09:34:18 2022 +0800
Merge remote-tracking branch 'origin/master' into lmh/RefactorAnalyzer
cli/pom.xml | 5 +
.../java/org/apache/iotdb/cli/AbstractCli.java | 5 +-
cli/src/main/java/org/apache/iotdb/cli/Cli.java | 2 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 30 +-
.../resources/conf/iotdb-confignode.properties | 38 +-
.../confignode/client/DataNodeRequestType.java | 46 +-
.../async/handlers/ConfigNodeHeartbeatHandler.java | 4 +-
.../async/handlers/DataNodeHeartbeatHandler.java | 8 +-
.../sync/datanode/SyncDataNodeClientPool.java | 2 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 75 +-
.../confignode/conf/ConfigNodeDescriptor.java | 38 +-
.../confignode/manager/ClusterSchemaManager.java | 2 +
.../iotdb/confignode/manager/ConfigManager.java | 11 +-
.../iotdb/confignode/manager/ConsensusManager.java | 1 +
.../apache/iotdb/confignode/manager/IManager.java | 2 +
.../iotdb/confignode/manager/ProcedureManager.java | 1 +
.../iotdb/confignode/manager/load/LoadManager.java | 6 +-
.../manager/load/LoadManagerMetrics.java | 4 +-
.../manager/load/balancer/RegionBalancer.java | 4 +-
.../manager/load/balancer/RouteBalancer.java | 4 +-
.../partition/GreedyPartitionAllocator.java | 2 +-
.../{load/heartbeat => node}/BaseNodeCache.java | 28 +-
.../ConfigNodeHeartbeatCache.java | 9 +-
.../heartbeat => node}/DataNodeHeartbeatCache.java | 28 +-
.../heartbeat => node}/NodeHeartbeatSample.java | 37 +-
.../confignode/manager/{ => node}/NodeManager.java | 63 +-
.../manager/{ => partition}/PartitionManager.java | 15 +-
.../{load/heartbeat => partition}/RegionCache.java | 6 +-
.../heartbeat => partition}/RegionGroupCache.java | 2 +-
.../RegionHeartbeatSample.java | 2 +-
.../confignode/procedure/ProcedureExecutor.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../procedure/env/DataNodeRemoveHandler.java | 21 +-
.../iotdb/confignode/service/ConfigNode.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 7 +
.../hash/DeviceGroupHashExecutorManualTest.java | 2 +-
.../load/balancer/router/LeaderRouterTest.java | 10 +-
.../balancer/router/LoadScoreGreedyRouterTest.java | 6 +-
.../confignode1conf/iotdb-confignode.properties | 18 +-
.../confignode2conf/iotdb-confignode.properties | 10 +-
.../confignode3conf/iotdb-confignode.properties | 10 +-
.../request/BatchIndexedConsensusRequest.java | 8 +-
.../exception/ConsensusGroupAddPeerException.java | 28 +-
.../multileader/MultiLeaderConsensus.java | 73 +-
.../multileader/MultiLeaderServerImpl.java | 331 ++++-
.../multileader/client/DispatchLogHandler.java | 10 +-
.../client/MultiLeaderConsensusClientPool.java | 24 +
.../client/SyncMultiLeaderServiceClient.java | 144 +++
.../multileader/logdispatcher/IndexController.java | 34 +-
.../multileader/logdispatcher/LogDispatcher.java | 101 +-
.../service/MultiLeaderRPCServiceProcessor.java | 179 ++-
.../multileader/snapshot/SnapshotFragment.java | 81 ++
.../snapshot/SnapshotFragmentReader.java | 69 ++
.../iotdb/consensus/ratis/SnapshotStorage.java | 13 +-
.../logdispatcher/IndexControllerTest.java | 8 +-
.../multileader/logdispatcher/SyncStatusTest.java | 8 +-
docs/UserGuide/Data-Concept/Time-Partition.md | 12 +-
docs/UserGuide/Data-Concept/Time-zone.md | 2 +-
.../Delete-Data.md | 0
.../{Operate-Metadata => Delete-Data}/TTL.md | 0
.../Maintenance-Tools/Maintenance-Command.md | 254 ++--
.../Reference/ConfigNode-Config-Manual.md | 12 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 30 +-
.../CSV-Tool.md | 0
.../Load-External-Tsfile.md | 0
.../TsFile-Tool.md | 0
.../Write-Data.md | 0
docs/zh/UserGuide/Data-Concept/Time-Partition.md | 16 +-
docs/zh/UserGuide/Data-Concept/Time-zone.md | 2 +-
.../Delete-Data.md | 0
.../{Operate-Metadata => Delete-Data}/TTL.md | 0
.../Maintenance-Tools/Maintenance-Command.md | 260 ++--
.../Reference/ConfigNode-Config-Manual.md | 16 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 12 +-
.../CSV-Tool.md | 0
.../Load-External-Tsfile.md | 0
.../TsFile-Tool.md | 0
.../Write-Data.md | 0
example/ext-pipe-plugin-example/pom.xml | 12 +-
.../iotdb/extpipe/ExtPipeSinkWriterFactory.java | 47 +-
.../iotdb/extpipe/ExtPipeSinkWriterImpl.java | 124 +-
example/pom.xml | 1 +
.../pipe/external/api/IExternalPipeSinkWriter.java | 30 +-
.../iotdb/influxdb/session/InfluxDBSession.java | 12 +-
integration-test/checkstyle.xml | 2 +-
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 14 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 6 +
.../org/apache/iotdb/it/env/RemoteServerEnv.java | 10 +
.../iotdb/it/framework/IoTDBTestReporter.java | 18 +-
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 65 +
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 48 +
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 23 +
.../iotdb/session/IoTDBSessionComplexIT.java | 407 ++++++
.../session/IoTDBSessionDisableMemControlIT.java | 220 ++++
.../java/org/apache/iotdb/session/SessionTest.java | 180 +++
.../apache/iotdb/session/pool/SessionPoolTest.java | 16 +-
metrics/interface/pom.xml | 2 +-
node-commons/pom.xml | 32 +
.../apache/iotdb/commons/cluster/NodeStatus.java | 1 +
.../apache/iotdb/commons/conf/CommonConfig.java | 49 +-
.../iotdb/commons/conf/CommonDescriptor.java | 11 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 23 +-
.../iotdb/commons/trigger/TriggerInformation.java | 17 +-
.../apache/iotdb/commons/trigger/TriggerTable.java | 17 +-
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 4 +-
pom.xml | 4 +-
.../resources/conf/iotdb-datanode.properties | 30 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 173 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 49 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 37 +-
.../statemachine/DataRegionStateMachine.java | 167 +--
.../apache/iotdb/db/engine/StorageEngineV2.java | 36 +-
.../compaction/cross/CrossSpaceCompactionTask.java | 17 +-
.../RewriteCrossSpaceCompactionSelector.java | 57 +-
.../rewrite/task/ReadPointPerformerSubTask.java | 10 +-
.../utils/AlignedSeriesCompactionExecutor.java | 1 +
.../inner/utils/MultiTsFileDeviceIterator.java | 76 +-
.../utils/SingleSeriesCompactionExecutor.java | 1 +
.../compaction/performer/ICompactionPerformer.java | 4 +-
.../impl/ReadChunkCompactionPerformer.java | 15 +-
.../impl/ReadPointCompactionPerformer.java | 174 +--
.../compaction/task/CompactionRecoverTask.java | 7 +
.../writer/AbstractCompactionWriter.java | 12 +-
.../writer/CrossSpaceCompactionWriter.java | 35 +-
.../writer/InnerSpaceCompactionWriter.java | 30 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 23 +-
.../iotdb/db/engine/load/AlignedChunkData.java | 10 +-
.../iotdb/db/engine/load/NonAlignedChunkData.java | 8 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 43 +-
.../engine/memtable/AlignedWritableMemChunk.java | 23 +-
.../memtable/AlignedWritableMemChunkGroup.java | 10 +-
.../db/engine/memtable/IWritableMemChunk.java | 16 +-
.../db/engine/memtable/IWritableMemChunkGroup.java | 5 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 29 +-
.../db/engine/memtable/WritableMemChunkGroup.java | 26 +-
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 6 +
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 13 +-
.../db/engine/storagegroup/TsFileProcessor.java | 23 +-
.../db/engine/storagegroup/TsFileResource.java | 20 +-
...Exception.java => ColossalRecordException.java} | 28 +-
.../schemafile/SchemaPageOverflowException.java | 3 +-
.../schemafile/SegmentNotFoundException.java | 11 +
.../schemafile/SegmentOverflowException.java | 7 +
.../store/disk/schemafile/AliasIndexPage.java | 518 ++++++++
.../mtree/store/disk/schemafile/ISchemaPage.java | 171 +--
.../mtree/store/disk/schemafile/ISegment.java | 71 +-
.../store/disk/schemafile/ISegmentedPage.java | 97 ++
.../mtree/store/disk/schemafile/InternalPage.java | 583 +++++++++
.../mtree/store/disk/schemafile/RecordUtils.java | 84 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 735 +----------
.../store/disk/schemafile/SchemaFileConfig.java | 88 ++
.../mtree/store/disk/schemafile/SchemaPage.java | 598 +--------
.../mtree/store/disk/schemafile/Segment.java | 737 ++++-------
.../mtree/store/disk/schemafile/SegmentedPage.java | 580 +++++++++
.../store/disk/schemafile/WrappedSegment.java | 479 +++++++
.../disk/schemafile/pagemgr/BTreePageManager.java | 496 ++++++++
.../disk/schemafile/pagemgr/IPageManager.java | 59 +
.../store/disk/schemafile/pagemgr/PageManager.java | 619 ++++++++++
.../iotdb/db/mpp/common/MPPQueryContext.java | 12 +-
.../db/mpp/common/header/ColumnHeaderConstant.java | 34 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../operator/schema/NodePathsConvertOperator.java | 2 +-
.../apache/iotdb/db/mpp/plan/TestRPCClient.java | 176 +++
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 7 +-
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 1 +
.../db/mpp/plan/analyze/cache/PartitionCache.java | 3 +-
.../iotdb/db/mpp/plan/constant/StatementType.java | 2 +
.../db/mpp/plan/execution/QueryExecution.java | 2 +
.../plan/execution/config/ConfigTaskVisitor.java | 10 +
.../config/executor/ClusterConfigTaskExecutor.java | 24 +
.../config/executor/IConfigTaskExecutor.java | 2 +
.../executor/StandaloneConfigTaskExecutor.java | 10 +
.../config/metadata/ShowTriggersTask.java | 93 ++
.../memory/StatementMemorySourceVisitor.java | 1 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 29 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 4 +-
.../SimpleFragmentParallelPlanner.java | 2 +-
.../plan/node/load/LoadSingleTsFileNode.java | 30 +-
.../plan/node/load/LoadTsFilePieceNode.java | 12 +-
.../plan/node/write/InsertMultiTabletsNode.java | 5 +
.../plan/planner/plan/node/write/InsertNode.java | 5 +
.../planner/plan/node/write/InsertRowNode.java | 23 +-
.../planner/plan/node/write/InsertRowsNode.java | 9 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 9 +-
.../planner/plan/node/write/InsertTabletNode.java | 55 +-
.../db/mpp/plan/scheduler/ClusterScheduler.java | 1 +
.../db/mpp/plan/statement/StatementVisitor.java | 5 +
.../crud/InsertMultiTabletsStatement.java | 4 +-
.../plan/statement/crud/InsertRowStatement.java | 6 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 5 +-
.../plan/statement/crud/InsertRowsStatement.java | 4 +-
.../plan/statement/crud/InsertTabletStatement.java | 22 +-
.../statement/metadata/ShowTriggersStatement.java} | 48 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 9 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 31 +-
.../iotdb/db/query/control/SessionManager.java | 8 +-
.../db/query/control/SessionTimeoutManager.java | 12 +-
.../iotdb/db/rescon/PrimitiveArrayManager.java | 3 +
.../java/org/apache/iotdb/db/service/DataNode.java | 45 +-
.../db/service/DataNodeServerCommandLine.java | 7 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 -
.../service/metrics/predefined/SystemMetrics.java | 46 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 3 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 116 +-
.../db/service/thrift/impl/TSServiceImpl.java | 3 +-
.../iotdb/db/sync/datasource/TsFileOpBlock.java | 692 ++++++++---
.../iotdb/db/sync/externalpipe/ExtPipePlugin.java | 29 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 4 +-
.../trigger/service/TriggerManagementService.java | 24 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 79 ++
.../apache/iotdb/db/utils/TimePartitionUtils.java | 41 +
.../db/utils/datastructure/AlignedTVList.java | 186 ++-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 85 ++
.../db/utils/datastructure/BooleanTVList.java | 45 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 39 +
.../iotdb/db/utils/datastructure/FloatTVList.java | 39 +
.../iotdb/db/utils/datastructure/IntTVList.java | 28 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 39 +
.../db/utils/datastructure/QuickAlignedTVList.java | 61 +
.../datastructure/QuickBinaryTVList.java} | 45 +-
.../datastructure/QuickBooleanTVList.java} | 44 +-
.../datastructure/QuickDoubleTVList.java} | 44 +-
.../datastructure/QuickFloatTVList.java} | 44 +-
.../datastructure/QuickIntTVList.java} | 44 +-
.../datastructure/QuickLongTVList.java} | 44 +-
.../iotdb/db/utils/datastructure/QuickSort.java | 73 ++
.../iotdb/db/utils/datastructure/TVList.java | 33 +-
.../utils/datastructure/TVListSortAlgorithm.java | 25 +-
.../db/utils/datastructure/TimAlignedTVList.java | 131 --
.../db/utils/datastructure/TimBinaryTVList.java | 33 -
.../db/utils/datastructure/TimBooleanTVList.java | 33 -
.../db/utils/datastructure/TimDoubleTVList.java | 32 -
.../db/utils/datastructure/TimFloatTVList.java | 32 -
.../iotdb/db/utils/datastructure/TimIntTVList.java | 16 -
.../db/utils/datastructure/TimLongTVList.java | 32 -
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 27 +-
.../file/AbstractTsFileRecoverPerformer.java | 18 +-
.../engine/compaction/AbstractCompactionTest.java | 121 +-
.../ReadPointCompactionPerformerTest.java | 1269 ++++++++++++++++++-
.../cross/RewriteCompactionFileSelectorTest.java | 30 +-
.../compaction/inner/InnerSeqCompactionTest.java | 7 +-
.../compaction/inner/InnerUnseqCompactionTest.java | 13 +-
.../inner/ReadChunkCompactionPerformerOldTest.java | 3 +-
.../db/engine/memtable/PrimitiveMemTableTest.java | 10 +-
.../mtree/schemafile/AliasIndexPageTest.java | 237 ++++
.../mtree/schemafile/InternalPageTest.java | 222 ++++
.../metadata/mtree/schemafile/RecordUtilTests.java | 21 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 290 ++++-
.../metadata/mtree/schemafile/SchemaPageTest.java | 71 +-
.../db/metadata/mtree/schemafile/SegmentTest.java | 151 ---
.../mtree/schemafile/WrappedSegmentTest.java | 248 ++++
.../distribution/DistributionPlannerBasicTest.java | 2 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 27 +-
.../db/sync/datasource/PipeOpManagerTest.java | 171 ++-
.../db/sync/datasource/TsFileOpBlockTest.java | 619 +++++++++-
.../db/utils/datastructure/BinaryTVListTest.java | 38 +-
.../db/utils/datastructure/BooleanTVListTest.java | 8 +-
.../db/utils/datastructure/DoubleTVListTest.java | 12 +-
.../db/utils/datastructure/FloatTVListTest.java | 10 +-
.../db/utils/datastructure/IntTVListTest.java | 12 +-
.../db/utils/datastructure/LongTVListTest.java | 12 +-
.../db/utils/datastructure/VectorTVListTest.java | 77 +-
.../resources/datanode1conf/iotdb-rest.properties | 61 +
.../resources/datanode2conf/iotdb-rest.properties | 61 +
.../resources/datanode3conf/iotdb-rest.properties | 61 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/session/ISession.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 136 +-
.../session/{Config.java => SessionConfig.java} | 2 +-
.../apache/iotdb/session/SessionConnection.java | 2 +-
.../org/apache/iotdb/session/SessionDataSet.java | 4 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 62 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 12 +-
site/src/main/.vuepress/config.js | 36 +-
.../src/main/thrift/confignode.thrift | 45 +-
.../src/main/thrift/mutlileader.thrift | 65 +-
thrift/src/main/thrift/client.thrift | 1 +
thrift/src/main/thrift/datanode.thrift | 15 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 8 +
.../file/metadata/MetadataIndexConstructor.java | 9 +-
.../tsfile/file/metadata/MetadataIndexNode.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 37 +
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 9 +-
.../iotdb/tsfile/read/TsFileDeviceIterator.java | 14 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 67 +
.../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +-
.../read/reader/chunk/AlignedChunkReader.java | 1 +
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 107 ++
.../write/writer/RestorableTsFileIOWriter.java | 12 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 344 ++++--
.../write/writer/tsmiterator/DiskTSMIterator.java | 132 ++
.../write/writer/tsmiterator/TSMIterator.java | 147 +++
.../iotdb/tsfile/read/common/TimeRangeTest.java | 19 +
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 15 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 251 ++++
.../writer/TsFileIOWriterMemoryControlTest.java | 1303 ++++++++++++++++++++
.../zeppelin/iotdb/IoTDBInterpreterTest.java | 4 +-
300 files changed, 16243 insertions(+), 4462 deletions(-)
[iotdb] 02/04: refactor Analysis
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/RefactorAnalyzer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2920ccb717322dab15f61a14f86c2d41efac35bd
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Sep 26 16:31:18 2022 +0800
refactor Analysis
---
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 93 +++-------------------
1 file changed, 12 insertions(+), 81 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
index 8955849d96..e393044b6c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
@@ -78,51 +78,37 @@ public class Analysis {
// input expressions of aggregations to be calculated
private Set<Expression> sourceTransformExpressions;
+ private Expression whereExpression;
+
// all aggregations that need to be calculated
private Set<Expression> aggregationExpressions;
- // expression of output column to be calculated
- private Set<Expression> selectExpressions;
-
- private Expression whereExpression;
-
// map from grouped path name to list of input aggregation in `GROUP BY LEVEL` clause
private Map<Expression, Set<Expression>> groupByLevelExpressions;
- // map from raw path to grouped path in `GROUP BY LEVEL` clause
- private Map<Expression, Expression> rawPathToGroupedPathMap;
-
- private boolean isRawDataSource;
-
/////////////////////////////////////////////////////////////////////////////////////////////////
// Query Analysis (used in ALIGN BY DEVICE)
/////////////////////////////////////////////////////////////////////////////////////////////////
- // used to planTransform after planDeviceView
- Set<Expression> transformInput;
- Set<Expression> transformOutput;
-
// map from device name to series/aggregation under this device
private Map<String, Set<Expression>> deviceToSourceExpressions;
// input expressions of aggregations to be calculated
private Map<String, Set<Expression>> deviceToSourceTransformExpressions;
+ // map from device name to query filter under this device
+ private Map<String, Expression> deviceToWhereExpression;
+
// all aggregations that need to be calculated
private Map<String, Set<Expression>> deviceToAggregationExpressions;
// expression of output column to be calculated
private Map<String, Set<Expression>> deviceToSelectExpressions;
- // map from device name to query filter under this device
- private Map<String, Expression> deviceToWhereExpression;
-
// e.g. [s1,s2,s3] is query, but [s1, s3] exists in device1, then device1 -> [1, 3], s1 is 1 but
// not 0 because device is the first column
private Map<String, List<Integer>> deviceToMeasurementIndexesMap;
- private Map<String, Boolean> deviceToIsRawDataSource;
-
private Set<Expression> deviceViewOutputExpressions;
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -132,25 +118,25 @@ public class Analysis {
// indicate is there a value filter
private boolean hasValueFilter = false;
- private Expression havingExpression;
-
- // true if nested expressions and UDFs exist in aggregation function
- private boolean isHasRawDataInputAggregation;
-
// a global time filter used in `initQueryDataSource` and filter push down
private Filter globalTimeFilter;
+ // expression of output column to be calculated
+ private Set<Expression> selectExpressions;
+
+ private Expression havingExpression;
+
// parameter of `FILL` clause
private FillDescriptor fillDescriptor;
// parameter of `GROUP BY TIME` clause
private GroupByTimeParameter groupByTimeParameter;
+ private OrderByParameter mergeOrderParameter;
+
// header of result dataset
private DatasetHeader respDatasetHeader;
- private OrderByParameter mergeOrderParameter;
-
/////////////////////////////////////////////////////////////////////////////////////////////////
// Schema Query Analysis
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -236,14 +222,6 @@ public class Analysis {
|| (schemaPartition != null && !schemaPartition.isEmpty());
}
- public boolean isHasRawDataInputAggregation() {
- return isHasRawDataInputAggregation;
- }
-
- public void setHasRawDataInputAggregation(boolean hasRawDataInputAggregation) {
- isHasRawDataInputAggregation = hasRawDataInputAggregation;
- }
-
public Map<Expression, Set<Expression>> getGroupByLevelExpressions() {
return groupByLevelExpressions;
}
@@ -252,21 +230,6 @@ public class Analysis {
this.groupByLevelExpressions = groupByLevelExpressions;
}
- public void setRawPathToGroupedPathMap(Map<Expression, Expression> rawPathToGroupedPathMap) {
- this.rawPathToGroupedPathMap = rawPathToGroupedPathMap;
- }
-
- public Expression getGroupedExpressionByLevel(Expression expression) {
- if (rawPathToGroupedPathMap.containsKey(expression)) {
- return rawPathToGroupedPathMap.get(expression);
- }
- if (rawPathToGroupedPathMap.containsValue(expression)) {
- return expression;
- }
- throw new IllegalArgumentException(
- String.format("GROUP BY LEVEL: Unknown input expression '%s'", expression));
- }
-
public FillDescriptor getFillDescriptor() {
return fillDescriptor;
}
@@ -364,22 +327,6 @@ public class Analysis {
this.selectExpressions = selectExpressions;
}
- public Set<Expression> getTransformInput() {
- return transformInput;
- }
-
- public void setTransformInput(Set<Expression> transformInput) {
- this.transformInput = transformInput;
- }
-
- public Set<Expression> getTransformOutput() {
- return transformOutput;
- }
-
- public void setTransformOutput(Set<Expression> transformOutput) {
- this.transformOutput = transformOutput;
- }
-
public Map<String, Set<Expression>> getDeviceToSourceExpressions() {
return deviceToSourceExpressions;
}
@@ -414,22 +361,6 @@ public class Analysis {
this.deviceToSelectExpressions = deviceToSelectExpressions;
}
- public boolean isRawDataSource() {
- return isRawDataSource;
- }
-
- public void setRawDataSource(boolean rawDataSource) {
- isRawDataSource = rawDataSource;
- }
-
- public Map<String, Boolean> getDeviceToIsRawDataSource() {
- return deviceToIsRawDataSource;
- }
-
- public void setDeviceToIsRawDataSource(Map<String, Boolean> deviceToIsRawDataSource) {
- this.deviceToIsRawDataSource = deviceToIsRawDataSource;
- }
-
public Set<TSchemaNode> getMatchedNodes() {
return matchedNodes;
}
[iotdb] 03/04: refactor LogicalPlanner
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/RefactorAnalyzer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b880760c6466d7b6b6b05c97f6fe9c5bb782ff54
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Sep 26 16:31:33 2022 +0800
refactor LogicalPlanner
---
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 132 ++++-------
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 252 ++++++---------------
2 files changed, 119 insertions(+), 265 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index ba0ff63e5d..997386a4d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -97,7 +97,6 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.COLUMN_DEVICE;
@@ -200,18 +199,16 @@ public class LogicalPlanBuilder {
}
public LogicalPlanBuilder planAggregationSource(
- Set<Expression> sourceExpressions,
AggregationStep curStep,
Ordering scanOrder,
Filter timeFilter,
GroupByTimeParameter groupByTimeParameter,
Set<Expression> aggregationExpressions,
- Set<Expression> aggregationTransformExpressions,
Map<Expression, Set<Expression>> groupByLevelExpressions) {
boolean needCheckAscending = groupByTimeParameter == null;
Map<PartialPath, List<AggregationDescriptor>> ascendingAggregations = new HashMap<>();
Map<PartialPath, List<AggregationDescriptor>> descendingAggregations = new HashMap<>();
- for (Expression sourceExpression : sourceExpressions) {
+ for (Expression sourceExpression : aggregationExpressions) {
createAggregationDescriptor(
(FunctionExpression) sourceExpression,
curStep,
@@ -228,73 +225,7 @@ public class LogicalPlanBuilder {
scanOrder,
timeFilter,
groupByTimeParameter);
- updateTypeProvider(sourceExpressions);
- updateTypeProvider(aggregationTransformExpressions);
-
- return convergeAggregationSource(
- sourceNodeList,
- curStep,
- scanOrder,
- groupByTimeParameter,
- aggregationExpressions,
- groupByLevelExpressions);
- }
-
- public LogicalPlanBuilder planAggregationSourceWithIndexAdjust(
- Set<Expression> sourceExpressions,
- AggregationStep curStep,
- Ordering scanOrder,
- Filter timeFilter,
- GroupByTimeParameter groupByTimeParameter,
- List<Integer> measurementIndexes,
- Set<Expression> aggregationExpressions,
- Set<Expression> aggregationTransformExpressions,
- Map<Expression, Set<Expression>> groupByLevelExpressions) {
- checkArgument(
- sourceExpressions.size() == measurementIndexes.size(),
- "Each aggregate should correspond to a column of output.");
-
- boolean needCheckAscending = groupByTimeParameter == null;
- Map<PartialPath, List<AggregationDescriptor>> ascendingAggregations = new HashMap<>();
- Map<PartialPath, List<AggregationDescriptor>> descendingAggregations = new HashMap<>();
- Map<AggregationDescriptor, Integer> aggregationToMeasurementIndexMap = new HashMap<>();
-
- int index = 0;
- for (Expression sourceExpression : sourceExpressions) {
- AggregationDescriptor aggregationDescriptor =
- createAggregationDescriptor(
- (FunctionExpression) sourceExpression,
- curStep,
- scanOrder,
- needCheckAscending,
- ascendingAggregations,
- descendingAggregations);
- aggregationToMeasurementIndexMap.put(aggregationDescriptor, measurementIndexes.get(index));
- index++;
- }
-
- List<PlanNode> sourceNodeList =
- constructSourceNodeFromAggregationDescriptors(
- ascendingAggregations,
- descendingAggregations,
- scanOrder,
- timeFilter,
- groupByTimeParameter);
- updateTypeProvider(sourceExpressions);
- updateTypeProvider(aggregationTransformExpressions);
-
- if (!curStep.isOutputPartial()) {
- // update measurementIndexes
- measurementIndexes.clear();
- measurementIndexes.addAll(
- sourceNodeList.stream()
- .map(
- planNode ->
- ((SeriesAggregationSourceNode) planNode).getAggregationDescriptorList())
- .flatMap(List::stream)
- .map(aggregationToMeasurementIndexMap::get)
- .collect(Collectors.toList()));
- }
+ updateTypeProvider(aggregationExpressions);
return convergeAggregationSource(
sourceNodeList,
@@ -305,7 +236,7 @@ public class LogicalPlanBuilder {
groupByLevelExpressions);
}
- private AggregationDescriptor createAggregationDescriptor(
+ private void createAggregationDescriptor(
FunctionExpression sourceExpression,
AggregationStep curStep,
Ordering scanOrder,
@@ -331,7 +262,6 @@ public class LogicalPlanBuilder {
.computeIfAbsent(selectPath, key -> new ArrayList<>())
.add(aggregationDescriptor);
}
- return aggregationDescriptor;
}
private List<PlanNode> constructSourceNodeFromAggregationDescriptors(
@@ -458,11 +388,13 @@ public class LogicalPlanBuilder {
public LogicalPlanBuilder planDeviceView(
Map<String, PlanNode> deviceNameToSourceNodesMap,
- Set<Expression> deviceViewOutput,
+ Set<Expression> deviceViewOutputExpressions,
Map<String, List<Integer>> deviceToMeasurementIndexesMap,
Ordering mergeOrder) {
List<String> outputColumnNames =
- deviceViewOutput.stream().map(Expression::getExpressionString).collect(Collectors.toList());
+ deviceViewOutputExpressions.stream()
+ .map(Expression::getExpressionString)
+ .collect(Collectors.toList());
DeviceViewNode deviceViewNode =
new DeviceViewNode(
context.getQueryId().genPlanNodeId(),
@@ -480,7 +412,7 @@ public class LogicalPlanBuilder {
}
context.getTypeProvider().setType(COLUMN_DEVICE, TSDataType.TEXT);
- updateTypeProvider(deviceViewOutput);
+ updateTypeProvider(deviceViewOutputExpressions);
this.root = deviceViewNode;
return this;
@@ -488,7 +420,6 @@ public class LogicalPlanBuilder {
public LogicalPlanBuilder planGroupByLevel(
Map<Expression, Set<Expression>> groupByLevelExpressions,
- AggregationStep curStep,
GroupByTimeParameter groupByTimeParameter,
Ordering scanOrder) {
if (groupByLevelExpressions == null) {
@@ -499,7 +430,7 @@ public class LogicalPlanBuilder {
createGroupByTLevelNode(
Collections.singletonList(this.getRoot()),
groupByLevelExpressions,
- curStep,
+ AggregationStep.FINAL,
groupByTimeParameter,
scanOrder);
return this;
@@ -641,12 +572,12 @@ public class LogicalPlanBuilder {
}
public LogicalPlanBuilder planFilterAndTransform(
- Expression queryFilter,
+ Expression filterExpression,
Set<Expression> selectExpressions,
boolean isGroupByTime,
ZoneId zoneId,
Ordering scanOrder) {
- if (queryFilter == null || selectExpressions.isEmpty()) {
+ if (filterExpression == null || selectExpressions.isEmpty()) {
return this;
}
@@ -655,7 +586,7 @@ public class LogicalPlanBuilder {
context.getQueryId().genPlanNodeId(),
this.getRoot(),
selectExpressions.toArray(new Expression[0]),
- queryFilter,
+ filterExpression,
isGroupByTime,
zoneId,
scanOrder);
@@ -664,12 +595,9 @@ public class LogicalPlanBuilder {
}
public LogicalPlanBuilder planTransform(
- Set<Expression> transformExpressions,
- boolean isGroupByTime,
- ZoneId zoneId,
- Ordering scanOrder) {
+ Set<Expression> selectExpressions, boolean isGroupByTime, ZoneId zoneId, Ordering scanOrder) {
boolean needTransform = false;
- for (Expression expression : transformExpressions) {
+ for (Expression expression : selectExpressions) {
if (ExpressionAnalyzer.checkIsNeedTransform(expression)) {
needTransform = true;
break;
@@ -683,11 +611,11 @@ public class LogicalPlanBuilder {
new TransformNode(
context.getQueryId().genPlanNodeId(),
this.getRoot(),
- transformExpressions.toArray(new Expression[0]),
+ selectExpressions.toArray(new Expression[0]),
isGroupByTime,
zoneId,
scanOrder);
- updateTypeProvider(transformExpressions);
+ updateTypeProvider(selectExpressions);
return this;
}
@@ -720,6 +648,34 @@ public class LogicalPlanBuilder {
return this;
}
+ public LogicalPlanBuilder planHaving(
+ Expression havingExpression,
+ Set<Expression> selectExpressions,
+ boolean isGroupByTime,
+ ZoneId zoneId,
+ Ordering scanOrder) {
+ if (havingExpression != null) {
+ return planFilterAndTransform(
+ havingExpression, selectExpressions, isGroupByTime, zoneId, scanOrder);
+ } else {
+ return planTransform(selectExpressions, isGroupByTime, zoneId, scanOrder);
+ }
+ }
+
+ public LogicalPlanBuilder planWhereAndSourceTransform(
+ Expression whereExpression,
+ Set<Expression> sourceTransformExpressions,
+ boolean isGroupByTime,
+ ZoneId zoneId,
+ Ordering scanOrder) {
+ if (whereExpression != null) {
+ return planFilterAndTransform(
+ whereExpression, sourceTransformExpressions, isGroupByTime, zoneId, scanOrder);
+ } else {
+ return planTransform(sourceTransformExpressions, isGroupByTime, zoneId, scanOrder);
+ }
+ }
+
/** Meta Query* */
public LogicalPlanBuilder planTimeSeriesSchemaSource(
PartialPath pathPattern,
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
index d3e99cd82f..4e2f0af2d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
@@ -67,7 +67,6 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ActivateTemplate
import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ShowPathsUsingTemplateStatement;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -112,18 +111,12 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
subPlanBuilder.withNewRoot(
visitQueryBody(
queryStatement,
- analysis.getDeviceToIsRawDataSource().get(deviceName),
analysis.getDeviceToSourceExpressions().get(deviceName),
- analysis.getDeviceToAggregationExpressions().get(deviceName),
analysis.getDeviceToSourceTransformExpressions().get(deviceName),
- analysis.hasValueFilter()
- ? analysis.getDeviceToSourceExpressions().get(deviceName)
- : Collections.emptySet(),
analysis.getDeviceToWhereExpression() != null
? analysis.getDeviceToWhereExpression().get(deviceName)
: null,
- null,
- analysis.getDeviceToMeasurementIndexesMap().get(deviceName),
+ analysis.getDeviceToAggregationExpressions().get(deviceName),
context));
deviceToSubPlanMap.put(deviceName, subPlanBuilder.getRoot());
}
@@ -131,42 +124,34 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
planBuilder =
planBuilder.planDeviceView(
deviceToSubPlanMap,
- analysis.getTransformInput(),
+ analysis.getDeviceViewOutputExpressions(),
analysis.getDeviceToMeasurementIndexesMap(),
queryStatement.getResultTimeOrder());
- if (queryStatement.hasHaving()) {
- planBuilder.planFilterAndTransform(
- analysis.getHavingExpression(),
- analysis.getTransformOutput(),
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- planBuilder.planTransform(
- analysis.getTransformOutput(),
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- }
} else {
planBuilder =
planBuilder.withNewRoot(
visitQueryBody(
queryStatement,
- analysis.isRawDataSource(),
analysis.getSourceExpressions(),
- analysis.getAggregationExpressions(),
analysis.getSourceTransformExpressions(),
- analysis.getSelectExpressions(),
analysis.getWhereExpression(),
- analysis.getHavingExpression(),
- null,
+ analysis.getAggregationExpressions(),
context));
}
- // other common upstream node
+ // other upstream node
planBuilder =
planBuilder
+ .planGroupByLevel(
+ analysis.getGroupByLevelExpressions(),
+ analysis.getGroupByTimeParameter(),
+ queryStatement.getResultTimeOrder())
+ .planHaving(
+ analysis.getHavingExpression(),
+ analysis.getSelectExpressions(),
+ queryStatement.isGroupByTime(),
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultTimeOrder())
.planFill(analysis.getFillDescriptor(), queryStatement.getResultTimeOrder())
.planOffset(queryStatement.getRowOffset())
.planLimit(queryStatement.getRowLimit());
@@ -176,48 +161,51 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
public PlanNode visitQueryBody(
QueryStatement queryStatement,
- boolean isRawDataSource,
Set<Expression> sourceExpressions,
+ Set<Expression> sourceTransformExpressions,
+ Expression whereExpression,
Set<Expression> aggregationExpressions,
- Set<Expression> aggregationTransformExpressions,
- Set<Expression> transformExpressions,
- Expression queryFilter,
- Expression havingExpression,
- List<Integer> measurementIndexes, // only used in ALIGN BY DEVICE
MPPQueryContext context) {
LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context);
- // plan data source node
- if (isRawDataSource) {
+ if (aggregationExpressions == null) {
+ // raw data query
planBuilder =
- planBuilder.planRawDataSource(
- sourceExpressions,
- queryStatement.getResultTimeOrder(),
- analysis.getGlobalTimeFilter());
-
- if (queryStatement.isAggregationQuery()) {
- if (analysis.hasValueFilter()) {
- planBuilder =
- planBuilder.planFilterAndTransform(
- queryFilter,
- aggregationTransformExpressions,
+ planBuilder
+ .planRawDataSource(
+ sourceExpressions,
+ queryStatement.getResultTimeOrder(),
+ analysis.getGlobalTimeFilter())
+ .planWhereAndSourceTransform(
+ whereExpression,
+ sourceTransformExpressions,
queryStatement.isGroupByTime(),
queryStatement.getSelectComponent().getZoneId(),
queryStatement.getResultTimeOrder());
- } else {
- planBuilder =
- planBuilder.planTransform(
- aggregationTransformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- }
+ } else {
+ // aggregation query
+ boolean isRawDataSource =
+ (whereExpression != null) || needTransform(sourceTransformExpressions);
+ AggregationStep curStep;
+ if (isRawDataSource) {
+ planBuilder =
+ planBuilder
+ .planRawDataSource(
+ sourceExpressions,
+ queryStatement.getResultTimeOrder(),
+ analysis.getGlobalTimeFilter())
+ .planWhereAndSourceTransform(
+ whereExpression,
+ sourceTransformExpressions,
+ queryStatement.isGroupByTime(),
+ queryStatement.getSelectComponent().getZoneId(),
+ queryStatement.getResultTimeOrder());
boolean outputPartial =
queryStatement.isGroupByLevel()
|| (queryStatement.isGroupByTime()
&& analysis.getGroupByTimeParameter().hasOverlap());
- AggregationStep curStep = outputPartial ? AggregationStep.PARTIAL : AggregationStep.SINGLE;
+ curStep = outputPartial ? AggregationStep.PARTIAL : AggregationStep.SINGLE;
planBuilder =
planBuilder.planAggregation(
aggregationExpressions,
@@ -225,147 +213,57 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
curStep,
queryStatement.getResultTimeOrder());
- if (curStep.isOutputPartial()) {
- if (queryStatement.isGroupByTime() && analysis.getGroupByTimeParameter().hasOverlap()) {
- curStep =
- queryStatement.isGroupByLevel()
- ? AggregationStep.INTERMEDIATE
- : AggregationStep.FINAL;
- planBuilder =
- planBuilder.planSlidingWindowAggregation(
- aggregationExpressions,
- analysis.getGroupByTimeParameter(),
- curStep,
- queryStatement.getResultTimeOrder());
- }
-
- if (queryStatement.isGroupByLevel()) {
- curStep = AggregationStep.FINAL;
- planBuilder =
- planBuilder.planGroupByLevel(
- analysis.getGroupByLevelExpressions(),
- curStep,
- analysis.getGroupByTimeParameter(),
- queryStatement.getResultTimeOrder());
- }
+ if (queryStatement.isGroupByTime() && analysis.getGroupByTimeParameter().hasOverlap()) {
+ curStep =
+ queryStatement.isGroupByLevel()
+ ? AggregationStep.INTERMEDIATE
+ : AggregationStep.FINAL;
+ planBuilder =
+ planBuilder.planSlidingWindowAggregation(
+ aggregationExpressions,
+ analysis.getGroupByTimeParameter(),
+ curStep,
+ queryStatement.getResultTimeOrder());
}
if (queryStatement.isGroupByLevel()) {
- planBuilder = // plan Having with GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- analysis.getGroupByLevelExpressions().keySet(),
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- if (havingExpression != null) {
- planBuilder = // plan Having without GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- planBuilder = // no Having
- planBuilder.planTransform(
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- }
- }
- } else {
- if (analysis.hasValueFilter()) {
planBuilder =
- planBuilder.planFilterAndTransform(
- queryFilter,
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- planBuilder =
- planBuilder.planTransform(
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
+ planBuilder.planGroupByLevel(
+ analysis.getGroupByLevelExpressions(),
+ analysis.getGroupByTimeParameter(),
queryStatement.getResultTimeOrder());
}
- }
- } else {
- AggregationStep curStep =
- (analysis.getGroupByLevelExpressions() != null
- || (analysis.getGroupByTimeParameter() != null
- && analysis.getGroupByTimeParameter().hasOverlap()))
- ? AggregationStep.PARTIAL
- : AggregationStep.SINGLE;
-
- boolean needTransform = false;
- for (Expression expression : transformExpressions) {
- if (ExpressionAnalyzer.checkIsNeedTransform(expression)) {
- needTransform = true;
- break;
- }
- }
-
- if (!needTransform && measurementIndexes != null) {
- planBuilder =
- planBuilder.planAggregationSourceWithIndexAdjust(
- sourceExpressions,
- curStep,
- queryStatement.getResultTimeOrder(),
- analysis.getGlobalTimeFilter(),
- analysis.getGroupByTimeParameter(),
- measurementIndexes,
- aggregationExpressions,
- aggregationTransformExpressions,
- analysis.getGroupByLevelExpressions());
} else {
+ curStep =
+ (analysis.getGroupByLevelExpressions() != null
+ || (analysis.getGroupByTimeParameter() != null
+ && analysis.getGroupByTimeParameter().hasOverlap()))
+ ? AggregationStep.PARTIAL
+ : AggregationStep.SINGLE;
+
planBuilder =
planBuilder.planAggregationSource(
- sourceExpressions,
curStep,
queryStatement.getResultTimeOrder(),
analysis.getGlobalTimeFilter(),
analysis.getGroupByTimeParameter(),
aggregationExpressions,
- aggregationTransformExpressions,
analysis.getGroupByLevelExpressions());
-
- if (queryStatement.isGroupByLevel()) {
- planBuilder = // plan Having with GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- analysis.getGroupByLevelExpressions().keySet(),
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- if (havingExpression != null) {
- planBuilder = // plan Having without GroupByLevel
- planBuilder.planFilterAndTransform(
- havingExpression,
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- } else {
- planBuilder = // no Having
- planBuilder.planTransform(
- transformExpressions,
- queryStatement.isGroupByTime(),
- queryStatement.getSelectComponent().getZoneId(),
- queryStatement.getResultTimeOrder());
- }
- }
}
}
return planBuilder.getRoot();
}
+ private boolean needTransform(Set<Expression> expressions) {
+ for (Expression expression : expressions) {
+ if (ExpressionAnalyzer.checkIsNeedTransform(expression)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public PlanNode visitCreateTimeseries(
CreateTimeSeriesStatement createTimeSeriesStatement, MPPQueryContext context) {
[iotdb] 04/04: add AnalyzeTest
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/RefactorAnalyzer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7bed3966034a0b10f1645cac46d86d483b20bc31
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Sep 26 16:32:00 2022 +0800
add AnalyzeTest
---
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 90 +++++++++++++++++++++-
1 file changed, 88 insertions(+), 2 deletions(-)
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
index d081b194f3..061ef9bb19 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
@@ -23,7 +23,10 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.mpp.common.MPPQueryContext;
import org.apache.iotdb.db.mpp.common.QueryId;
+import org.apache.iotdb.db.mpp.common.header.ColumnHeader;
+import org.apache.iotdb.db.mpp.common.header.DatasetHeader;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
+import org.apache.iotdb.db.mpp.plan.expression.binary.AdditionExpression;
import org.apache.iotdb.db.mpp.plan.expression.binary.GreaterThanExpression;
import org.apache.iotdb.db.mpp.plan.expression.binary.LessThanExpression;
import org.apache.iotdb.db.mpp.plan.expression.binary.LogicAndExpression;
@@ -36,18 +39,57 @@ import org.apache.iotdb.db.mpp.plan.statement.Statement;
import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.ratis.thirdparty.com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.Test;
import java.time.ZonedDateTime;
+import java.util.Arrays;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class AnalyzeTest {
@Test
public void testRawDataQuery() {
- // TODO: @lmh add UTs
+ String sql = "select s1, status, s1 + 1 as t from root.sg.d1 where s2 > 10;";
+ try {
+ Analysis actualAnalysis = analyzeSQL(sql);
+
+ Analysis expectedAnalysis = new Analysis();
+ expectedAnalysis.setSelectExpressions(
+ Sets.newHashSet(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
+ new AdditionExpression(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
+ new ConstantOperand(TSDataType.INT64, "1"))));
+ expectedAnalysis.setWhereExpression(
+ new GreaterThanExpression(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s2")),
+ new ConstantOperand(TSDataType.INT64, "10")));
+ expectedAnalysis.setSourceTransformExpressions(
+ Sets.newHashSet(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
+ new AdditionExpression(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
+ new ConstantOperand(TSDataType.INT64, "1"))));
+ expectedAnalysis.setSourceExpressions(
+ Sets.newHashSet(
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s1")),
+ new TimeSeriesOperand(new PartialPath("root.sg.d1.s2"))));
+ expectedAnalysis.setRespDatasetHeader(
+ new DatasetHeader(
+ Arrays.asList(
+ new ColumnHeader("root.sg.d1.s1", TSDataType.INT32),
+ new ColumnHeader("root.sg.d1.s1", TSDataType.INT32, "status"),
+ new ColumnHeader("root.sg.d1.s1 + 1", TSDataType.INT32, "t")),
+ false));
+ alignByTimeAnalysisEqualTest(actualAnalysis, expectedAnalysis);
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
}
@Test
@@ -135,7 +177,7 @@ public class AnalyzeTest {
if (predicates[i] == null) {
Assert.assertNull(queryStatement.getWhereCondition());
} else {
- Assert.assertEquals(predicates[i], queryStatement.getWhereCondition().getPredicate());
+ assertEquals(predicates[i], queryStatement.getWhereCondition().getPredicate());
}
}
}
@@ -155,4 +197,48 @@ public class AnalyzeTest {
fail();
return null;
}
+
+ private void alignByTimeAnalysisEqualTest(Analysis actualAnalysis, Analysis expectedAnalysis) {
+ commonAnalysisEqualTest(actualAnalysis, expectedAnalysis);
+ assertEquals(actualAnalysis.getSourceExpressions(), expectedAnalysis.getSourceExpressions());
+ assertEquals(
+ actualAnalysis.getSourceTransformExpressions(),
+ expectedAnalysis.getSourceTransformExpressions());
+ assertEquals(actualAnalysis.getWhereExpression(), expectedAnalysis.getWhereExpression());
+ assertEquals(
+ actualAnalysis.getAggregationExpressions(), expectedAnalysis.getAggregationExpressions());
+ assertEquals(
+ actualAnalysis.getGroupByLevelExpressions(), expectedAnalysis.getGroupByLevelExpressions());
+ }
+
+ private void alignByDeviceAnalysisEqualTest(Analysis actualAnalysis, Analysis expectedAnalysis) {
+ commonAnalysisEqualTest(actualAnalysis, expectedAnalysis);
+ assertEquals(
+ actualAnalysis.getDeviceToSourceExpressions(),
+ expectedAnalysis.getDeviceToSourceExpressions());
+ assertEquals(
+ actualAnalysis.getDeviceToSourceTransformExpressions(),
+ expectedAnalysis.getDeviceToSourceTransformExpressions());
+ assertEquals(
+ actualAnalysis.getDeviceToWhereExpression(), expectedAnalysis.getDeviceToWhereExpression());
+ assertEquals(
+ actualAnalysis.getDeviceToAggregationExpressions(),
+ expectedAnalysis.getDeviceToWhereExpression());
+ assertEquals(
+ actualAnalysis.getDeviceToSelectExpressions(),
+ expectedAnalysis.getDeviceToSelectExpressions());
+ assertEquals(
+ actualAnalysis.getDeviceToMeasurementIndexesMap(),
+ expectedAnalysis.getDeviceToMeasurementIndexesMap());
+ assertEquals(
+ actualAnalysis.getDeviceViewOutputExpressions(),
+ actualAnalysis.getDeviceViewOutputExpressions());
+ }
+
+ private void commonAnalysisEqualTest(Analysis actualAnalysis, Analysis expectedAnalysis) {
+ assertEquals(expectedAnalysis.getSelectExpressions(), actualAnalysis.getSelectExpressions());
+ assertEquals(expectedAnalysis.getHavingExpression(), actualAnalysis.getHavingExpression());
+ assertEquals(expectedAnalysis.getRespDatasetHeader(), actualAnalysis.getRespDatasetHeader());
+ assertEquals(expectedAnalysis.getGlobalTimeFilter(), actualAnalysis.getGlobalTimeFilter());
+ }
}