You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/05/15 02:57:46 UTC
[iotdb] branch native_raft updated (1de73174fea -> 0b81ea470a7)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 1de73174fea refactor dispatcher sleep
add aaf7fa2804f [IOTDB-5811] Pipe: PipeHistoricalCollector (#9700)
add 6118e6cec6b [IOTDB-5837] Fix exception in select-into of ALIGN BY DEVICE query
add b5d13d1876b Delete abandoned API in document (#9773)
add 109b15434c3 Use two params to distinguish fsync delay in the wal sync mode and async mode (#9707)
add 6160095863c [IOTDB-5841] Modify IoTConsensus default parameters to improve performance in more scenarios (#9771)
add e39060e1d1b update ratis version to 2.5.1 (#9785)
add 299f33c80ca [IOTDB-5821] Pipe: PipeCollector Stage (#9789)
add db8849fce19 [IOTDB-5740] Fix drop database does not delete its template (#9792)
add f579e3ca0d6 Support Create and Query Schema of Logical View (#9742)
add 3b44a55bb20 [Doc] Fix error path of Dockerfile (#9725)
add 16011faf943 [To rel/1.1] [IOTDB-5844] Fix compaction module getting stuck (#9776) (#9790)
add dde59e204e4 Refactor SchemaIT for Acceleration (#9794)
add 5c20ee49d39 [IOTDB-5831] Fix create region failure after recreate db (#9800)
add f3e7ce22864 [IOTDB-5818][Atmos][Compaction]Cross_space compaction of Aligned timeseries is stucked (#9795)
add c84a66e669f [IOTDB-5827] Fix the description for dn_multi_dir_strategy in iotdb-datanode.properties and update corresponding user manual (#9804)
add 15a3adc2eb2 [IOTDB-5848] Pipe: task construction and life cycle management (#9808)
add a52beb4e996 [IOTDB-5851] Fix Npe with limit clause
add 7533fe5bec6 Fix a bug: crash while restarting caused by logical view (#9815)
add 52eb858616e Follower doesn't need to update last cache when using IoT_consensus (#9811)
add 8346ab743e6 Support FIFO policy for DataNodeSchemaCache Eviction (#9809)
add 921faa061be [IOTDB-5858] Remove useless class cast and update schema cache hit
add 115fc115f94 [IOTDB-5843] Stall write requests when system shutting down (#9819)
add a99aeff5410 [IOTDB-5863]add configNode configuration information print (#9807)
add b938948bc71 Fix only update last cache on follower (#9827)
add f8a41d74c2b Refactor compaction task metrics (#9816)
add b48bc0a4858 [IOTDB-5858] fix the clear of metric in datanode schema cache
add d77c3994208 [IOTDB-5719] Cluster-Wide Time Series Limit Control (#9820)
add 1c2251394bf Use interface for MeasurementSchema and Move viewExpression to node commons (#9826)
add 467024bca30 [IOTDB-5865] Fix route balancing not trigger bug (#9793)
add 568aabe942e [IOTDB-5854] Pipe: Tsfile Resource Manager (#9805)
add 52324d4ec41 [IOTDB-5845] Pipe: Support handling leader change in PipeRuntimeCoordinator (#9777)
add 34fb7f532f6 [IOTDB-5799] Pipe: Support `SHOW PIPES` clause (#9838)
add 888fe13f13d build device timeIndex when validating files after compaction (#9831)
add af2d050b54b [IOTDB-5870] Pipe: Event reference management (#9836)
add 10d4eba1943 [IOTDB-5846] Optimize the memory estimate for HorizontallyConcatOperator (#9781)
add 0940f2a765f bump ratis version to 2.5.2-snapshot (#9839)
add e8f4897d741 [IOTDB-5872] Reduce some IoTConsensus index logs (#9844)
add 803c87fcdb0 [IOTDB-5873] Pipe: Support `CREATE`, `DROP`, `START`, `STOP` & `SHOW` Clauses (#9849)
add 1b1b95172f0 [IOTDB-5852] Add some IoTConsensus and RatisConsensus properties (#9797)
new 0b81ea470a7 Merge branch 'master' into native_raft
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 22 ++
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
client-py/tests/test_dataframe.py | 2 +
.../heartbeat/DataNodeHeartbeatHandler.java | 25 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 49 +++
.../confignode/conf/ConfigNodeDescriptor.java | 16 +
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 4 +
.../request/read/pipe/task/ShowPipePlanV2.java | 23 +-
.../coordinator/PipeHandleLeaderChangePlan.java | 74 ++++
.../response/pipe/task/PipeTableResp.java | 123 +++++++
.../statemachine/ConfigRegionStateMachine.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 22 ++
.../iotdb/confignode/manager/load/LoadManager.java | 2 +
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../confignode/manager/load/cache/LoadCache.java | 11 +-
.../manager/load/cache/route/RegionRouteCache.java | 4 +
.../manager/load/service/HeartbeatService.java | 4 +-
.../manager/load/service/StatisticsService.java | 9 +-
.../iotdb/confignode/manager/node/NodeManager.java | 5 +-
.../manager/partition/PartitionManager.java | 17 +-
.../manager/partition/PartitionMetrics.java | 2 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 7 +
.../manager/pipe/PipeRuntimeCoordinator.java | 82 +++++
.../manager/pipe/PipeTaskCoordinator.java | 32 +-
.../manager/{ => schema}/ClusterSchemaManager.java | 35 +-
.../schema/ClusterSchemaQuotaStatistics.java | 33 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 13 +
.../persistence/pipe/PipePluginInfo.java | 49 ++-
.../confignode/persistence/pipe/PipeTaskInfo.java | 56 +++
.../persistence/pipe/PipeTaskOperation.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 196 +++++++++++
.../pipe/task/AbstractOperatePipeProcedureV2.java | 28 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 27 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 12 +-
.../impl/schema/DeleteDatabaseProcedure.java | 8 +-
.../procedure/store/ProcedureFactory.java | 14 +
.../confignode/procedure/store/ProcedureType.java | 11 +-
.../iotdb/confignode/service/ConfigNode.java | 6 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 18 +
.../manager/ClusterSchemaManagerTest.java | 2 +
.../PipeHandleLeaderChangeProcedureTest.java | 63 ++++
consensus/pom.xml | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 36 +-
.../consensus/iot/IoTConsensusServerImpl.java | 2 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 10 +-
.../logdispatcher/LogDispatcherThreadMetrics.java | 22 ++
.../consensus/iot/logdispatcher/SyncStatus.java | 10 +-
.../ratis/ApplicationStateMachineProxy.java | 10 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 3 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 17 +
.../iot/logdispatcher/SyncStatusTest.java | 10 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 98 ------
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 8 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 85 ++++-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../UserGuide/API/Programming-Java-Native-API.md | 93 -----
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 24 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 87 ++++-
.../UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../iotdb/it/env/cluster/MppCommonConfig.java | 12 +
.../it/env/cluster/MppSharedCommonConfig.java | 14 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 10 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +
.../db/it/schema/IoTDBAutoCreateSchemaIT.java | 17 +-
.../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 +++
.../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 +++++++
.../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBCreateStorageGroupIT.java | 18 +-
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 26 +-
.../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 18 +-
.../db/it/schema/IoTDBDeleteStorageGroupIT.java | 17 +-
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 18 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 49 +--
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 63 ++--
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 49 +--
.../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 20 +-
.../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 17 +-
.../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 17 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 30 ++
.../org/apache/iotdb/util/AbstractSchemaIT.java | 48 ++-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 14 +-
.../metrics/metricsets/jvm/JvmMemoryMetrics.java | 6 +-
.../resources/conf/iotdb-common.properties | 55 ++-
.../apache/iotdb/commons/path/MeasurementPath.java | 31 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +-
.../plugin/builtin/collector/DefaultCollector.java | 60 ++++
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 9 +-
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 27 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 92 +++--
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 59 +++-
.../commons/schema/ClusterSchemaQuotaLevel.java | 19 +-
.../node/common/AbstractMeasurementMNode.java | 4 +
.../schema/node/role/IMeasurementMNode.java | 2 +
.../commons/schema/node/utils/IMNodeFactory.java | 4 +
.../commons/schema/view/LogicalViewSchema.java | 224 ++++++++++++
.../schema/view/viewExpression/ViewExpression.java | 318 +++++++++++++++++
.../view/viewExpression/ViewExpressionType.java | 76 ++++
.../binary/BinaryViewExpression.java | 119 +++++++
.../binary/arithmetic/AdditionViewExpression.java | 61 ++++
.../arithmetic/ArithmeticBinaryViewExpression.java | 52 +++
.../binary/arithmetic/DivisionViewExpression.java | 61 ++++
.../binary/arithmetic/ModuloViewExpression.java | 61 ++++
.../arithmetic/MultiplicationViewExpression.java | 63 ++++
.../arithmetic/SubtractionViewExpression.java | 61 ++++
.../compare/CompareBinaryViewExpression.java | 52 +++
.../binary/compare/EqualToViewExpression.java | 57 +++
.../binary/compare/GreaterEqualViewExpression.java | 57 +++
.../binary/compare/GreaterThanViewExpression.java | 57 +++
.../binary/compare/LessEqualViewExpression.java | 57 +++
.../binary/compare/LessThanViewExpression.java | 57 +++
.../binary/compare/NonEqualViewExpression.java | 57 +++
.../binary/logic/LogicAndViewExpression.java | 57 +++
.../binary/logic/LogicBinaryViewExpression.java | 52 +++
.../binary/logic/LogicOrViewExpression.java | 57 +++
.../viewExpression/leaf/ConstantViewOperand.java | 98 ++++++
.../view/viewExpression/leaf/LeafViewOperand.java | 20 +-
.../view/viewExpression/leaf/NullViewOperand.java | 61 ++++
.../viewExpression/leaf/TimeSeriesViewOperand.java | 87 +++++
.../viewExpression/leaf/TimestampViewOperand.java | 72 ++++
.../multi/FunctionViewExpression.java | 198 +++++++++++
.../ternary/BetweenViewExpression.java | 111 ++++++
.../ternary/TernaryViewExpression.java | 105 ++++++
.../viewExpression/unary/InViewExpression.java | 106 ++++++
.../viewExpression/unary/IsNullViewExpression.java | 89 +++++
.../viewExpression/unary/LikeViewExpression.java | 163 +++++++++
.../unary/LogicNotViewExpression.java | 61 ++++
.../unary/NegationViewExpression.java | 61 ++++
.../unary/RegularViewExpression.java | 105 ++++++
.../viewExpression/unary/UnaryViewExpression.java | 73 ++++
.../visitor/ViewExpressionVisitor.java | 219 ++++++++++++
.../commons/pipe/task/meta/PipeMetaDeSerTest.java | 77 ++++
.../org/apache/iotdb/pipe/api/PipeCollector.java | 1 +
.../api/exception/PipeConnectionException.java | 18 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 22 +-
.../rocksdb/mnode/RMeasurementMNode.java | 5 +
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 22 +-
.../resources/conf/iotdb-datanode.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 133 ++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 92 ++++-
.../db/consensus/DataRegionConsensusImpl.java | 9 +
.../statemachine/DataRegionStateMachine.java | 3 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 7 +-
.../compaction/constant/CompactionTaskStatus.java} | 9 +-
.../compaction/constant/CompactionTaskType.java} | 8 +-
.../CompactionMemoryNotEnoughException.java} | 9 +-
.../execute/task/AbstractCompactionTask.java | 1 -
.../execute/task/CrossSpaceCompactionTask.java | 6 +-
.../compaction/execute/utils/CompactionUtils.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 70 +++-
.../compaction/schedule/CompactionWorker.java | 3 -
.../estimator/AbstractCompactionEstimator.java | 3 +-
.../ReadPointCrossCompactionEstimator.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 37 +-
.../db/engine/storagegroup/TsFileResource.java | 5 +
.../metadata/SchemaQuotaExceededException.java} | 38 +-
.../apache/iotdb/db/metadata/MetadataConstant.java | 6 +
.../db/metadata/cache/DataNodeSchemaCache.java | 14 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 31 +-
.../cache/DeviceUsingTemplateSchemaCache.java | 18 +-
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 +-
.../db/metadata/cache/TimeSeriesSchemaCache.java | 9 +-
.../dualkeycache/impl/DualKeyCacheBuilder.java | 18 +-
.../dualkeycache/impl/DualKeyCachePolicy.java | 3 +-
.../dualkeycache/impl/FIFOCacheEntryManager.java | 189 ++++++++++
.../mnode/config/factory/ConfigMNodeFactory.java | 7 +
.../mnode/mem/factory/MemMNodeFactory.java | 14 +
...MeasurementMNode.java => LogicalViewMNode.java} | 31 +-
.../metadata/mnode/mem/impl/MeasurementMNode.java | 5 +
.../metadata/mnode/mem/info/LogicalViewInfo.java | 163 +++++++++
.../schemafile/factory/CacheMNodeFactory.java | 7 +
.../schemafile/impl/CachedMeasurementMNode.java | 5 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 31 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 94 ++++-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 48 ++-
.../db/metadata/mtree/store/CachedMTreeStore.java | 3 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +
.../plan/schemaregion/SchemaRegionPlanType.java | 2 +
.../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 +
.../impl/SchemaRegionPlanDeserializer.java | 24 ++
.../impl/SchemaRegionPlanSerializer.java | 27 ++
.../impl/SchemaRegionPlanTxtSerializer.java | 21 ++
.../impl/write/CreateLogicalViewPlanImpl.java | 75 ++++
.../schemaregion/result/ShowTimeSeriesResult.java | 13 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 66 ++++
.../metadata/query/info/ITimeSeriesSchemaInfo.java | 6 +-
.../rescon/DataNodeSchemaQuotaManager.java | 92 +++++
.../metadata/rescon/MemSchemaRegionStatistics.java | 8 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 21 +-
.../db/metadata/schemaregion/SchemaEngine.java | 92 ++++-
.../schemaregion/SchemaRegionMemoryImpl.java | 121 ++++---
.../schemaregion/SchemaRegionSchemaFileImpl.java | 78 ++--
.../metadata/template/ClusterTemplateManager.java | 36 ++
.../visitor/GetSourcePathsVisitor.java | 96 +++++
.../visitor/TransformToExpressionVisitor.java | 321 +++++++++++++++++
.../metadata/visitor/SchemaExecutionVisitor.java | 22 ++
.../db/mpp/common/header/ColumnHeaderConstant.java | 6 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 6 +-
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +-
.../common/schematree/IMeasurementSchemaInfo.java | 8 +-
.../common/schematree/MeasurementSchemaInfo.java | 22 +-
.../schematree/node/SchemaMeasurementNode.java | 42 ++-
.../mpp/execution/exchange/SharedTsBlockQueue.java | 12 +-
.../execution/executor/RegionWriteExecutor.java | 133 ++++++-
.../operator/process/DeviceViewIntoOperator.java | 6 +-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../schema/source/TimeSeriesSchemaSource.java | 14 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 49 +++
.../analyze/schema/ClusterSchemaFetchExecutor.java | 5 +-
.../config/executor/ClusterConfigTaskExecutor.java | 5 +-
.../visitor/TransformToViewExpressionVisitor.java | 391 +++++++++++++++++++++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 98 +++++-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 21 ++
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../node/metedata/write/CreateLogicalViewNode.java | 250 +++++++++++++
.../write/InternalCreateMultiTimeSeriesNode.java | 1 +
.../plan/planner/plan/node/write/InsertNode.java | 9 +
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 2 +-
.../iotdb/db/mpp/plan/statement/StatementType.java | 2 +
.../db/mpp/plan/statement/StatementVisitor.java | 7 +
.../metadata/CreateLogicalViewStatement.java | 246 +++++++++++++
.../db/pipe/agent/plugin/PipePluginAgent.java | 26 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 349 ++++++++++++++----
.../PipeCollectorConstant.java} | 21 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 6 +
.../config/PipeConnectorConstant.java} | 14 +-
.../config/PipeProcessorConstant.java} | 14 +-
.../core/collector/IoTDBDataRegionCollector.java | 85 +++++
.../PipeHistoricalDataRegionTsFileCollector.java | 103 ++++++
.../realtime/PipeRealtimeDataRegionCollector.java | 23 +-
... => PipeRealtimeDataRegionHybridCollector.java} | 64 +++-
.../realtime/assigner/PipeDataRegionAssigner.java | 20 +-
.../matcher/CachedSchemaPatternMatcher.java | 6 +-
.../realtime/matcher/PipeDataRegionMatcher.java | 9 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 31 +-
.../connector/PipeConnectorSubtaskManager.java | 42 ++-
.../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 +++
.../core/event/impl/PipeTabletInsertionEvent.java | 27 +-
.../core/event/impl/PipeTsFileInsertionEvent.java | 47 ++-
.../event/realtime/PipeRealtimeCollectEvent.java | 20 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 2 -
.../event/view/collector/PipeEventCollector.java | 65 +++-
...anager.java => PipeSubtaskExecutorManager.java} | 12 +-
.../execution/scheduler/PipeTaskScheduler.java | 18 +-
.../db/pipe/resource/PipeFileResourceManager.java | 186 ++++++++++
.../db/pipe/resource/PipeResourceManager.java | 16 +-
.../iotdb/db/pipe/resource/PipeTsFileHolder.java | 22 --
.../iotdb/db/pipe/resource/PipeWALHolder.java | 22 --
.../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 66 ++++
.../org/apache/iotdb/db/pipe/task/PipeTask.java | 9 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 60 +++-
.../apache/iotdb/db/pipe/task/PipeTaskManager.java | 96 +++++
.../EventSupplier.java} | 24 +-
.../ListenableBlockingPendingQueue.java} | 18 +-
.../db/pipe/task/queue/ListenablePendingQueue.java | 159 +++++++++
.../ListenableUnblockingPendingQueue.java} | 18 +-
.../PendingQueueEmptyToNotEmptyListener.java} | 8 +-
.../queue/PendingQueueFullToNotFullListener.java} | 8 +-
.../queue/PendingQueueNotEmptyToEmptyListener.java | 12 +-
.../queue/PendingQueueNotFullToFullListener.java} | 8 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 98 ++++--
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 100 +-----
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 131 +++++--
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 95 ++++-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 107 ++++--
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 42 ++-
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 6 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 20 +-
.../metrics/recorder/CompactionMetricsManager.java | 81 +++--
.../impl/DataNodeInternalRPCServiceImpl.java | 18 +-
.../datastructure/FixedPriorityBlockingQueue.java | 10 +
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 9 +-
.../engine/compaction/AbstractCompactionTest.java | 3 +-
.../FastCrossCompactionPerformerTest.java | 5 +
...eCompactionWithFastPerformerValidationTest.java | 16 +-
.../cache/dualkeycache/DualKeyCacheTest.java | 17 +
.../metadata/view/ViewExpressionToStringTest.java | 183 ++++++++++
.../mpp/execution/operator/OperatorMemoryTest.java | 34 ++
.../schema/SchemaFetchScanOperatorTest.java | 3 +-
.../collector/CachedSchemaPatternMatcherTest.java | 54 ++-
.../core/collector/PipeRealtimeCollectTest.java | 104 +++++-
.../executor/PipeAssignerSubtaskExecutorTest.java | 40 ---
.../executor/PipeConnectorSubtaskExecutorTest.java | 8 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 10 +-
.../pipe/resource/PipeFileResourceManagerTest.java | 218 ++++++++++++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../src/main/thrift/confignode.thrift | 5 +-
thrift/src/main/thrift/datanode.thrift | 18 +-
.../tsfile/write/schema/IMeasurementSchema.java | 4 +
.../tsfile/write/schema/MeasurementSchema.java | 10 +
.../tsfile/write/schema/MeasurementSchemaType.java | 23 +-
.../write/schema/VectorMeasurementSchema.java | 10 +
306 files changed, 11561 insertions(+), 1609 deletions(-)
copy server/src/main/java/org/apache/iotdb/db/pipe/task/subtask/PipeAssignerSubtask.java => confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java (56%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/coordinator/PipeHandleLeaderChangePlan.java
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/PipeRuntimeCoordinator.java
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{ => schema}/ClusterSchemaManager.java (96%)
copy server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeResourceManager.java => confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaQuotaStatistics.java (52%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java
create mode 100644 confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBClusterDeviceQuotaIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBClusterMeasurementQuotaIT.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/collector/DefaultCollector.java
copy server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/IMeasurementSchemaInfo.java => node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java (76%)
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/LogicalViewSchema.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ViewExpressionType.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/BinaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/AdditionViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/ArithmeticBinaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/DivisionViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/ModuloViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/MultiplicationViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/arithmetic/SubtractionViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/CompareBinaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/EqualToViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/GreaterEqualViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/GreaterThanViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/LessEqualViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/LessThanViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/compare/NonEqualViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/logic/LogicAndViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/logic/LogicBinaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/binary/logic/LogicOrViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/leaf/ConstantViewOperand.java
copy server/src/main/java/org/apache/iotdb/db/pipe/task/subtask/PipeAssignerSubtask.java => node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/leaf/LeafViewOperand.java (65%)
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/leaf/NullViewOperand.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/leaf/TimeSeriesViewOperand.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/leaf/TimestampViewOperand.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/multi/FunctionViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ternary/BetweenViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/ternary/TernaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/InViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/IsNullViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/LikeViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/LogicNotViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/NegationViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/RegularViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/unary/UnaryViewExpression.java
create mode 100644 node-commons/src/main/java/org/apache/iotdb/commons/schema/view/viewExpression/visitor/ViewExpressionVisitor.java
create mode 100644 node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/meta/PipeMetaDeSerTest.java
copy server/src/main/java/org/apache/iotdb/db/pipe/task/subtask/PipeAssignerSubtask.java => pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeConnectionException.java (74%)
copy server/src/main/java/org/apache/iotdb/db/{metadata/cache/dualkeycache/impl/DualKeyCachePolicy.java => engine/compaction/constant/CompactionTaskStatus.java} (85%)
copy server/src/main/java/org/apache/iotdb/db/{metadata/cache/dualkeycache/impl/DualKeyCachePolicy.java => engine/compaction/constant/CompactionTaskType.java} (86%)
copy server/src/main/java/org/apache/iotdb/db/{metadata/cache/dualkeycache/impl/DualKeyCachePolicy.java => engine/compaction/execute/exception/CompactionMemoryNotEnoughException.java} (78%)
copy server/src/main/java/org/apache/iotdb/db/{pipe/resource/PipeResourceManager.java => exception/metadata/SchemaQuotaExceededException.java} (52%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/cache/dualkeycache/impl/FIFOCacheEntryManager.java
copy server/src/main/java/org/apache/iotdb/db/metadata/mnode/mem/impl/{MeasurementMNode.java => LogicalViewMNode.java} (65%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mnode/mem/info/LogicalViewInfo.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/plan/schemaregion/impl/write/CreateLogicalViewPlanImpl.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/plan/schemaregion/write/ICreateLogicalViewPlan.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/rescon/DataNodeSchemaQuotaManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/view/viewExpression/visitor/GetSourcePathsVisitor.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/view/viewExpression/visitor/TransformToExpressionVisitor.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/visitor/TransformToViewExpressionVisitor.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/write/CreateLogicalViewNode.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateLogicalViewStatement.java
copy server/src/main/java/org/apache/iotdb/db/pipe/{task/subtask/PipeAssignerSubtask.java => config/PipeCollectorConstant.java} (64%)
copy server/src/main/java/org/apache/iotdb/db/{mpp/common/schematree/IMeasurementSchemaInfo.java => pipe/config/PipeConnectorConstant.java} (77%)
copy server/src/main/java/org/apache/iotdb/db/{mpp/common/schematree/IMeasurementSchemaInfo.java => pipe/config/PipeProcessorConstant.java} (77%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/collector/IoTDBDataRegionCollector.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/collector/historical/PipeHistoricalDataRegionTsFileCollector.java
rename server/src/main/java/org/apache/iotdb/db/pipe/core/collector/realtime/{PipeRealtimeHybridDataRegionCollector.java => PipeRealtimeDataRegionHybridCollector.java} (68%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/event/EnrichedEvent.java
rename server/src/main/java/org/apache/iotdb/db/pipe/execution/executor/{PipeTaskExecutorManager.java => PipeSubtaskExecutorManager.java} (86%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeFileResourceManager.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeTsFileHolder.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeWALHolder.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/task/PipeBuilder.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/task/PipeTaskManager.java
copy server/src/main/java/org/apache/iotdb/db/pipe/task/{subtask/PipeAssignerSubtask.java => queue/EventSupplier.java} (65%)
copy server/src/main/java/org/apache/iotdb/db/pipe/task/{subtask/PipeAssignerSubtask.java => queue/ListenableBlockingPendingQueue.java} (69%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/task/queue/ListenablePendingQueue.java
copy server/src/main/java/org/apache/iotdb/db/pipe/task/{subtask/PipeAssignerSubtask.java => queue/ListenableUnblockingPendingQueue.java} (71%)
rename server/src/main/java/org/apache/iotdb/db/pipe/{core/collector/historical/PipeHistoricalCollector.java => task/queue/PendingQueueEmptyToNotEmptyListener.java} (83%)
rename server/src/main/java/org/apache/iotdb/db/pipe/{resource/PipeRaftlogHolder.java => task/queue/PendingQueueFullToNotFullListener.java} (83%)
copy confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskOperation.java => server/src/main/java/org/apache/iotdb/db/pipe/task/queue/PendingQueueNotEmptyToEmptyListener.java (83%)
rename server/src/main/java/org/apache/iotdb/db/pipe/{resource/PipeFileManager.java => task/queue/PendingQueueNotFullToFullListener.java} (83%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/view/ViewExpressionToStringTest.java
delete mode 100644 server/src/test/java/org/apache/iotdb/db/pipe/execution/executor/PipeAssignerSubtaskExecutorTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/pipe/resource/PipeFileResourceManagerTest.java
rename server/src/main/java/org/apache/iotdb/db/pipe/task/subtask/PipeAssignerSubtask.java => tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchemaType.java (70%)
[iotdb] 01/01: Merge branch 'master' into native_raft
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0b81ea470a76d22403afc888e34b6ef70237fe9f
Merge: 1de73174fea 1b1b95172f0
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon May 15 11:00:39 2023 +0800
Merge branch 'master' into native_raft
# Conflicts:
# confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
# server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 22 ++
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
client-py/tests/test_dataframe.py | 2 +
.../heartbeat/DataNodeHeartbeatHandler.java | 25 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 49 +++
.../confignode/conf/ConfigNodeDescriptor.java | 16 +
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 4 +
.../request/read/pipe/task/ShowPipePlanV2.java | 23 +-
.../coordinator/PipeHandleLeaderChangePlan.java | 74 ++++
.../response/pipe/task/PipeTableResp.java | 123 +++++++
.../statemachine/ConfigRegionStateMachine.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 22 ++
.../iotdb/confignode/manager/load/LoadManager.java | 2 +
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../confignode/manager/load/cache/LoadCache.java | 11 +-
.../manager/load/cache/route/RegionRouteCache.java | 4 +
.../manager/load/service/HeartbeatService.java | 4 +-
.../manager/load/service/StatisticsService.java | 9 +-
.../iotdb/confignode/manager/node/NodeManager.java | 5 +-
.../manager/partition/PartitionManager.java | 17 +-
.../manager/partition/PartitionMetrics.java | 2 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 7 +
.../manager/pipe/PipeRuntimeCoordinator.java | 82 +++++
.../manager/pipe/PipeTaskCoordinator.java | 32 +-
.../manager/{ => schema}/ClusterSchemaManager.java | 35 +-
.../schema/ClusterSchemaQuotaStatistics.java | 33 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 13 +
.../persistence/pipe/PipePluginInfo.java | 49 ++-
.../confignode/persistence/pipe/PipeTaskInfo.java | 56 +++
.../persistence/pipe/PipeTaskOperation.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 196 +++++++++++
.../pipe/task/AbstractOperatePipeProcedureV2.java | 28 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 27 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 12 +-
.../impl/schema/DeleteDatabaseProcedure.java | 8 +-
.../procedure/store/ProcedureFactory.java | 14 +
.../confignode/procedure/store/ProcedureType.java | 11 +-
.../iotdb/confignode/service/ConfigNode.java | 6 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 18 +
.../manager/ClusterSchemaManagerTest.java | 2 +
.../PipeHandleLeaderChangeProcedureTest.java | 63 ++++
consensus/pom.xml | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 36 +-
.../consensus/iot/IoTConsensusServerImpl.java | 2 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 10 +-
.../logdispatcher/LogDispatcherThreadMetrics.java | 22 ++
.../consensus/iot/logdispatcher/SyncStatus.java | 10 +-
.../ratis/ApplicationStateMachineProxy.java | 10 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 3 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 17 +
.../iot/logdispatcher/SyncStatusTest.java | 10 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 98 ------
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 8 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 85 ++++-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../UserGuide/API/Programming-Java-Native-API.md | 93 -----
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 24 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 87 ++++-
.../UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../iotdb/it/env/cluster/MppCommonConfig.java | 12 +
.../it/env/cluster/MppSharedCommonConfig.java | 14 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 10 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +
.../db/it/schema/IoTDBAutoCreateSchemaIT.java | 17 +-
.../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 +++
.../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 +++++++
.../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBCreateStorageGroupIT.java | 18 +-
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 26 +-
.../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 18 +-
.../db/it/schema/IoTDBDeleteStorageGroupIT.java | 17 +-
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 18 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 49 +--
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 63 ++--
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 49 +--
.../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 20 +-
.../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 17 +-
.../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 17 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 30 ++
.../org/apache/iotdb/util/AbstractSchemaIT.java | 48 ++-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 14 +-
.../metrics/metricsets/jvm/JvmMemoryMetrics.java | 6 +-
.../resources/conf/iotdb-common.properties | 55 ++-
.../apache/iotdb/commons/path/MeasurementPath.java | 31 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +-
.../plugin/builtin/collector/DefaultCollector.java | 60 ++++
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 9 +-
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 27 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 92 +++--
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 59 +++-
.../commons/schema/ClusterSchemaQuotaLevel.java | 19 +-
.../node/common/AbstractMeasurementMNode.java | 4 +
.../schema/node/role/IMeasurementMNode.java | 2 +
.../commons/schema/node/utils/IMNodeFactory.java | 4 +
.../commons/schema/view/LogicalViewSchema.java | 224 ++++++++++++
.../schema/view/viewExpression/ViewExpression.java | 318 +++++++++++++++++
.../view/viewExpression/ViewExpressionType.java | 76 ++++
.../binary/BinaryViewExpression.java | 119 +++++++
.../binary/arithmetic/AdditionViewExpression.java | 61 ++++
.../arithmetic/ArithmeticBinaryViewExpression.java | 52 +++
.../binary/arithmetic/DivisionViewExpression.java | 61 ++++
.../binary/arithmetic/ModuloViewExpression.java | 61 ++++
.../arithmetic/MultiplicationViewExpression.java | 63 ++++
.../arithmetic/SubtractionViewExpression.java | 61 ++++
.../compare/CompareBinaryViewExpression.java | 52 +++
.../binary/compare/EqualToViewExpression.java | 57 +++
.../binary/compare/GreaterEqualViewExpression.java | 57 +++
.../binary/compare/GreaterThanViewExpression.java | 57 +++
.../binary/compare/LessEqualViewExpression.java | 57 +++
.../binary/compare/LessThanViewExpression.java | 57 +++
.../binary/compare/NonEqualViewExpression.java | 57 +++
.../binary/logic/LogicAndViewExpression.java | 57 +++
.../binary/logic/LogicBinaryViewExpression.java | 52 +++
.../binary/logic/LogicOrViewExpression.java | 57 +++
.../viewExpression/leaf/ConstantViewOperand.java | 98 ++++++
.../view/viewExpression/leaf/LeafViewOperand.java | 20 +-
.../view/viewExpression/leaf/NullViewOperand.java | 61 ++++
.../viewExpression/leaf/TimeSeriesViewOperand.java | 87 +++++
.../viewExpression/leaf/TimestampViewOperand.java | 72 ++++
.../multi/FunctionViewExpression.java | 198 +++++++++++
.../ternary/BetweenViewExpression.java | 111 ++++++
.../ternary/TernaryViewExpression.java | 105 ++++++
.../viewExpression/unary/InViewExpression.java | 106 ++++++
.../viewExpression/unary/IsNullViewExpression.java | 89 +++++
.../viewExpression/unary/LikeViewExpression.java | 163 +++++++++
.../unary/LogicNotViewExpression.java | 61 ++++
.../unary/NegationViewExpression.java | 61 ++++
.../unary/RegularViewExpression.java | 105 ++++++
.../viewExpression/unary/UnaryViewExpression.java | 73 ++++
.../visitor/ViewExpressionVisitor.java | 219 ++++++++++++
.../commons/pipe/task/meta/PipeMetaDeSerTest.java | 77 ++++
.../org/apache/iotdb/pipe/api/PipeCollector.java | 1 +
.../api/exception/PipeConnectionException.java | 18 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 22 +-
.../rocksdb/mnode/RMeasurementMNode.java | 5 +
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 22 +-
.../resources/conf/iotdb-datanode.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 133 ++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 92 ++++-
.../db/consensus/DataRegionConsensusImpl.java | 9 +
.../statemachine/DataRegionStateMachine.java | 3 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 7 +-
.../compaction/constant/CompactionTaskStatus.java} | 9 +-
.../compaction/constant/CompactionTaskType.java} | 8 +-
.../CompactionMemoryNotEnoughException.java} | 9 +-
.../execute/task/AbstractCompactionTask.java | 1 -
.../execute/task/CrossSpaceCompactionTask.java | 6 +-
.../compaction/execute/utils/CompactionUtils.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 70 +++-
.../compaction/schedule/CompactionWorker.java | 3 -
.../estimator/AbstractCompactionEstimator.java | 3 +-
.../ReadPointCrossCompactionEstimator.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 37 +-
.../db/engine/storagegroup/TsFileResource.java | 5 +
.../metadata/SchemaQuotaExceededException.java} | 38 +-
.../apache/iotdb/db/metadata/MetadataConstant.java | 6 +
.../db/metadata/cache/DataNodeSchemaCache.java | 14 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 31 +-
.../cache/DeviceUsingTemplateSchemaCache.java | 18 +-
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 +-
.../db/metadata/cache/TimeSeriesSchemaCache.java | 9 +-
.../dualkeycache/impl/DualKeyCacheBuilder.java | 18 +-
.../dualkeycache/impl/DualKeyCachePolicy.java | 3 +-
.../dualkeycache/impl/FIFOCacheEntryManager.java | 189 ++++++++++
.../mnode/config/factory/ConfigMNodeFactory.java | 7 +
.../mnode/mem/factory/MemMNodeFactory.java | 14 +
...MeasurementMNode.java => LogicalViewMNode.java} | 31 +-
.../metadata/mnode/mem/impl/MeasurementMNode.java | 5 +
.../metadata/mnode/mem/info/LogicalViewInfo.java | 163 +++++++++
.../schemafile/factory/CacheMNodeFactory.java | 7 +
.../schemafile/impl/CachedMeasurementMNode.java | 5 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 31 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 94 ++++-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 48 ++-
.../db/metadata/mtree/store/CachedMTreeStore.java | 3 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +
.../plan/schemaregion/SchemaRegionPlanType.java | 2 +
.../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 +
.../impl/SchemaRegionPlanDeserializer.java | 24 ++
.../impl/SchemaRegionPlanSerializer.java | 27 ++
.../impl/SchemaRegionPlanTxtSerializer.java | 21 ++
.../impl/write/CreateLogicalViewPlanImpl.java | 75 ++++
.../schemaregion/result/ShowTimeSeriesResult.java | 13 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 66 ++++
.../metadata/query/info/ITimeSeriesSchemaInfo.java | 6 +-
.../rescon/DataNodeSchemaQuotaManager.java | 92 +++++
.../metadata/rescon/MemSchemaRegionStatistics.java | 8 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 21 +-
.../db/metadata/schemaregion/SchemaEngine.java | 92 ++++-
.../schemaregion/SchemaRegionMemoryImpl.java | 121 ++++---
.../schemaregion/SchemaRegionSchemaFileImpl.java | 78 ++--
.../metadata/template/ClusterTemplateManager.java | 36 ++
.../visitor/GetSourcePathsVisitor.java | 96 +++++
.../visitor/TransformToExpressionVisitor.java | 321 +++++++++++++++++
.../metadata/visitor/SchemaExecutionVisitor.java | 22 ++
.../db/mpp/common/header/ColumnHeaderConstant.java | 6 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 6 +-
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +-
.../common/schematree/IMeasurementSchemaInfo.java | 8 +-
.../common/schematree/MeasurementSchemaInfo.java | 22 +-
.../schematree/node/SchemaMeasurementNode.java | 42 ++-
.../mpp/execution/exchange/SharedTsBlockQueue.java | 12 +-
.../execution/executor/RegionWriteExecutor.java | 133 ++++++-
.../operator/process/DeviceViewIntoOperator.java | 6 +-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../schema/source/TimeSeriesSchemaSource.java | 14 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 49 +++
.../analyze/schema/ClusterSchemaFetchExecutor.java | 5 +-
.../config/executor/ClusterConfigTaskExecutor.java | 5 +-
.../visitor/TransformToViewExpressionVisitor.java | 391 +++++++++++++++++++++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 98 +++++-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 21 ++
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../node/metedata/write/CreateLogicalViewNode.java | 250 +++++++++++++
.../write/InternalCreateMultiTimeSeriesNode.java | 1 +
.../plan/planner/plan/node/write/InsertNode.java | 9 +
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 2 +-
.../iotdb/db/mpp/plan/statement/StatementType.java | 2 +
.../db/mpp/plan/statement/StatementVisitor.java | 7 +
.../metadata/CreateLogicalViewStatement.java | 246 +++++++++++++
.../db/pipe/agent/plugin/PipePluginAgent.java | 26 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 349 ++++++++++++++----
.../PipeCollectorConstant.java} | 21 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 6 +
.../config/PipeConnectorConstant.java} | 14 +-
.../config/PipeProcessorConstant.java} | 14 +-
.../core/collector/IoTDBDataRegionCollector.java | 85 +++++
.../PipeHistoricalDataRegionTsFileCollector.java | 103 ++++++
.../realtime/PipeRealtimeDataRegionCollector.java | 23 +-
... => PipeRealtimeDataRegionHybridCollector.java} | 64 +++-
.../realtime/assigner/PipeDataRegionAssigner.java | 20 +-
.../matcher/CachedSchemaPatternMatcher.java | 6 +-
.../realtime/matcher/PipeDataRegionMatcher.java | 9 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 31 +-
.../connector/PipeConnectorSubtaskManager.java | 42 ++-
.../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 +++
.../core/event/impl/PipeTabletInsertionEvent.java | 27 +-
.../core/event/impl/PipeTsFileInsertionEvent.java | 47 ++-
.../event/realtime/PipeRealtimeCollectEvent.java | 20 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 2 -
.../event/view/collector/PipeEventCollector.java | 65 +++-
...anager.java => PipeSubtaskExecutorManager.java} | 12 +-
.../execution/scheduler/PipeTaskScheduler.java | 18 +-
.../db/pipe/resource/PipeFileResourceManager.java | 186 ++++++++++
.../db/pipe/resource/PipeResourceManager.java | 16 +-
.../iotdb/db/pipe/resource/PipeTsFileHolder.java | 22 --
.../iotdb/db/pipe/resource/PipeWALHolder.java | 22 --
.../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 66 ++++
.../org/apache/iotdb/db/pipe/task/PipeTask.java | 9 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 60 +++-
.../apache/iotdb/db/pipe/task/PipeTaskManager.java | 96 +++++
.../EventSupplier.java} | 24 +-
.../ListenableBlockingPendingQueue.java} | 18 +-
.../db/pipe/task/queue/ListenablePendingQueue.java | 159 +++++++++
.../ListenableUnblockingPendingQueue.java} | 18 +-
.../PendingQueueEmptyToNotEmptyListener.java} | 8 +-
.../queue/PendingQueueFullToNotFullListener.java} | 8 +-
.../queue/PendingQueueNotEmptyToEmptyListener.java | 12 +-
.../queue/PendingQueueNotFullToFullListener.java} | 8 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 98 ++++--
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 100 +-----
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 131 +++++--
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 95 ++++-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 107 ++++--
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 42 ++-
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 6 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 20 +-
.../metrics/recorder/CompactionMetricsManager.java | 81 +++--
.../impl/DataNodeInternalRPCServiceImpl.java | 18 +-
.../datastructure/FixedPriorityBlockingQueue.java | 10 +
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 9 +-
.../engine/compaction/AbstractCompactionTest.java | 3 +-
.../FastCrossCompactionPerformerTest.java | 5 +
...eCompactionWithFastPerformerValidationTest.java | 16 +-
.../cache/dualkeycache/DualKeyCacheTest.java | 17 +
.../metadata/view/ViewExpressionToStringTest.java | 183 ++++++++++
.../mpp/execution/operator/OperatorMemoryTest.java | 34 ++
.../schema/SchemaFetchScanOperatorTest.java | 3 +-
.../collector/CachedSchemaPatternMatcherTest.java | 54 ++-
.../core/collector/PipeRealtimeCollectTest.java | 104 +++++-
.../executor/PipeAssignerSubtaskExecutorTest.java | 40 ---
.../executor/PipeConnectorSubtaskExecutorTest.java | 8 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 10 +-
.../pipe/resource/PipeFileResourceManagerTest.java | 218 ++++++++++++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../src/main/thrift/confignode.thrift | 5 +-
thrift/src/main/thrift/datanode.thrift | 18 +-
.../tsfile/write/schema/IMeasurementSchema.java | 4 +
.../tsfile/write/schema/MeasurementSchema.java | 10 +
.../tsfile/write/schema/MeasurementSchemaType.java | 23 +-
.../write/schema/VectorMeasurementSchema.java | 10 +
306 files changed, 11561 insertions(+), 1609 deletions(-)
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 991b2da7a4c,1def0dbb920..4cd9033da50
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@@ -29,7 -29,8 +29,9 @@@ import org.apache.iotdb.consensus.Conse
import org.apache.iotdb.rpc.RpcUtils;
import java.io.File;
+ import java.lang.reflect.Field;
+ import java.util.Arrays;
+import java.util.Properties;
public class ConfigNodeConfig {
@@@ -1139,11 -1161,28 +1163,36 @@@
this.forceWalPeriodForConfigNodeSimpleInMs = forceWalPeriodForConfigNodeSimpleInMs;
}
+ public Properties getOtherProperties() {
+ return otherProperties;
+ }
+
+ public void setOtherProperties(Properties otherProperties) {
+ this.otherProperties = otherProperties;
+ }
++
+ public String getConfigMessage() {
+ StringBuilder configMessage = new StringBuilder();
+ String configContent;
+ for (Field configField : ConfigNodeConfig.class.getDeclaredFields()) {
+ try {
+ String configType = configField.getGenericType().getTypeName();
+ if (configType.contains("java.lang.String[]")) {
+ String[] configList = (String[]) configField.get(this);
+ configContent = Arrays.asList(configList).toString();
+ } else {
+ configContent = configField.get(this).toString();
+ }
+ configMessage
+ .append("\n\t")
+ .append(configField.getName())
+ .append("=")
+ .append(configContent)
+ .append(";");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ return configMessage.toString();
+ }
}
diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f657ee635b3,c7e6c09d633..cbe12747d0a
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@@ -1099,27 -1124,40 +1124,59 @@@ public class IoTDBConfig
*/
private String RateLimiterType = "FixedIntervalRateLimiter";
+ /** The minimum/maximum number of subtask threads of each stage when flushing one MemTable. */
+ private int flushMemTableMinSubThread = 1;
+
+ private int flushMemTableMaxSubThread = 16;
+ /**
+ * If the idle ratio of a DynamicThread is below this value, it will try to add a new thread in
+ * its group if there are fewer threads than flushMemTableMaxSubThread.
+ */
+ private double dynamicThreadMinIdleRatio = 0.1;
+ /**
+ * If the idle ratio of a DynamicThread is over this value, it will try to exit if there are more
+ * threads than flushMemTableMinSubThread.
+ */
+ private double dynamicThreadMaxIdleRatio = 0.5;
+ /** A DynamicThread will not automatically exit unless its running time exceeds the value. */
+ private long dynamicThreadMinRunningTimeNS = 10_000_000_000L;
+
+ private boolean ignoreStateMachine = false;
+
IoTDBConfig() {}
+ public int getMaxLogEntriesNumPerBatch() {
+ return maxLogEntriesNumPerBatch;
+ }
+
+ public int getMaxSizePerBatch() {
+ return maxSizePerBatch;
+ }
+
+ public int getMaxPendingBatchesNum() {
+ return maxPendingBatchesNum;
+ }
+
+ public double getMaxMemoryRatioForQueue() {
+ return maxMemoryRatioForQueue;
+ }
+
+ public void setMaxLogEntriesNumPerBatch(int maxLogEntriesNumPerBatch) {
+ this.maxLogEntriesNumPerBatch = maxLogEntriesNumPerBatch;
+ }
+
+ public void setMaxSizePerBatch(int maxSizePerBatch) {
+ this.maxSizePerBatch = maxSizePerBatch;
+ }
+
+ public void setMaxPendingBatchesNum(int maxPendingBatchesNum) {
+ this.maxPendingBatchesNum = maxPendingBatchesNum;
+ }
+
+ public void setMaxMemoryRatioForQueue(double maxMemoryRatioForQueue) {
+ this.maxMemoryRatioForQueue = maxMemoryRatioForQueue;
+ }
+
public float getUdfMemoryBudgetInMB() {
return udfMemoryBudgetInMB;
}
@@@ -3857,11 -3881,19 +3940,27 @@@
return sortTmpDir;
}
+ public boolean isIgnoreStateMachine() {
+ return ignoreStateMachine;
+ }
+
+ public void setIgnoreStateMachine(boolean ignoreStateMachine) {
+ this.ignoreStateMachine = ignoreStateMachine;
+ }
++
+ public String getClusterSchemaLimitLevel() {
+ return clusterSchemaLimitLevel;
+ }
+
+ public void setClusterSchemaLimitLevel(String clusterSchemaLimitLevel) {
+ this.clusterSchemaLimitLevel = clusterSchemaLimitLevel;
+ }
+
+ public long getClusterSchemaLimitThreshold() {
+ return clusterSchemaLimitThreshold;
+ }
+
+ public void setClusterSchemaLimitThreshold(long clusterSchemaLimitThreshold) {
+ this.clusterSchemaLimitThreshold = clusterSchemaLimitThreshold;
+ }
}
diff --cc server/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
index 44719d1f05a,fd5976adc2f..e8b5ea3c11d
--- a/server/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
@@@ -120,9 -116,10 +125,10 @@@ public class DataRegionConsensusImpl
conf.getDataRatisConsensusSnapshotTriggerThreshold())
.build())
.setLog(
- RatisConfig.Log.newBuilder()
+ Log.newBuilder()
.setUnsafeFlushEnabled(
conf.isDataRatisConsensusLogUnsafeFlushEnable())
+ .setForceSyncNum(conf.getDataRatisConsensusLogForceSyncNum())
.setSegmentSizeMax(
SizeInBytes.valueOf(
conf.getDataRatisConsensusLogSegmentSizeMax()))
@@@ -134,9 -131,12 +140,12 @@@
.setFlowControlWindow(
SizeInBytes.valueOf(
conf.getDataRatisConsensusGrpcFlowControlWindow()))
+ .setLeaderOutstandingAppendsMax(
+ conf
+ .getDataRatisConsensusGrpcLeaderOutstandingAppendsMax())
.build())
.setRpc(
- RatisConfig.Rpc.newBuilder()
+ Rpc.newBuilder()
.setTimeoutMin(
TimeDuration.valueOf(
conf