You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/05/16 01:22:09 UTC
[iotdb] branch caLastOpt updated (42a8fc114dc -> b9148b90292)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 42a8fc114dc Merge remote-tracking branch 'origin/master' into caLastOpt
new dded9f0bf2a opt
new 8ce8f450194 opt
add f8a41d74c2b Refactor compaction task metrics (#9816)
add b48bc0a4858 [IOTDB-5858] fix the clear of metric in datanode schema cache
add d77c3994208 [IOTDB-5719] Cluster-Wide Time Series Limit Control (#9820)
add 1c2251394bf Use interface for MeasurementSchema and Move viewExpression to node commons (#9826)
add 467024bca30 [IOTDB-5865] Fix route balancing not trigger bug (#9793)
add 568aabe942e [IOTDB-5854] Pipe: Tsfile Resource Manager (#9805)
add 52324d4ec41 [IOTDB-5845] Pipe: Support handling leader change in PipeRuntimeCoordinator (#9777)
add 34fb7f532f6 [IOTDB-5799] Pipe: Support `SHOW PIPES` clause (#9838)
add 888fe13f13d build device timeIndex when validating files after compaction (#9831)
add af2d050b54b [IOTDB-5870] Pipe: Event reference management (#9836)
add 10d4eba1943 [IOTDB-5846] Optimize the memory estimate for HorizontallyConcatOperator (#9781)
add 0940f2a765f bump ratis version to 2.5.2-snapshot (#9839)
add e8f4897d741 [IOTDB-5872] Reduce some IoTConsensus index logs (#9844)
add 803c87fcdb0 [IOTDB-5873] Pipe: Support `CREATE`, `DROP`, `START`, `STOP` & `SHOW` Clauses (#9849)
new fd48eeb051e Merge remote-tracking branch 'origin/master' into caLastOpt
add 1b1b95172f0 [IOTDB-5852] Add some IoTConsensus and RatisConsensus properties (#9797)
add c9fe5c44637 [IOTDB-5823] wal pipe handler for the pipe module (#9708)
add a2bacfcb077 [IOTDB-5877] Fix StringIndexOutOfBoundsException when invoking Session.createTimeseriesUsingSchemaTemplate with list contains null (#9853)
new b9148b90292 Merge remote-tracking branch 'origin/master' into caLastOpt
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../heartbeat/DataNodeHeartbeatHandler.java | 25 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 22 ++
.../confignode/conf/ConfigNodeDescriptor.java | 16 ++
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 4 +
.../request/read/pipe/task/ShowPipePlanV2.java | 25 +-
.../coordinator/PipeHandleLeaderChangePlan.java | 74 ++++++
.../response/pipe/task/PipeTableResp.java | 123 ++++++++++
.../statemachine/ConfigRegionStateMachine.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 22 ++
.../iotdb/confignode/manager/load/LoadManager.java | 2 +
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../confignode/manager/load/cache/LoadCache.java | 11 +-
.../manager/load/cache/route/RegionRouteCache.java | 4 +
.../manager/load/service/HeartbeatService.java | 4 +-
.../manager/load/service/StatisticsService.java | 9 +-
.../iotdb/confignode/manager/node/NodeManager.java | 5 +-
.../manager/partition/PartitionManager.java | 17 +-
.../manager/partition/PartitionMetrics.java | 2 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 7 +
.../manager/pipe/PipeRuntimeCoordinator.java | 82 +++++++
.../manager/pipe/PipeTaskCoordinator.java | 32 ++-
.../manager/{ => schema}/ClusterSchemaManager.java | 22 +-
.../schema/ClusterSchemaQuotaStatistics.java | 33 +--
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 13 ++
.../persistence/pipe/PipePluginInfo.java | 49 +++-
.../confignode/persistence/pipe/PipeTaskInfo.java | 56 +++++
.../persistence/pipe/PipeTaskOperation.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 196 ++++++++++++++++
.../pipe/task/AbstractOperatePipeProcedureV2.java | 28 ++-
.../impl/pipe/task/CreatePipeProcedureV2.java | 27 +--
.../impl/pipe/task/DropPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 12 +-
.../impl/schema/DeleteDatabaseProcedure.java | 8 +-
.../procedure/store/ProcedureFactory.java | 14 ++
.../confignode/procedure/store/ProcedureType.java | 11 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 18 ++
.../manager/ClusterSchemaManagerTest.java | 2 +
.../PipeHandleLeaderChangeProcedureTest.java | 63 +++++
consensus/pom.xml | 2 +-
.../consensus/iot/IoTConsensusServerImpl.java | 2 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 2 +-
.../consensus/iot/logdispatcher/SyncStatus.java | 8 +-
.../consensus/iot/wal/ConsensusReqReader.java | 3 +
.../iot/logdispatcher/SyncStatusTest.java | 10 +-
.../consensus/iot/util/FakeConsensusReqReader.java | 5 +
docs/UserGuide/Reference/Common-Config-Manual.md | 56 +++++
.../zh/UserGuide/Reference/Common-Config-Manual.md | 58 +++++
.../iotdb/it/env/cluster/MppCommonConfig.java | 12 +
.../it/env/cluster/MppSharedCommonConfig.java | 14 ++
.../iotdb/it/env/remote/RemoteCommonConfig.java | 10 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 ++++
.../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 ++++++++++
.../session/it/IoTDBSessionSchemaTemplateIT.java | 15 ++
.../resources/conf/iotdb-common.properties | 36 +++
.../apache/iotdb/commons/path/MeasurementPath.java | 31 +--
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 9 +-
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 27 ++-
.../commons/pipe/task/meta/PipeStaticMeta.java | 88 +++++--
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 59 +++--
.../commons/schema/ClusterSchemaQuotaLevel.java | 16 +-
.../commons/schema/view}/LogicalViewSchema.java | 23 +-
.../view/viewExpression/ViewExpression.java | 56 ++---
.../view/viewExpression/ViewExpressionType.java | 2 +-
.../binary/BinaryViewExpression.java | 6 +-
.../binary/arithmetic/AdditionViewExpression.java | 8 +-
.../arithmetic/ArithmeticBinaryViewExpression.java | 8 +-
.../binary/arithmetic/DivisionViewExpression.java | 8 +-
.../binary/arithmetic/ModuloViewExpression.java | 8 +-
.../arithmetic/MultiplicationViewExpression.java | 8 +-
.../arithmetic/SubtractionViewExpression.java | 8 +-
.../compare/CompareBinaryViewExpression.java | 8 +-
.../binary/compare/EqualToViewExpression.java | 8 +-
.../binary/compare/GreaterEqualViewExpression.java | 8 +-
.../binary/compare/GreaterThanViewExpression.java | 8 +-
.../binary/compare/LessEqualViewExpression.java | 8 +-
.../binary/compare/LessThanViewExpression.java | 8 +-
.../binary/compare/NonEqualViewExpression.java | 8 +-
.../binary/logic/LogicAndViewExpression.java | 8 +-
.../binary/logic/LogicBinaryViewExpression.java | 8 +-
.../binary/logic/LogicOrViewExpression.java | 8 +-
.../viewExpression/leaf/ConstantViewOperand.java | 6 +-
.../view/viewExpression/leaf/LeafViewOperand.java | 4 +-
.../view/viewExpression/leaf/NullViewOperand.java | 6 +-
.../viewExpression/leaf/TimeSeriesViewOperand.java | 6 +-
.../viewExpression/leaf/TimestampViewOperand.java | 6 +-
.../multi/FunctionViewExpression.java | 8 +-
.../ternary/BetweenViewExpression.java | 8 +-
.../ternary/TernaryViewExpression.java | 6 +-
.../viewExpression/unary/InViewExpression.java | 8 +-
.../viewExpression/unary/IsNullViewExpression.java | 8 +-
.../viewExpression/unary/LikeViewExpression.java | 8 +-
.../unary/LogicNotViewExpression.java | 8 +-
.../unary/NegationViewExpression.java | 8 +-
.../unary/RegularViewExpression.java | 8 +-
.../viewExpression/unary/UnaryViewExpression.java | 6 +-
.../visitor/ViewExpressionVisitor.java | 70 +++---
.../commons/pipe/task/meta/PipeMetaDeSerTest.java | 77 +++++++
.../org/apache/iotdb/pipe/api/PipeCollector.java | 1 +
.../schemaregion/rocksdb/RSchemaRegion.java | 15 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 15 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 103 ++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 62 ++++-
.../db/consensus/DataRegionConsensusImpl.java | 9 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 149 ++++++------
.../compaction/constant/CompactionTaskStatus.java} | 11 +-
.../compaction/constant/CompactionTaskType.java} | 10 +-
.../execute/task/AbstractCompactionTask.java | 1 -
.../compaction/execute/utils/CompactionUtils.java | 22 +-
.../compaction/schedule/CompactionTaskManager.java | 70 +++++-
.../compaction/schedule/CompactionWorker.java | 3 -
.../db/engine/storagegroup/TsFileResource.java | 5 +
.../metadata/SchemaQuotaExceededException.java} | 27 +--
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 14 +-
.../cache/DeviceUsingTemplateSchemaCache.java | 18 +-
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 ++-
.../db/metadata/cache/TimeSeriesSchemaCache.java | 6 +-
.../metadata/mnode/mem/impl/LogicalViewMNode.java | 3 +-
.../metadata/mnode/mem/info/LogicalViewInfo.java | 12 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 21 ++
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 23 +-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 2 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 3 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +
.../impl/SchemaRegionPlanDeserializer.java | 2 +-
.../impl/SchemaRegionPlanSerializer.java | 2 +-
.../impl/SchemaRegionPlanTxtSerializer.java | 2 +-
.../impl/write/CreateLogicalViewPlanImpl.java | 2 +-
.../schemaregion/result/ShowTimeSeriesResult.java | 3 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 2 +-
.../rescon/DataNodeSchemaQuotaManager.java | 92 ++++++++
.../db/metadata/schemaregion/ISchemaRegion.java | 16 +-
.../db/metadata/schemaregion/SchemaEngine.java | 92 ++++++--
.../schemaregion/SchemaRegionMemoryImpl.java | 83 +++----
.../schemaregion/SchemaRegionSchemaFileImpl.java | 71 ++----
.../visitor/GetSourcePathsVisitor.java | 13 +-
.../visitor/TransformToExpressionVisitor.java | 67 +++---
.../metadata/visitor/SchemaExecutionVisitor.java | 2 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 6 +-
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +-
.../common/schematree/IMeasurementSchemaInfo.java | 8 +-
.../common/schematree/MeasurementSchemaInfo.java | 22 +-
.../schematree/node/SchemaMeasurementNode.java | 42 +++-
.../iotdb/db/mpp/execution/driver/Driver.java | 17 +-
.../execution/executor/RegionWriteExecutor.java | 84 ++++++-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 4 +
.../analyze/schema/ClusterSchemaFetchExecutor.java | 5 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 6 +
.../config/executor/ClusterConfigTaskExecutor.java | 5 +-
.../visitor/TransformToViewExpressionVisitor.java | 52 ++---
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 12 +-
.../db/mpp/plan/parser/StatementGenerator.java | 7 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 2 +-
.../node/metedata/write/CreateLogicalViewNode.java | 2 +-
.../write/InternalCreateMultiTimeSeriesNode.java | 1 +
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 2 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 214 +++++++++++++++++
.../db/pipe/config/PipeCollectorConstant.java | 5 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 6 +
.../realtime/PipeRealtimeDataRegionCollector.java | 6 +-
.../PipeRealtimeDataRegionHybridCollector.java | 31 ++-
.../realtime/assigner/PipeDataRegionAssigner.java | 20 +-
.../matcher/CachedSchemaPatternMatcher.java | 6 +-
.../realtime/matcher/PipeDataRegionMatcher.java | 9 +-
.../connector/PipeConnectorSubtaskManager.java | 23 +-
.../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 +++++
.../core/event/impl/PipeTabletInsertionEvent.java | 27 ++-
.../core/event/impl/PipeTsFileInsertionEvent.java | 48 +++-
.../event/realtime/PipeRealtimeCollectEvent.java | 20 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 2 -
.../event/view/collector/PipeEventCollector.java | 6 +
.../db/pipe/resource/PipeFileResourceManager.java | 186 +++++++++++++++
.../iotdb/db/pipe/resource/PipeRaftlogHolder.java | 22 --
.../db/pipe/resource/PipeResourceManager.java | 16 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 2 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 25 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 44 +++-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 26 ++-
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 32 +--
.../metrics/recorder/CompactionMetricsManager.java | 81 ++++---
.../service/thrift/impl/ClientRPCServiceImpl.java | 3 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 17 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 104 ++++-----
.../datastructure/FixedPriorityBlockingQueue.java | 10 +
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 26 ++-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 32 +--
.../org/apache/iotdb/db/wal/buffer/WALEntry.java | 4 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 118 +++++++---
.../iotdb/db/wal/checkpoint/MemTableInfo.java | 28 ++-
.../MemTablePinException.java} | 17 +-
.../WALPipeException.java} | 17 +-
.../apache/iotdb/db/wal/io/WALByteBufReader.java | 6 +-
.../org/apache/iotdb/db/wal/node/WALFakeNode.java | 24 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 57 ++++-
.../iotdb/db/wal/utils/WALEntryPosition.java | 157 +++++++++++++
.../apache/iotdb/db/wal/utils/WALFileUtils.java | 11 +
.../iotdb/db/wal/utils/WALInsertNodeCache.java | 170 ++++++++++++++
.../apache/iotdb/db/wal/utils/WALPipeHandler.java | 132 +++++++++++
.../db/wal/utils/listener/WALFlushListener.java | 13 +-
.../engine/compaction/AbstractCompactionTest.java | 3 +-
.../FastCrossCompactionPerformerTest.java | 5 +
...eCompactionWithFastPerformerValidationTest.java | 16 +-
.../metadata/view/ViewExpressionToStringTest.java | 16 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 34 +++
.../schema/SchemaFetchScanOperatorTest.java | 3 +-
.../collector/CachedSchemaPatternMatcherTest.java | 12 +-
.../core/collector/PipeRealtimeCollectTest.java | 53 +++--
.../pipe/resource/PipeFileResourceManagerTest.java | 218 ++++++++++++++++++
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 7 +-
.../iotdb/db/wal/node/WALPipeHandlerTest.java | 256 +++++++++++++++++++++
.../iotdb/db/wal/utils/WALInsertNodeCacheTest.java | 167 ++++++++++++++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/session/Session.java | 4 +
.../src/main/thrift/confignode.thrift | 5 +-
thrift/src/main/thrift/datanode.thrift | 18 +-
.../tsfile/write/schema/IMeasurementSchema.java | 4 +
.../tsfile/write/schema/MeasurementSchema.java | 10 +
.../tsfile/write/schema/MeasurementSchemaType.java | 19 +-
.../write/schema/VectorMeasurementSchema.java | 10 +
229 files changed, 5196 insertions(+), 1142 deletions(-)
copy server/src/main/java/org/apache/iotdb/db/metadata/view/viewExpression/leaf/LeafViewOperand.java => confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/pipe/task/ShowPipePlanV2.java (57%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/coordinator/PipeHandleLeaderChangePlan.java
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/PipeRuntimeCoordinator.java
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/{ => schema}/ClusterSchemaManager.java (97%)
copy server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeResourceManager.java => confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaQuotaStatistics.java (52%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java
create mode 100644 confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBClusterDeviceQuotaIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBClusterMeasurementQuotaIT.java
rename server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeFileManager.java => node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java (76%)
rename {server/src/main/java/org/apache/iotdb/db/metadata/mnode/mem/impl => node-commons/src/main/java/org/apache/iotdb/commons/schema/view}/LogicalViewSchema.java (91%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/ViewExpression.java (77%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/ViewExpressionType.java (97%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/BinaryViewExpression.java (93%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/AdditionViewExpression.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/ArithmeticBinaryViewExpression.java (82%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/DivisionViewExpression.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/ModuloViewExpression.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/MultiplicationViewExpression.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/arithmetic/SubtractionViewExpression.java (84%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/CompareBinaryViewExpression.java (82%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/EqualToViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/GreaterEqualViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/GreaterThanViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/LessEqualViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/LessThanViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/compare/NonEqualViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/logic/LogicAndViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/logic/LogicBinaryViewExpression.java (82%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/binary/logic/LogicOrViewExpression.java (83%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/leaf/ConstantViewOperand.java (92%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/leaf/LeafViewOperand.java (89%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/leaf/NullViewOperand.java (87%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/leaf/TimeSeriesViewOperand.java (90%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/leaf/TimestampViewOperand.java (88%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/multi/FunctionViewExpression.java (95%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/ternary/BetweenViewExpression.java (91%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/ternary/TernaryViewExpression.java (93%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/InViewExpression.java (90%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/IsNullViewExpression.java (88%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/LikeViewExpression.java (94%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/LogicNotViewExpression.java (85%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/NegationViewExpression.java (85%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/RegularViewExpression.java (90%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/unary/UnaryViewExpression.java (89%)
rename {server/src/main/java/org/apache/iotdb/db/metadata => node-commons/src/main/java/org/apache/iotdb/commons/schema}/view/viewExpression/visitor/ViewExpressionVisitor.java (68%)
create mode 100644 node-commons/src/test/java/org/apache/iotdb/commons/pipe/task/meta/PipeMetaDeSerTest.java
rename server/src/main/java/org/apache/iotdb/db/{pipe/resource/PipeWALHolder.java => engine/compaction/constant/CompactionTaskStatus.java} (80%)
rename server/src/main/java/org/apache/iotdb/db/{pipe/resource/PipeTsFileHolder.java => engine/compaction/constant/CompactionTaskType.java} (81%)
copy server/src/main/java/org/apache/iotdb/db/{pipe/agent/runtime/PipeRuntimeAgent.java => exception/metadata/SchemaQuotaExceededException.java} (52%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/rescon/DataNodeSchemaQuotaManager.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/core/event/EnrichedEvent.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeFileResourceManager.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/pipe/resource/PipeRaftlogHolder.java
copy server/src/main/java/org/apache/iotdb/db/wal/{utils/listener/WALFlushListener.java => exception/MemTablePinException.java} (70%)
copy server/src/main/java/org/apache/iotdb/db/wal/{utils/listener/WALFlushListener.java => exception/WALPipeException.java} (71%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/wal/utils/WALEntryPosition.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/wal/utils/WALInsertNodeCache.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/wal/utils/WALPipeHandler.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/pipe/resource/PipeFileResourceManagerTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/wal/node/WALPipeHandlerTest.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/wal/utils/WALInsertNodeCacheTest.java
copy server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/IMeasurementSchemaInfo.java => tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchemaType.java (69%)
[iotdb] 01/04: opt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dded9f0bf2acecabf3481c02a61764a9eabda482
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Thu May 11 21:52:30 2023 +0800
opt
---
.../db/engine/cache/TimeSeriesMetadataCache.java | 149 ++++++++++-----------
.../iotdb/db/mpp/execution/driver/Driver.java | 17 ++-
.../db/mpp/plan/parser/StatementGenerator.java | 7 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 3 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 104 +++++++-------
5 files changed, 128 insertions(+), 152 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index 296e373dbfc..16cc83646d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -52,9 +52,6 @@ import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_CACHE;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_FILE;
-
/**
* This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is
* LRU.
@@ -124,92 +121,86 @@ public class TimeSeriesMetadataCache {
throws IOException {
long startTime = System.nanoTime();
boolean cacheHit = true;
- try {
- if (!CACHE_ENABLE) {
- cacheHit = false;
-
- // bloom filter part
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
- BloomFilter bloomFilter = reader.readBloomFilter();
- if (bloomFilter != null
- && !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
- return null;
- }
- TimeseriesMetadata timeseriesMetadata =
- reader.readTimeseriesMetadata(
- new Path(key.device, key.measurement, true), ignoreNotExists);
- return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0)
- ? null
- : timeseriesMetadata;
+ if (!CACHE_ENABLE) {
+ cacheHit = false;
+
+ // bloom filter part
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ BloomFilter bloomFilter = reader.readBloomFilter();
+ if (bloomFilter != null
+ && !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
+ return null;
}
+ TimeseriesMetadata timeseriesMetadata =
+ reader.readTimeseriesMetadata(
+ new Path(key.device, key.measurement, true), ignoreNotExists);
+ return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0)
+ ? null
+ : timeseriesMetadata;
+ }
- TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
+ TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
- if (timeseriesMetadata == null) {
- if (debug) {
- DEBUG_LOGGER.info(
- "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
- DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
- }
- // allow for the parallelism of different devices
- synchronized (
- devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
- // double check
- timeseriesMetadata = lruCache.getIfPresent(key);
- if (timeseriesMetadata == null) {
- cacheHit = false;
-
- Path path = new Path(key.device, key.measurement, false);
- // bloom filter part
- BloomFilter bloomFilter =
- BloomFilterCache.getInstance()
- .get(new BloomFilterCache.BloomFilterCacheKey(key.filePath), debug);
- if (bloomFilter != null) {
- if (!bloomFilter.contains(path.getFullPath())) {
- if (debug) {
- DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
- }
- return null;
+ if (timeseriesMetadata == null) {
+ if (debug) {
+ DEBUG_LOGGER.info(
+ "Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
+ DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
+ }
+ // allow for the parallelism of different devices
+ synchronized (
+ devices.computeIfAbsent(key.device + SEPARATOR + key.filePath, WeakReference::new)) {
+ // double check
+ timeseriesMetadata = lruCache.getIfPresent(key);
+ if (timeseriesMetadata == null) {
+ cacheHit = false;
+
+ Path path = new Path(key.device, key.measurement, false);
+ // bloom filter part
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(new BloomFilterCache.BloomFilterCacheKey(key.filePath), debug);
+ if (bloomFilter != null) {
+ if (!bloomFilter.contains(path.getFullPath())) {
+ if (debug) {
+ DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
+ return null;
}
- TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
- List<TimeseriesMetadata> timeSeriesMetadataList =
- reader.readTimeseriesMetadata(path, allSensors);
- // put TimeSeriesMetadata of all sensors used in this query into cache
- for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
- TimeSeriesMetadataCacheKey k =
- new TimeSeriesMetadataCacheKey(
- key.filePath, key.device, metadata.getMeasurementId());
- if (metadata.getStatistics().getCount() != 0) {
- lruCache.put(k, metadata);
- }
- if (metadata.getMeasurementId().equals(key.measurement)) {
- timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata;
- }
+ }
+ TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
+ List<TimeseriesMetadata> timeSeriesMetadataList =
+ reader.readTimeseriesMetadata(path, allSensors);
+ // put TimeSeriesMetadata of all sensors used in this query into cache
+ for (TimeseriesMetadata metadata : timeSeriesMetadataList) {
+ TimeSeriesMetadataCacheKey k =
+ new TimeSeriesMetadataCacheKey(
+ key.filePath, key.device, metadata.getMeasurementId());
+ if (metadata.getStatistics().getCount() != 0) {
+ lruCache.put(k, metadata);
+ }
+ if (metadata.getMeasurementId().equals(key.measurement)) {
+ timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata;
}
}
}
}
- if (timeseriesMetadata == null) {
- if (debug) {
- DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
- }
- return null;
- } else {
- if (debug) {
- DEBUG_LOGGER.info(
- "Get timeseries: {}.{} metadata in file: {} from cache: {}.",
- key.device,
- key.measurement,
- key.filePath,
- timeseriesMetadata);
- }
- return new TimeseriesMetadata(timeseriesMetadata);
+ }
+ if (timeseriesMetadata == null) {
+ if (debug) {
+ DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
+ }
+ return null;
+ } else {
+ if (debug) {
+ DEBUG_LOGGER.info(
+ "Get timeseries: {}.{} metadata in file: {} from cache: {}.",
+ key.device,
+ key.measurement,
+ key.filePath,
+ timeseriesMetadata);
}
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- cacheHit ? READ_TIMESERIES_METADATA_CACHE : READ_TIMESERIES_METADATA_FILE,
- System.nanoTime() - startTime);
+ return new TimeseriesMetadata(timeseriesMetadata);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
index a214522ca65..2ce330e9767 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.execution.exchange.sink.ISink;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
-import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTaskId;
import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
@@ -383,14 +382,14 @@ public abstract class Driver implements IDriver {
sink.setNoMoreTsBlocks();
// record operator execution statistics to metrics
- List<OperatorContext> operatorContexts = driverContext.getOperatorContexts();
- for (OperatorContext operatorContext : operatorContexts) {
- String operatorType = operatorContext.getOperatorType();
- QUERY_METRICS.recordOperatorExecutionCost(
- operatorType, operatorContext.getTotalExecutionTimeInNanos());
- QUERY_METRICS.recordOperatorExecutionCount(
- operatorType, operatorContext.getNextCalledCount());
- }
+ // List<OperatorContext> operatorContexts = driverContext.getOperatorContexts();
+ // for (OperatorContext operatorContext : operatorContexts) {
+ // String operatorType = operatorContext.getOperatorType();
+ // QUERY_METRICS.recordOperatorExecutionCost(
+ // operatorType, operatorContext.getTotalExecutionTimeInNanos());
+ // QUERY_METRICS.recordOperatorExecutionCount(
+ // operatorType, operatorContext.getNextCalledCount());
+ // }
} catch (InterruptedException t) {
// don't record the stack
wasInterrupted = true;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index a4506737024..26b86e812ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -181,12 +181,7 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : lastDataQueryReq.getPaths()) {
- PartialPath path;
- if (lastDataQueryReq.isLegalPathNodes()) {
- path = new PartialPath(pathStr.split("\\."));
- } else {
- path = new PartialPath(pathStr);
- }
+ PartialPath path = new PartialPath(pathStr.split("\\."));
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
index 89cacc2361d..01bf9233621 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
@@ -437,7 +437,8 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
addStatementExecutionLatency(
OperationType.EXECUTE_LAST_DATA_QUERY,
StatementType.QUERY,
- System.nanoTime() - startNanoTime);
+ System.nanoTime() - startNanoTime,
+ TimeUnit.NANOSECONDS);
COORDINATOR.cleanupQueryExecution(queryId, t);
}
SESSION_MANAGER.updateIdleTime();
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index d558d0cf015..6ad023695e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -53,8 +53,6 @@ import java.util.Map.Entry;
import java.util.Set;
import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_MEM;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_DISK;
-import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_MEM;
import static org.apache.iotdb.db.mpp.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED;
public class FileLoaderUtils {
@@ -119,66 +117,58 @@ public class FileLoaderUtils {
throws IOException {
long t1 = System.nanoTime();
boolean loadFromMem = false;
- try {
- // common path
- TimeseriesMetadata timeSeriesMetadata;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- timeSeriesMetadata =
- TimeSeriesMetadataCache.getInstance()
- .get(
- new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
- resource.getTsFilePath(),
- seriesPath.getDevice(),
- seriesPath.getMeasurement()),
- allSensors,
- resource.getTimeIndexType() != 1,
- context.isDebug());
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
- }
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- loadFromMem = true;
-
- timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new MemChunkMetadataLoader(resource, seriesPath, context, filter));
- }
+ // common path
+ TimeseriesMetadata timeSeriesMetadata;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ timeSeriesMetadata =
+ TimeSeriesMetadataCache.getInstance()
+ .get(
+ new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
+ resource.getTsFilePath(),
+ seriesPath.getDevice(),
+ seriesPath.getMeasurement()),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ context.isDebug());
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
}
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ loadFromMem = true;
+ timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
if (timeSeriesMetadata != null) {
- long t2 = System.nanoTime();
- try {
- List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(), seriesPath);
- timeSeriesMetadata.setModified(!pathModifications.isEmpty());
- if (timeSeriesMetadata.getStatistics().getStartTime()
- > timeSeriesMetadata.getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- timeSeriesMetadata.getStatistics().getStartTime(),
- timeSeriesMetadata.getStatistics().getEndTime())) {
- return null;
- }
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
+ }
+
+ if (timeSeriesMetadata != null) {
+ long t2 = System.nanoTime();
+ try {
+ List<Modification> pathModifications =
+ context.getPathModifications(resource.getModFile(), seriesPath);
+ timeSeriesMetadata.setModified(!pathModifications.isEmpty());
+ if (timeSeriesMetadata.getStatistics().getStartTime()
+ > timeSeriesMetadata.getStatistics().getEndTime()) {
+ return null;
}
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ timeSeriesMetadata.getStatistics().getStartTime(),
+ timeSeriesMetadata.getStatistics().getEndTime())) {
+ return null;
+ }
+ } finally {
+ QUERY_METRICS.recordSeriesScanCost(
+ TIMESERIES_METADATA_MODIFICATION_NONALIGNED, System.nanoTime() - t2);
}
- return timeSeriesMetadata;
- } finally {
- QUERY_METRICS.recordSeriesScanCost(
- loadFromMem
- ? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM
- : LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,
- System.nanoTime() - t1);
}
+ return timeSeriesMetadata;
}
/**
@@ -278,7 +268,7 @@ public class FileLoaderUtils {
}
return alignedTimeSeriesMetadata;
} finally {
- if (loadFromMem) {
+ if (!loadFromMem) {
QUERY_METRICS.recordSeriesScanCost(
LOAD_TIMESERIES_METADATA_ALIGNED_MEM, System.nanoTime() - t1);
}
[iotdb] 04/04: Merge remote-tracking branch 'origin/master' into caLastOpt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b9148b902923dbc42b91dad88591a0d0f75571d2
Merge: fd48eeb051e a2bacfcb077
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue May 16 09:21:58 2023 +0800
Merge remote-tracking branch 'origin/master' into caLastOpt
.../iotdb/confignode/conf/ConfigNodeConfig.java | 22 ++
.../confignode/conf/ConfigNodeDescriptor.java | 16 ++
.../iotdb/confignode/manager/node/NodeManager.java | 3 +
.../consensus/iot/wal/ConsensusReqReader.java | 3 +
.../consensus/iot/util/FakeConsensusReqReader.java | 5 +
docs/UserGuide/Reference/Common-Config-Manual.md | 56 +++++
.../zh/UserGuide/Reference/Common-Config-Manual.md | 58 +++++
.../session/it/IoTDBSessionSchemaTemplateIT.java | 15 ++
.../resources/conf/iotdb-common.properties | 22 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 81 ++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 35 +++
.../db/consensus/DataRegionConsensusImpl.java | 9 +
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 26 ++-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 32 +--
.../org/apache/iotdb/db/wal/buffer/WALEntry.java | 4 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 118 +++++++---
.../iotdb/db/wal/checkpoint/MemTableInfo.java | 28 ++-
.../MemTablePinException.java} | 17 +-
.../WALPipeException.java} | 17 +-
.../apache/iotdb/db/wal/io/WALByteBufReader.java | 6 +-
.../org/apache/iotdb/db/wal/node/WALFakeNode.java | 24 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 57 ++++-
.../iotdb/db/wal/utils/WALEntryPosition.java | 157 +++++++++++++
.../apache/iotdb/db/wal/utils/WALFileUtils.java | 11 +
.../iotdb/db/wal/utils/WALInsertNodeCache.java | 170 ++++++++++++++
.../apache/iotdb/db/wal/utils/WALPipeHandler.java | 132 +++++++++++
.../db/wal/utils/listener/WALFlushListener.java | 13 +-
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 7 +-
.../iotdb/db/wal/node/WALPipeHandlerTest.java | 256 +++++++++++++++++++++
.../iotdb/db/wal/utils/WALInsertNodeCacheTest.java | 167 ++++++++++++++
.../java/org/apache/iotdb/session/Session.java | 4 +
.../src/main/thrift/confignode.thrift | 3 +
32 files changed, 1479 insertions(+), 95 deletions(-)
[iotdb] 03/04: Merge remote-tracking branch 'origin/master' into caLastOpt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fd48eeb051e12f69f6a6a4d17fa1d257ddd8c7ac
Merge: 8ce8f450194 803c87fcdb0
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon May 15 09:05:47 2023 +0800
Merge remote-tracking branch 'origin/master' into caLastOpt
.../heartbeat/DataNodeHeartbeatHandler.java | 25 ++-
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 4 +
.../request/read/pipe/task/ShowPipePlanV2.java | 25 +--
.../coordinator/PipeHandleLeaderChangePlan.java | 74 +++++++
.../response/pipe/task/PipeTableResp.java | 123 ++++++++++++
.../statemachine/ConfigRegionStateMachine.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 16 +-
.../apache/iotdb/confignode/manager/IManager.java | 1 +
.../iotdb/confignode/manager/ProcedureManager.java | 22 +++
.../iotdb/confignode/manager/load/LoadManager.java | 2 +
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../confignode/manager/load/cache/LoadCache.java | 11 +-
.../manager/load/cache/route/RegionRouteCache.java | 4 +
.../manager/load/service/HeartbeatService.java | 4 +-
.../manager/load/service/StatisticsService.java | 9 +-
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 17 +-
.../manager/partition/PartitionMetrics.java | 2 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 7 +
.../manager/pipe/PipeRuntimeCoordinator.java | 82 ++++++++
.../manager/pipe/PipeTaskCoordinator.java | 32 ++-
.../manager/{ => schema}/ClusterSchemaManager.java | 22 ++-
.../schema/ClusterSchemaQuotaStatistics.java | 33 ++--
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 13 ++
.../persistence/pipe/PipePluginInfo.java | 49 ++++-
.../confignode/persistence/pipe/PipeTaskInfo.java | 56 ++++++
.../persistence/pipe/PipeTaskOperation.java | 1 +
.../procedure/env/ConfigNodeProcedureEnv.java | 9 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 196 ++++++++++++++++++
.../pipe/task/AbstractOperatePipeProcedureV2.java | 28 ++-
.../impl/pipe/task/CreatePipeProcedureV2.java | 27 +--
.../impl/pipe/task/DropPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 12 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 12 +-
.../impl/schema/DeleteDatabaseProcedure.java | 8 +-
.../procedure/store/ProcedureFactory.java | 14 ++
.../confignode/procedure/store/ProcedureType.java | 11 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 18 ++
.../manager/ClusterSchemaManagerTest.java | 2 +
.../PipeHandleLeaderChangeProcedureTest.java | 63 ++++++
consensus/pom.xml | 2 +-
.../consensus/iot/IoTConsensusServerImpl.java | 2 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 2 +-
.../consensus/iot/logdispatcher/SyncStatus.java | 8 +-
.../iot/logdispatcher/SyncStatusTest.java | 10 +-
.../iotdb/it/env/cluster/MppCommonConfig.java | 12 ++
.../it/env/cluster/MppSharedCommonConfig.java | 14 ++
.../iotdb/it/env/remote/RemoteCommonConfig.java | 10 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 +++++
.../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 ++++++++++++
.../resources/conf/iotdb-common.properties | 14 ++
.../apache/iotdb/commons/path/MeasurementPath.java | 31 +--
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 9 +-
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 27 ++-
.../commons/pipe/task/meta/PipeStaticMeta.java | 88 +++++++--
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 59 ++++--
.../commons/schema/ClusterSchemaQuotaLevel.java | 16 +-
.../commons/schema/view}/LogicalViewSchema.java | 23 ++-
.../view/viewExpression/ViewExpression.java | 56 +++---
.../view/viewExpression/ViewExpressionType.java | 2 +-
.../binary/BinaryViewExpression.java | 6 +-
.../binary/arithmetic/AdditionViewExpression.java | 8 +-
.../arithmetic/ArithmeticBinaryViewExpression.java | 8 +-
.../binary/arithmetic/DivisionViewExpression.java | 8 +-
.../binary/arithmetic/ModuloViewExpression.java | 8 +-
.../arithmetic/MultiplicationViewExpression.java | 8 +-
.../arithmetic/SubtractionViewExpression.java | 8 +-
.../compare/CompareBinaryViewExpression.java | 8 +-
.../binary/compare/EqualToViewExpression.java | 8 +-
.../binary/compare/GreaterEqualViewExpression.java | 8 +-
.../binary/compare/GreaterThanViewExpression.java | 8 +-
.../binary/compare/LessEqualViewExpression.java | 8 +-
.../binary/compare/LessThanViewExpression.java | 8 +-
.../binary/compare/NonEqualViewExpression.java | 8 +-
.../binary/logic/LogicAndViewExpression.java | 8 +-
.../binary/logic/LogicBinaryViewExpression.java | 8 +-
.../binary/logic/LogicOrViewExpression.java | 8 +-
.../viewExpression/leaf/ConstantViewOperand.java | 6 +-
.../view/viewExpression/leaf/LeafViewOperand.java | 4 +-
.../view/viewExpression/leaf/NullViewOperand.java | 6 +-
.../viewExpression/leaf/TimeSeriesViewOperand.java | 6 +-
.../viewExpression/leaf/TimestampViewOperand.java | 6 +-
.../multi/FunctionViewExpression.java | 8 +-
.../ternary/BetweenViewExpression.java | 8 +-
.../ternary/TernaryViewExpression.java | 6 +-
.../viewExpression/unary/InViewExpression.java | 8 +-
.../viewExpression/unary/IsNullViewExpression.java | 8 +-
.../viewExpression/unary/LikeViewExpression.java | 8 +-
.../unary/LogicNotViewExpression.java | 8 +-
.../unary/NegationViewExpression.java | 8 +-
.../unary/RegularViewExpression.java | 8 +-
.../viewExpression/unary/UnaryViewExpression.java | 6 +-
.../visitor/ViewExpressionVisitor.java | 70 +++----
.../commons/pipe/task/meta/PipeMetaDeSerTest.java | 77 ++++++++
.../org/apache/iotdb/pipe/api/PipeCollector.java | 1 +
.../schemaregion/rocksdb/RSchemaRegion.java | 15 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 15 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++-
.../compaction/constant/CompactionTaskStatus.java} | 11 +-
.../compaction/constant/CompactionTaskType.java} | 10 +-
.../execute/task/AbstractCompactionTask.java | 1 -
.../compaction/execute/utils/CompactionUtils.java | 22 +--
.../compaction/schedule/CompactionTaskManager.java | 70 ++++++-
.../compaction/schedule/CompactionWorker.java | 3 -
.../db/engine/storagegroup/TsFileResource.java | 5 +
.../metadata/SchemaQuotaExceededException.java} | 27 ++-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 14 +-
.../cache/DeviceUsingTemplateSchemaCache.java | 18 +-
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 ++--
.../db/metadata/cache/TimeSeriesSchemaCache.java | 6 +-
.../metadata/mnode/mem/impl/LogicalViewMNode.java | 3 +-
.../metadata/mnode/mem/info/LogicalViewInfo.java | 12 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 21 ++
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 23 ++-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 2 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 3 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +
.../impl/SchemaRegionPlanDeserializer.java | 2 +-
.../impl/SchemaRegionPlanSerializer.java | 2 +-
.../impl/SchemaRegionPlanTxtSerializer.java | 2 +-
.../impl/write/CreateLogicalViewPlanImpl.java | 2 +-
.../schemaregion/result/ShowTimeSeriesResult.java | 3 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 2 +-
.../rescon/DataNodeSchemaQuotaManager.java | 92 +++++++++
.../db/metadata/schemaregion/ISchemaRegion.java | 16 +-
.../db/metadata/schemaregion/SchemaEngine.java | 92 +++++++--
.../schemaregion/SchemaRegionMemoryImpl.java | 83 +++-----
.../schemaregion/SchemaRegionSchemaFileImpl.java | 71 ++-----
.../visitor/GetSourcePathsVisitor.java | 13 +-
.../visitor/TransformToExpressionVisitor.java | 67 +++----
.../metadata/visitor/SchemaExecutionVisitor.java | 2 +-
.../mpp/common/schematree/ClusterSchemaTree.java | 6 +-
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +-
.../common/schematree/IMeasurementSchemaInfo.java | 8 +-
.../common/schematree/MeasurementSchemaInfo.java | 22 ++-
.../schematree/node/SchemaMeasurementNode.java | 42 +++-
.../execution/executor/RegionWriteExecutor.java | 84 +++++++-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 4 +
.../analyze/schema/ClusterSchemaFetchExecutor.java | 5 +-
.../config/executor/ClusterConfigTaskExecutor.java | 5 +-
.../visitor/TransformToViewExpressionVisitor.java | 52 ++---
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 12 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 2 +-
.../node/metedata/write/CreateLogicalViewNode.java | 2 +-
.../write/InternalCreateMultiTimeSeriesNode.java | 1 +
.../planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 2 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 5 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 214 ++++++++++++++++++++
.../db/pipe/config/PipeCollectorConstant.java | 5 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 6 +
.../realtime/PipeRealtimeDataRegionCollector.java | 6 +-
.../PipeRealtimeDataRegionHybridCollector.java | 31 ++-
.../realtime/assigner/PipeDataRegionAssigner.java | 20 +-
.../matcher/CachedSchemaPatternMatcher.java | 6 +-
.../realtime/matcher/PipeDataRegionMatcher.java | 9 +-
.../connector/PipeConnectorSubtaskManager.java | 23 ++-
.../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 +++++
.../core/event/impl/PipeTabletInsertionEvent.java | 27 ++-
.../core/event/impl/PipeTsFileInsertionEvent.java | 48 ++++-
.../event/realtime/PipeRealtimeCollectEvent.java | 20 +-
.../realtime/PipeRealtimeCollectEventFactory.java | 2 -
.../event/view/collector/PipeEventCollector.java | 6 +
.../db/pipe/resource/PipeFileResourceManager.java | 186 ++++++++++++++++++
.../db/pipe/resource/PipeResourceManager.java | 16 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 2 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 25 ++-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 44 +++--
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 26 ++-
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 32 +--
.../metrics/recorder/CompactionMetricsManager.java | 81 ++++----
.../impl/DataNodeInternalRPCServiceImpl.java | 17 +-
.../datastructure/FixedPriorityBlockingQueue.java | 10 +
.../engine/compaction/AbstractCompactionTest.java | 3 +-
.../FastCrossCompactionPerformerTest.java | 5 +
...eCompactionWithFastPerformerValidationTest.java | 16 +-
.../metadata/view/ViewExpressionToStringTest.java | 16 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 34 ++++
.../schema/SchemaFetchScanOperatorTest.java | 3 +-
.../collector/CachedSchemaPatternMatcherTest.java | 12 +-
.../core/collector/PipeRealtimeCollectTest.java | 53 +++--
.../pipe/resource/PipeFileResourceManagerTest.java | 218 +++++++++++++++++++++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../src/main/thrift/confignode.thrift | 2 +-
thrift/src/main/thrift/datanode.thrift | 18 +-
.../tsfile/write/schema/IMeasurementSchema.java | 4 +
.../tsfile/write/schema/MeasurementSchema.java | 10 +
.../tsfile/write/schema/MeasurementSchemaType.java | 20 +-
.../write/schema/VectorMeasurementSchema.java | 10 +
195 files changed, 3588 insertions(+), 869 deletions(-)
[iotdb] 02/04: opt
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch caLastOpt
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8ce8f4501945e47fec48e40d4887f3bb680b5dc7
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Fri May 12 09:22:43 2023 +0800
opt
---
.../iotdb/db/mpp/plan/analyze/schema/ClusterSchemaFetcher.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/ClusterSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/ClusterSchemaFetcher.java
index 53fa5ee1287..b1d57c6cbd1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/ClusterSchemaFetcher.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/schema/ClusterSchemaFetcher.java
@@ -39,6 +39,9 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -50,6 +53,8 @@ import java.util.stream.IntStream;
public class ClusterSchemaFetcher implements ISchemaFetcher {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSchemaFetcher.class);
+
private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private final Coordinator coordinator = Coordinator.getInstance();
@@ -134,6 +139,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
for (PartialPath fullPath : fullPathList) {
cachedSchema = schemaCache.get(fullPath);
if (cachedSchema.isEmpty()) {
+ LOGGER.info("Schema Cache Miss {}", fullPath.getFullPath());
isAllCached = false;
break;
} else {