You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/05/14 12:38:54 UTC
[iotdb] 01/02: Merge remote-tracking branch 'upstream/master' into lmh/groupByTest
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/groupByTest
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c248ebfdfc936b728f49eb31a0bb4cc663752719
Merge: e8cc0dcb1ca 10d4eba1943
Author: liuminghui233 <54...@qq.com>
AuthorDate: Sun May 14 20:35:30 2023 +0800
Merge remote-tracking branch 'upstream/master' into lmh/groupByTest
# Conflicts:
# server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
# server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/sink/LocalSinkChannel.java
# server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
# server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 66 +-
.../org/apache/iotdb/db/qp/sql/PathParser.g4 | 14 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 +-
client-py/SessionExample.py | 1 -
client-py/iotdb/Session.py | 323 ++-
client-py/iotdb/utils/IoTDBConnectionException.py | 29 +
client-py/iotdb/utils/IoTDBRpcDataSet.py | 10 +-
client-py/tests/test_dataframe.py | 2 +
.../confignode/client/DataNodeRequestType.java | 7 +-
.../client/async/AsyncDataNodeClientPool.java | 35 +-
.../client/async/handlers/AsyncClientHandler.java | 18 +-
.../heartbeat/ConfigNodeHeartbeatHandler.java | 17 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 88 +-
...ava => CheckTimeSeriesExistenceRPCHandler.java} | 47 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 27 +
.../consensus/request/ConfigPhysicalPlan.java | 82 +-
.../consensus/request/ConfigPhysicalPlanType.java | 44 +-
.../consensus/request/auth/AuthorPlan.java | 29 +-
...lotListPlan.java => CountTimeSlotListPlan.java} | 52 +-
.../read/partition/GetSeriesSlotListPlan.java | 18 +-
.../read/partition/GetTimeSlotListPlan.java | 44 +-
.../pipe/task/ShowPipePlanV2.java} | 26 +-
.../request/read/region/GetRegionIdPlan.java | 46 +-
.../coordinator/PipeHandleLeaderChangePlan.java | 74 +
.../task/CreatePipePlanV2.java} | 41 +-
.../task/DropPipePlanV2.java} | 12 +-
.../task/SetPipeStatusPlanV2.java} | 34 +-
...PipeSinkPlan.java => CreatePipeSinkPlanV1.java} | 10 +-
.../{DropPipePlan.java => DropPipePlanV1.java} | 10 +-
...opPipeSinkPlan.java => DropPipeSinkPlanV1.java} | 10 +-
...GetPipeSinkPlan.java => GetPipeSinkPlanV1.java} | 10 +-
...reatePipePlan.java => PreCreatePipePlanV1.java} | 10 +-
.../request/write/sync/RecordPipeMessagePlan.java | 4 +-
...ipeStatusPlan.java => SetPipeStatusPlanV1.java} | 24 +-
.../{ShowPipePlan.java => ShowPipePlanV1.java} | 10 +-
.../CommitSetSchemaTemplatePlan.java} | 54 +-
.../ExtendSchemaTemplatePlan.java} | 29 +-
.../PreSetSchemaTemplatePlan.java} | 54 +-
...egionIdResp.java => CountTimeSlotListResp.java} | 19 +-
.../response/partition/GetRegionIdResp.java | 2 +-
.../response/partition/GetSeriesSlotListResp.java | 2 +-
.../response/partition/GetTimeSlotListResp.java | 2 +-
.../task/PipeTableResp.java} | 42 +-
.../statemachine/ConfigRegionStateMachine.java | 12 +-
.../confignode/manager/ClusterQuotaManager.java | 18 +-
.../iotdb/confignode/manager/ConfigManager.java | 188 +-
.../apache/iotdb/confignode/manager/IManager.java | 83 +-
.../confignode/manager/PermissionManager.java | 3 +-
.../iotdb/confignode/manager/ProcedureManager.java | 170 +-
.../confignode/manager/RetryFailedTasksThread.java | 85 +-
.../iotdb/confignode/manager/SyncManager.java | 306 ---
.../manager/consensus/ConsensusManager.java | 12 +-
.../iotdb/confignode/manager/load/LoadManager.java | 488 ++--
.../manager/load/balancer/RegionBalancer.java | 9 +-
.../manager/load/balancer/RouteBalancer.java | 276 +-
.../load/balancer/router/RegionRouteMap.java | 171 --
.../confignode/manager/load/cache/LoadCache.java | 600 ++++
.../cache/node}/BaseNodeCache.java | 59 +-
.../cache/node}/ConfigNodeHeartbeatCache.java | 29 +-
.../cache/node}/DataNodeHeartbeatCache.java | 25 +-
.../cache/node}/NodeHeartbeatSample.java | 7 +-
.../cache/node}/NodeStatistics.java | 11 +-
.../cache/region}/RegionCache.java | 7 +-
.../cache/region}/RegionGroupCache.java | 38 +-
.../cache/region}/RegionGroupStatistics.java | 16 +-
.../cache/region}/RegionHeartbeatSample.java | 3 +-
.../cache/region}/RegionStatistics.java | 16 +-
.../manager/load/cache/route/RegionRouteCache.java | 142 +
.../manager/load/service/HeartbeatService.java | 193 ++
.../manager/load/service/StatisticsService.java | 252 ++
.../load/subscriber/IClusterStatusSubscriber.java | 17 +-
.../manager/load/subscriber/RouteChangeEvent.java | 49 +
.../load/subscriber/StatisticsChangeEvent.java | 53 +
.../iotdb/confignode/manager/node/NodeManager.java | 330 +--
.../iotdb/confignode/manager/observer/IEvent.java | 21 -
.../manager/partition/PartitionManager.java | 249 +-
.../manager/partition/PartitionMetrics.java | 20 +-
.../manager/partition/RegionGroupStatus.java | 21 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 19 +-
.../manager/pipe/PipePluginCoordinator.java | 10 +-
.../manager/pipe/PipeRuntimeCoordinator.java | 82 +
.../manager/pipe/PipeTaskCoordinator.java | 94 +
.../manager/{ => schema}/ClusterSchemaManager.java | 211 +-
.../ClusterSchemaQuotaStatistics.java} | 29 +-
.../iotdb/confignode/persistence/AuthorInfo.java | 34 +-
.../persistence/executor/ConfigPlanExecutor.java | 75 +-
.../confignode/persistence/node/NodeInfo.java | 47 +-
.../partition/DatabasePartitionTable.java | 87 +-
.../persistence/partition/PartitionInfo.java | 196 +-
.../persistence/partition/RegionGroup.java | 13 +-
.../confignode/persistence/pipe/PipeInfo.java | 59 +-
.../persistence/pipe/PipePluginInfo.java | 82 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 277 ++
.../persistence/pipe/PipeTaskOperation.java | 13 +-
.../confignode/persistence/quota/QuotaInfo.java | 36 +-
.../persistence/schema/ClusterSchemaInfo.java | 108 +-
.../persistence/schema/TemplatePreSetTable.java | 195 ++
.../persistence/schema/TemplateTable.java | 48 +-
.../persistence/sync/ClusterSyncInfo.java | 225 --
.../procedure/env/ConfigNodeProcedureEnv.java | 96 +-
.../procedure/env/DataNodeRemoveHandler.java | 19 +-
.../pipe/plugin/CreatePipePluginProcedure.java | 30 +-
.../impl/pipe/plugin/DropPipePluginProcedure.java | 30 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 196 ++
.../pipe/task/AbstractOperatePipeProcedureV2.java | 219 ++
.../impl/pipe/task/CreatePipeProcedureV2.java | 254 ++
.../impl/pipe/task/DropPipeProcedureV2.java | 149 +
.../impl/pipe/task/StartPipeProcedureV2.java | 161 ++
.../impl/pipe/task/StopPipeProcedureV2.java | 161 ++
.../impl/schema/DataNodeRegionTaskExecutor.java | 4 +-
.../impl/schema/DeleteDatabaseProcedure.java | 12 +-
.../impl/schema/SetTemplateProcedure.java | 534 ++++
.../statemachine/CreateRegionGroupsProcedure.java | 1 -
.../impl/statemachine/RegionMigrateProcedure.java | 3 +
.../impl/sync/AbstractOperatePipeProcedure.java | 147 -
.../procedure/impl/sync/CreatePipeProcedure.java | 131 +-
.../procedure/impl/sync/DropPipeProcedure.java | 97 +-
.../procedure/impl/sync/StartPipeProcedure.java | 136 +-
.../procedure/impl/sync/StopPipeProcedure.java | 136 +-
.../task/OperatePipeTaskState.java} | 13 +-
.../procedure/state/schema/SetTemplateState.java | 13 +-
.../procedure/store/ProcedureFactory.java | 26 +
.../confignode/procedure/store/ProcedureType.java | 14 +-
.../iotdb/confignode/service/ConfigNode.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 78 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 199 +-
.../manager/ClusterSchemaManagerTest.java | 2 +
.../load/balancer/router/RegionRouteMapTest.java | 82 -
.../router/priority/GreedyPriorityTest.java | 8 +-
.../priority/LeaderPriorityBalancerTest.java | 8 +-
.../confignode/manager/node/NodeCacheTest.java | 8 +-
.../manager/partition/RegionGroupCacheTest.java | 4 +-
.../confignode/persistence/AuthorInfoTest.java | 30 +-
.../persistence/ClusterSyncInfoTest.java | 164 --
.../iotdb/confignode/persistence/PipeInfoTest.java | 100 +
.../persistence/node/NodeStatisticsTest.java | 2 +-
.../statistics/RegionGroupStatisticsTest.java | 4 +-
.../partition/statistics/RegionStatisticsTest.java | 2 +-
.../{ => schema}/ClusterSchemaInfoTest.java | 3 +-
.../schema/TemplatePreSetTableTest.java | 114 +
.../{ => schema}/TemplateTableTest.java | 3 +-
.../CreatePipePluginProcedureTest.java | 5 +-
.../{ => plugin}/DropPipePluginProcedureTest.java | 3 +-
.../PipeHandleLeaderChangeProcedureTest.java} | 27 +-
.../CreatePipeProcedureV2Test.java} | 33 +-
.../DropPipeProcedureV2Test.java} | 12 +-
.../StartPipeProcedureV2Test.java} | 12 +-
.../StopPipeProcedureV2Test.java} | 12 +-
consensus/pom.xml | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 36 +-
.../consensus/iot/IoTConsensusServerImpl.java | 13 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 8 +-
.../logdispatcher/LogDispatcherThreadMetrics.java | 22 +
.../consensus/iot/logdispatcher/SyncStatus.java | 2 -
.../ratis/ApplicationStateMachineProxy.java | 10 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 3 +-
.../iotdb/consensus/ratis/SnapshotStorage.java | 8 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 17 +
docs/Community/Materials.md | 103 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 98 -
docs/UserGuide/Cluster/Cluster-Maintenance.md | 176 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 60 +-
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 8 +-
docs/UserGuide/Operate-Metadata/Template.md | 15 +-
docs/UserGuide/Operators-Functions/Aggregation.md | 1 +
docs/UserGuide/Operators-Functions/Sample.md | 50 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 49 +-
docs/UserGuide/QuickStart/Files.md | 2 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 40 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 18 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 4 +-
docs/zh/UserGuide/API/Programming-Go-Native-API.md | 1 +
.../UserGuide/API/Programming-Java-Native-API.md | 93 -
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 177 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 68 +-
.../UserGuide/Ecosystem-Integration/Workbench.md | 79 +-
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 27 +-
docs/zh/UserGuide/Operate-Metadata/Template.md | 14 +
.../UserGuide/Operators-Functions/Aggregation.md | 1 +
docs/zh/UserGuide/Operators-Functions/Sample.md | 58 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 48 +-
docs/zh/UserGuide/QuickStart/Files.md | 2 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 38 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 27 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 4 +-
grafana-plugin/README.md | 63 +-
grafana-plugin/src/plugin.json | 2 +-
.../iotdb/it/env/cluster/AbstractNodeWrapper.java | 1 +
.../iotdb/it/env/cluster/MppCommonConfig.java | 30 +
.../it/env/cluster/MppSharedCommonConfig.java | 35 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 25 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 10 +
.../confignode/it/IoTDBClusterAuthorityIT.java | 88 +-
.../it/partition/IoTDBPartitionGetterIT.java | 96 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 40 +
.../db/it/aggregation/IoTDBTagAggregationIT.java | 13 +
.../db/it/last/IoTDBLastQueryLastCache2IT.java | 60 +
.../db/it/last/IoTDBLastQueryLastCacheIT.java | 145 +
.../iotdb/db/it/orderBy/IoTDBOrderBy2IT.java | 48 +
.../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 1226 +++++++++
.../iotdb/db/it/query/IoTDBPaginationIT.java | 59 +-
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +
.../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java | 535 ++++
.../db/it/schema/IoTDBAutoCreateSchemaIT.java | 18 +-
.../db/it/schema/IoTDBClusterDeviceQuotaIT.java | 44 +
.../it/schema/IoTDBClusterMeasurementQuotaIT.java | 126 +
.../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 18 +-
.../db/it/schema/IoTDBCreateStorageGroupIT.java | 19 +-
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 18 +-
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 30 +-
.../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 19 +-
.../db/it/schema/IoTDBDeleteStorageGroupIT.java | 18 +-
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 19 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 228 ++
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 113 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 210 +-
.../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 21 +-
.../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 18 +-
.../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 18 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 30 +
.../org/apache/iotdb/db/it/udf/IoTDBUDFM4IT.java | 304 ---
.../it/udf/IoTDBUDTFBuiltinScalarFunctionIT.java | 688 ++++-
.../iotdb/libudf/it/dprofile/DProfileIT.java | 17 +
.../session/it/IoTDBSessionSchemaTemplateIT.java | 116 +-
.../{db/it/schema => util}/AbstractSchemaIT.java | 52 +-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 14 +-
library-udf/Readme.md | 27 +
library-udf/pom.xml | 45 +-
.../apache/iotdb/library/anomaly/UDTFKSigma.java | 6 +-
.../apache/iotdb/library/anomaly/UDTFRange.java | 6 +-
.../iotdb/library/anomaly/UDTFTwoSidedFilter.java | 3 -
.../org/apache/iotdb/library/dprofile/UDAFMad.java | 5 +-
.../apache/iotdb/library/dprofile/UDAFMedian.java | 5 +-
.../iotdb/library/dprofile/UDAFPercentile.java | 10 +-
.../iotdb/library/dprofile/UDAFQuantile.java | 6 +-
.../apache/iotdb/library/dprofile/UDAFSpread.java | 10 +-
.../iotdb/library/dprofile/UDTFDistinct.java | 6 +-
.../apache/iotdb/library/dprofile/UDTFMvAvg.java | 6 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 7 +-
.../dprofile/util/ExactOrderStatistics.java | 41 +-
.../java/org/apache/iotdb/library/util/Util.java | 5 +-
.../metrics/metricsets/jvm/JvmMemoryMetrics.java | 6 +-
.../metrics/metricsets/jvm/JvmThreadMetrics.java | 31 +-
.../metricsets/net/LinuxNetMetricManager.java | 27 +-
.../iotdb/metrics/metricsets/net/NetMetrics.java | 12 +-
.../apache/iotdb/metrics/utils/MetricLevel.java | 3 +-
.../resources/conf/iotdb-common.properties | 58 +-
.../commons/auth/authorizer/BasicAuthorizer.java | 35 +-
.../iotdb/commons/auth/authorizer/IAuthorizer.java | 33 +-
.../commons/auth/authorizer/OpenIdAuthorizer.java | 3 +-
.../iotdb/commons/auth/entity/PathPrivilege.java | 29 +-
.../org/apache/iotdb/commons/auth/entity/Role.java | 13 +-
.../org/apache/iotdb/commons/auth/entity/User.java | 13 +-
.../iotdb/commons/auth/role/BasicRoleManager.java | 5 +-
.../iotdb/commons/auth/role/IRoleManager.java | 6 +-
.../iotdb/commons/auth/user/BasicUserManager.java | 5 +-
.../iotdb/commons/auth/user/IUserManager.java | 6 +-
.../iotdb/commons/concurrent/ThreadName.java | 2 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 10 +
.../iotdb/commons/conf/CommonDescriptor.java | 8 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 32 +-
.../commons/partition/DataPartitionTable.java | 45 +-
.../commons/partition/SchemaPartitionTable.java | 10 +
.../commons/partition/SeriesPartitionTable.java | 32 +-
.../org/apache/iotdb/commons/path/AlignedPath.java | 6 +
.../apache/iotdb/commons/path/MeasurementPath.java | 31 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 48 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 59 +
.../plugin/builtin/collector/DefaultCollector.java | 60 +
.../builtin/connector/DoNothingConnector.java | 72 +
.../builtin/processor/DoNothingProcessor.java | 68 +
.../meta/ConfigNodePipePluginMetaKeeper.java | 47 +-
.../plugin/meta/DataNodePipePluginMetaKeeper.java | 28 +-
.../commons/pipe/plugin/meta/PipePluginMeta.java | 65 +-
.../pipe/plugin/meta/PipePluginMetaKeeper.java | 76 +-
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 101 +
.../commons/pipe/task/meta/PipeMetaKeeper.java | 101 +
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 125 +
.../commons/pipe/task/meta/PipeStaticMeta.java | 183 ++
.../{PipeTaskMetaAccessor.java => PipeStatus.java} | 30 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 149 +
.../commons/schema/ClusterSchemaQuotaLevel.java | 16 +-
.../node/common/AbstractMeasurementMNode.java | 4 +
.../schema/node/role/IMeasurementMNode.java | 2 +
.../commons/schema/node/utils/IMNodeFactory.java | 4 +
.../commons/schema/tree/AbstractTreeVisitor.java | 122 +-
.../commons/schema/view/LogicalViewSchema.java | 224 ++
.../schema/view/viewExpression/ViewExpression.java | 318 +++
.../view/viewExpression/ViewExpressionType.java | 76 +
.../binary/BinaryViewExpression.java | 119 +
.../binary/arithmetic/AdditionViewExpression.java | 61 +
.../arithmetic/ArithmeticBinaryViewExpression.java | 52 +
.../binary/arithmetic/DivisionViewExpression.java | 61 +
.../binary/arithmetic/ModuloViewExpression.java | 61 +
.../arithmetic/MultiplicationViewExpression.java | 63 +
.../arithmetic/SubtractionViewExpression.java | 61 +
.../compare/CompareBinaryViewExpression.java | 52 +
.../binary/compare/EqualToViewExpression.java | 57 +
.../binary/compare/GreaterEqualViewExpression.java | 57 +
.../binary/compare/GreaterThanViewExpression.java | 57 +
.../binary/compare/LessEqualViewExpression.java | 57 +
.../binary/compare/LessThanViewExpression.java | 57 +
.../binary/compare/NonEqualViewExpression.java | 57 +
.../binary/logic/LogicAndViewExpression.java | 57 +
.../binary/logic/LogicBinaryViewExpression.java | 52 +
.../binary/logic/LogicOrViewExpression.java | 57 +
.../viewExpression/leaf/ConstantViewOperand.java | 98 +
.../view/viewExpression/leaf/LeafViewOperand.java | 19 +-
.../view/viewExpression/leaf/NullViewOperand.java | 61 +
.../viewExpression/leaf/TimeSeriesViewOperand.java | 87 +
.../viewExpression/leaf/TimestampViewOperand.java | 72 +
.../multi/FunctionViewExpression.java | 198 ++
.../ternary/BetweenViewExpression.java | 111 +
.../ternary/TernaryViewExpression.java | 105 +
.../viewExpression/unary/InViewExpression.java | 106 +
.../viewExpression/unary/IsNullViewExpression.java | 89 +
.../viewExpression/unary/LikeViewExpression.java | 163 ++
.../unary/LogicNotViewExpression.java | 61 +
.../unary/NegationViewExpression.java | 61 +
.../unary/RegularViewExpression.java | 105 +
.../viewExpression/unary/UnaryViewExpression.java | 73 +
.../visitor/ViewExpressionVisitor.java | 219 ++
.../iotdb/commons/sync/pipe/SyncOperation.java | 1 -
.../BuiltinTimeSeriesGeneratingFunction.java | 10 +
.../commons/udf/service/UDFClassLoaderManager.java | 6 +-
.../org/apache/iotdb/commons/utils/AuthUtils.java | 96 +-
.../org/apache/iotdb/commons/utils/IOUtils.java | 8 +-
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 12 +-
.../apache/iotdb/commons/path/PartialPathTest.java | 2 +-
.../org/apache/iotdb/pipe/api/PipeCollector.java | 100 +
.../org/apache/iotdb/pipe/api/PipeConnector.java | 6 +-
.../org/apache/iotdb/pipe/api/PipeProcessor.java | 6 +-
.../iotdb/pipe/api/collector/EventCollector.java | 6 +-
.../iotdb/pipe/api/collector/RowCollector.java | 2 +-
.../iotdb/pipe/api/customizer/PipeParameters.java | 22 +
.../PipeCollectorRuntimeConfiguration.java | 14 +-
.../org/apache/iotdb/pipe/api/event/Event.java | 6 +-
.../org/apache/iotdb/pipe/api/event/EventType.java | 8 +-
.../event/{ => dml}/deletion/DeletionEvent.java | 8 +-
.../{ => dml}/insertion/TabletInsertionEvent.java | 8 +-
.../{ => dml}/insertion/TsFileInsertionEvent.java | 8 +-
.../api/exception/PipeConnectionException.java | 13 +-
.../exception/PipeRuntimeCriticalException.java | 21 +-
.../pipe/api/exception/PipeRuntimeException.java | 21 +-
.../exception/PipeRuntimeNonCriticalException.java | 20 +-
pom.xml | 12 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 24 +-
.../rocksdb/mnode/RMeasurementMNode.java | 5 +
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 24 +-
server/pom.xml | 6 +-
.../resources/conf/iotdb-datanode.properties | 17 +-
.../src/assembly/resources/sbin/stop-datanode.sh | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 8 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 10 +-
.../apache/iotdb/db/auth/AuthorizerManager.java | 15 +-
.../iotdb/db/auth/ClusterAuthorityFetcher.java | 38 +-
.../apache/iotdb/db/auth/IAuthorityFetcher.java | 3 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 44 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 139 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 84 +-
.../directories/strategy/SequenceStrategy.java | 2 +-
.../statemachine/DataRegionStateMachine.java | 3 +-
.../iotdb/db/engine/TsFileMetricManager.java | 63 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 7 +-
.../compaction/constant/CompactionTaskStatus.java} | 9 +-
.../compaction/constant/CompactionTaskType.java} | 10 +-
.../CompactionMemoryNotEnoughException.java} | 9 +-
.../performer/impl/FastCompactionPerformer.java | 30 +-
.../impl/ReadChunkCompactionPerformer.java | 11 +-
.../impl/ReadPointCompactionPerformer.java | 24 +-
.../execute/task/AbstractCompactionTask.java | 13 +-
.../execute/task/CompactionTaskSummary.java | 18 +
.../execute/task/CrossSpaceCompactionTask.java | 17 +-
.../execute/task/InnerSpaceCompactionTask.java | 7 +-
.../compaction/execute/utils/CompactionUtils.java | 22 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 4 +-
.../fast/AlignedSeriesCompactionExecutor.java | 19 +-
.../readchunk/AlignedSeriesCompactionExecutor.java | 6 -
.../readchunk/SingleSeriesCompactionExecutor.java | 38 +-
.../utils/writer/AbstractCompactionWriter.java | 10 +-
.../compaction/schedule/CompactionTaskManager.java | 70 +-
.../compaction/schedule/CompactionWorker.java | 3 -
.../estimator/AbstractCompactionEstimator.java | 7 +
.../ReadPointCrossCompactionEstimator.java | 110 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 13 +-
.../utils/CrossCompactionTaskResource.java | 12 +
.../iotdb/db/engine/flush/MemTableFlushTask.java | 2 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 87 +-
.../db/engine/storagegroup/TsFileProcessor.java | 17 +-
.../db/engine/storagegroup/TsFileResource.java | 5 +
...tion.java => SchemaQuotaExceededException.java} | 24 +-
.../template/TemplateImcompatibeException.java | 19 +-
.../apache/iotdb/db/metadata/MetadataConstant.java | 6 +
.../db/metadata/cache/DataNodeDevicePathCache.java | 69 +
.../db/metadata/cache/DataNodeSchemaCache.java | 229 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 31 +-
.../cache/DeviceUsingTemplateSchemaCache.java | 193 ++
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 36 +-
...SchemaCache.java => TimeSeriesSchemaCache.java} | 112 +-
.../metadata/cache/dualkeycache/IDualKeyCache.java | 4 +
.../dualkeycache/impl/CacheEntryGroupImpl.java | 2 +-
.../dualkeycache/impl/DualKeyCacheBuilder.java | 18 +-
.../cache/dualkeycache/impl/DualKeyCacheImpl.java | 102 +-
.../dualkeycache/impl/DualKeyCachePolicy.java | 3 +-
.../dualkeycache/impl/FIFOCacheEntryManager.java | 189 ++
.../mnode/config/factory/ConfigMNodeFactory.java | 7 +
.../mnode/mem/factory/MemMNodeFactory.java | 14 +
...MeasurementMNode.java => LogicalViewMNode.java} | 31 +-
.../metadata/mnode/mem/impl/MeasurementMNode.java | 5 +
.../metadata/mnode/mem/info/LogicalViewInfo.java | 163 ++
.../schemafile/factory/CacheMNodeFactory.java | 7 +
.../schemafile/impl/CachedMeasurementMNode.java | 5 +
.../iotdb/db/metadata/mtree/ConfigMTree.java | 4 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 40 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 103 +-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 48 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 3 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +
.../mtree/store/disk/cache/CacheMemoryManager.java | 4 +-
.../store/disk/schemafile/SchemaFileConfig.java | 2 +-
.../mtree/store/disk/schemafile/SegmentedPage.java | 1 +
.../store/disk/schemafile/pagemgr/PageManager.java | 90 +-
.../db/metadata/mtree/traverser/Traverser.java | 61 +
.../traverser/TraverserWithLimitOffsetWrapper.java | 5 +
.../mtree/traverser/basic/DatabaseTraverser.java | 5 +
.../mtree/traverser/basic/EntityTraverser.java | 8 +
.../mtree/traverser/basic/MNodeTraverser.java | 5 +
.../traverser/basic/MeasurementTraverser.java | 5 +
.../plan/schemaregion/SchemaRegionPlanType.java | 2 +
.../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 +
.../impl/SchemaRegionPlanDeserializer.java | 24 +
.../impl/SchemaRegionPlanSerializer.java | 27 +
.../impl/SchemaRegionPlanTxtSerializer.java | 21 +
.../impl/write/CreateLogicalViewPlanImpl.java | 75 +
.../schemaregion/result/ShowTimeSeriesResult.java | 13 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 66 +
.../metadata/query/info/ITimeSeriesSchemaInfo.java | 6 +-
.../rescon/DataNodeSchemaQuotaManager.java | 92 +
.../metadata/rescon/MemSchemaEngineStatistics.java | 26 +-
.../metadata/rescon/MemSchemaRegionStatistics.java | 8 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 23 +-
.../db/metadata/schemaregion/SchemaEngine.java | 100 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 123 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 95 +-
.../metadata/template/ClusterTemplateManager.java | 306 ++-
.../db/metadata/template/ITemplateManager.java | 10 +-
.../iotdb/db/metadata/template/Template.java | 16 +-
...teType.java => TemplateAlterOperationType.java} | 24 +-
.../template/TemplateInternalRPCUpdateType.java | 20 +-
.../metadata/template/TemplateInternalRPCUtil.java | 50 +-
.../template/alter/TemplateAlterInfo.java} | 24 +-
.../template/alter/TemplateAlterOperationUtil.java | 51 +
.../template/alter/TemplateExtendInfo.java | 161 ++
.../apache/iotdb/db/metadata/utils/MetaUtils.java | 11 +-
.../visitor/GetSourcePathsVisitor.java | 96 +
.../visitor/TransformToExpressionVisitor.java | 321 +++
.../metadata/visitor/SchemaExecutionVisitor.java | 22 +
.../org/apache/iotdb/db/mpp/common/QueryId.java | 2 +-
.../db/mpp/common/header/ColumnHeaderConstant.java | 19 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../mpp/common/schematree/ClusterSchemaTree.java | 6 +-
.../db/mpp/common/schematree/DeviceSchemaInfo.java | 18 +-
.../common/schematree/IMeasurementSchemaInfo.java | 8 +-
.../common/schematree/MeasurementSchemaInfo.java | 22 +-
.../schematree/node/SchemaMeasurementNode.java | 42 +-
.../visitor/SchemaTreeDeviceVisitor.java | 5 +
.../visitor/SchemaTreeMeasurementVisitor.java | 5 +
.../SchemaTreeVisitorWithLimitOffsetWrapper.java | 6 +
.../iotdb/db/mpp/execution/QueryStateMachine.java | 78 +-
.../iotdb/db/mpp/execution/driver/Driver.java | 28 +-
.../db/mpp/execution/driver/DriverContext.java | 9 +
.../execution/exchange/MPPDataExchangeManager.java | 8 +-
.../mpp/execution/exchange/SharedTsBlockQueue.java | 12 +-
.../execution/exchange/sink/LocalSinkChannel.java | 23 +-
.../execution/exchange/sink/ShuffleSinkHandle.java | 7 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 44 +-
.../execution/exchange/source/SourceHandle.java | 25 +-
.../execution/executor/RegionWriteExecutor.java | 133 +-
.../fragment/FragmentInstanceContext.java | 10 +
.../fragment/FragmentInstanceExecution.java | 17 +
.../fragment/FragmentInstanceManager.java | 8 +-
.../operator/process/DeviceViewIntoOperator.java | 6 +-
.../operator/process/MergeSortOperator.java | 9 +-
.../execution/operator/process/SortOperator.java | 258 +-
.../operator/process/TransformOperator.java | 13 +-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../process/join/merge/MergeSortComparator.java | 199 +-
.../process/join/merge/SortKeyComparator.java | 52 +
.../operator/schema/SchemaCountOperator.java | 23 +-
.../operator/schema/SchemaQueryScanOperator.java | 2 +-
.../operator/schema/source/DeviceSchemaSource.java | 12 +
.../operator/schema/source/ISchemaSource.java | 4 +
.../operator/schema/source/NodeSchemaSource.java | 10 +
.../schema/source/PathsUsingTemplateSource.java | 10 +
.../schema/source/SchemaSourceFactory.java | 5 +
.../schema/source/TimeSeriesSchemaSource.java | 26 +-
.../execution/operator/source/SeriesScanUtil.java | 58 +-
.../execution/schedule/AbstractDriverThread.java | 9 +
.../db/mpp/execution/schedule/DriverScheduler.java | 116 +-
.../mpp/execution/schedule/DriverTaskThread.java | 7 +-
.../schedule/queue/IndexedBlockingQueue.java | 1 +
.../multilevelqueue/MultilevelPriorityQueue.java | 4 +-
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 140 +-
.../org/apache/iotdb/db/mpp/plan/Coordinator.java | 8 +-
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 68 +
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 327 ++-
.../db/mpp/plan/analyze/ConcatPathRewriter.java | 22 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 40 +-
.../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 3 +-
.../db/mpp/plan/analyze/cache/PartitionCache.java | 10 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 254 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 12 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 190 +-
.../db/mpp/plan/analyze/schema/ISchemaFetcher.java | 7 +-
.../plan/analyze/schema/NormalSchemaFetcher.java | 203 ++
.../plan/analyze/schema/TemplateSchemaFetcher.java | 246 ++
.../db/mpp/plan/execution/IQueryExecution.java | 4 +-
.../db/mpp/plan/execution/QueryExecution.java | 57 +-
.../mpp/plan/execution/config/ConfigExecution.java | 5 +-
.../plan/execution/config/ConfigTaskVisitor.java | 17 +-
.../config/executor/ClusterConfigTaskExecutor.java | 132 +-
.../config/executor/IConfigTaskExecutor.java | 10 +-
...lotListTask.java => CountTimeSlotListTask.java} | 38 +-
.../config/metadata/GetTimeSlotListTask.java | 9 +-
.../config/metadata/ShowPipePluginsTask.java | 18 +-
...plateTask.java => AlterSchemaTemplateTask.java} | 15 +-
.../metadata/template/SetSchemaTemplateTask.java | 7 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 7 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 34 +-
.../visitor/ColumnTransformerVisitor.java | 40 +-
.../visitor/IntermediateLayerVisitor.java | 4 +-
.../visitor/TransformToViewExpressionVisitor.java | 391 +++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 345 ++-
.../db/mpp/plan/parser/StatementGenerator.java | 45 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 187 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 40 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 46 +-
.../planner/distribution/DistributionPlanner.java | 5 +-
.../planner/distribution/ExchangeNodeAdder.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 52 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/load/LoadSingleTsFileNode.java | 59 +-
.../planner/plan/node/load/LoadTsFileNode.java | 16 +-
.../node/metedata/write/CreateLogicalViewNode.java | 250 ++
.../write/InternalCreateMultiTimeSeriesNode.java | 1 +
.../source/AlignedSeriesAggregationScanNode.java | 50 +-
.../plan/node/source/AlignedSeriesScanNode.java | 4 +
.../node/source/SeriesAggregationScanNode.java | 50 +-
.../node/source/SeriesAggregationSourceNode.java | 49 +-
.../planner/plan/node/source/SeriesScanNode.java | 4 +
.../plan/planner/plan/node/write/InsertNode.java | 9 +
.../planner/plan/node/write/InsertRowNode.java | 6 +-
.../planner/plan/node/write/InsertRowsNode.java | 5 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 7 +
.../planner/plan/node/write/InsertTabletNode.java | 10 +-
.../planner/plan/parameter/SeriesScanOptions.java | 5 +-
.../db/mpp/plan/scheduler/AsyncPlanNodeSender.java | 74 +-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 27 +-
.../db/mpp/plan/scheduler/ClusterScheduler.java | 4 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 20 +-
.../db/mpp/plan/scheduler/IQueryTerminator.java | 2 +-
.../iotdb/db/mpp/plan/scheduler/IScheduler.java | 2 +-
.../mpp/plan/scheduler/SimpleQueryTerminator.java | 35 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 20 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 203 +-
.../iotdb/db/mpp/plan/statement/StatementType.java | 3 +
.../db/mpp/plan/statement/StatementVisitor.java | 19 +
.../component/{SortKey.java => NullOrdering.java} | 19 +-
.../plan/statement/component/OrderByComponent.java | 56 +-
.../component/{SortKey.java => OrderByKey.java} | 16 +-
.../db/mpp/plan/statement/component/SortItem.java | 61 +-
.../plan/statement/crud/InsertBaseStatement.java | 4 -
.../crud/InsertMultiTabletsStatement.java | 18 -
.../plan/statement/crud/InsertRowStatement.java | 19 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 9 -
.../plan/statement/crud/InsertRowsStatement.java | 17 -
.../plan/statement/crud/InsertTabletStatement.java | 13 -
.../db/mpp/plan/statement/crud/QueryStatement.java | 80 +
...tement.java => CountTimeSlotListStatement.java} | 59 +-
.../metadata/CreateLogicalViewStatement.java | 246 ++
.../statement/metadata/GetRegionIdStatement.java | 54 +-
.../metadata/GetSeriesSlotListStatement.java | 20 +-
.../metadata/GetTimeSlotListStatement.java | 46 +-
.../template/AlterSchemaTemplateStatement.java | 85 +
.../plan/statement/sys/ShowQueriesStatement.java | 4 +-
.../dag/builder/EvaluationDAGBuilder.java | 4 +-
.../dag/input/QueryDataSetInputLayer.java | 5 +-
.../intermediate/ConstantIntermediateLayer.java | 3 +-
.../dag/intermediate/IntermediateLayer.java | 4 +-
.../MultiInputColumnIntermediateLayer.java | 2 +-
...InputColumnMultiReferenceIntermediateLayer.java | 2 +-
...nputColumnSingleReferenceIntermediateLayer.java | 2 +-
.../db/mpp/transformation/dag/udf/UDTFContext.java | 2 +-
.../mpp/transformation/dag/udf/UDTFExecutor.java | 2 +-
.../datastructure/SerializableList.java | 6 +-
.../row/ElasticSerializableRowRecordList.java | 6 +-
.../row/SerializableRowRecordList.java | 2 +-
.../tv/ElasticSerializableBinaryTVList.java | 2 +-
.../tv/ElasticSerializableTVList.java | 8 +-
.../datastructure/tv/SerializableTVList.java | 2 +-
.../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 11 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 89 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 18 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 504 +++-
.../PipeCollectorConstant.java} | 15 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 71 +
.../config/PipeConnectorConstant.java} | 11 +-
.../config/PipeProcessorConstant.java} | 11 +-
.../core/collector/IoTDBDataRegionCollector.java | 85 +
.../core/collector/PipeCollectorEventSelector.java | 22 -
.../historical/PipeHistoricalCollector.java | 22 -
.../PipeHistoricalDataRegionTsFileCollector.java | 103 +
.../collector/realtime/PipeRealtimeCollector.java | 22 -
.../realtime/PipeRealtimeDataRegionCollector.java | 76 +
.../PipeRealtimeDataRegionHybridCollector.java | 201 ++
.../realtime/assigner/DisruptorQueue.java | 111 +
.../realtime/assigner/PipeDataRegionAssigner.java | 84 +
.../realtime/cache/PipeRealtimeEventCache.java | 22 -
.../realtime/listener/IoTLogListerner.java | 22 -
.../listener/PipeInsertionDataNodeListener.java | 115 +
.../realtime/listener/RatisLogListener.java | 22 -
.../realtime/listener/SimpleLogListener.java | 22 -
.../listener/TsFileGenerationListener.java | 22 -
.../matcher/CachedSchemaPatternMatcher.java | 200 ++
.../realtime/matcher/PipeDataRegionMatcher.java | 52 +
.../pipe/core/collector/realtime/matcher/Rule.java | 22 -
.../realtime/matcher/RulePrefixMatchTree.java | 22 -
.../collector/realtime/recorder/TsFileEpoch.java | 22 -
.../realtime/recorder/TsFileEpochRecorder.java | 22 -
.../core/connector/PipeConnectorContainer.java | 22 -
.../pipe/core/connector/PipeConnectorManager.java | 22 -
.../PipeConnectorPluginRuntimeWrapper.java | 73 -
.../connector/PipeConnectorSubtaskLifeCycle.java | 125 +
.../connector/PipeConnectorSubtaskManager.java | 122 +
.../iotdb/db/pipe/core/event/EnrichedEvent.java | 54 +
.../core/event/collector/PipeEventCollector.java | 39 -
.../event/{ => impl}/PipeTabletInsertionEvent.java | 49 +-
.../core/event/impl/PipeTsFileInsertionEvent.java | 92 +
.../pipe/core/event/indexer/PipeEventIndexer.java | 22 -
.../core/event/indexer/PipeIoTEventIndexer.java | 22 -
.../core/event/indexer/PipeRatisEventIndexer.java | 22 -
.../core/event/indexer/PipeSimpleEventIndexer.java | 22 -
.../core/event/indexer/PipeTsFileEventIndexer.java | 22 -
.../event/realtime/PipeRealtimeCollectEvent.java | 84 +
.../realtime/PipeRealtimeCollectEventFactory.java | 47 +
.../db/pipe/core/event/realtime/TsFileEpoch.java | 68 +
.../core/event/realtime/TsFileEpochManager.java | 70 +
.../event/realtime/TsFileEpochStateMigrator.java} | 9 +-
.../pipe/core/event/{ => view}/access/PipeRow.java | 2 +-
.../event/{ => view}/access/PipeRowIterator.java | 2 +-
.../event/view/collector/PipeEventCollector.java | 96 +
.../{ => view}/collector/PipeRowCollector.java | 2 +-
.../PipeProcessorPluginRuntimeWrapper.java | 77 -
.../execution/executor/PipeSubtaskExecutor.java | 12 +-
...anager.java => PipeSubtaskExecutorManager.java} | 12 +-
.../execution/scheduler/PipeTaskScheduler.java | 18 +-
.../db/pipe/resource/PipeFileResourceManager.java | 186 ++
.../iotdb/db/pipe/resource/PipeRaftlogHolder.java | 22 -
...peFileManager.java => PipeResourceManager.java} | 19 +-
.../iotdb/db/pipe/resource/PipeTsFileHolder.java | 22 -
.../iotdb/db/pipe/resource/PipeWALHolder.java | 22 -
.../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 66 +
.../org/apache/iotdb/db/pipe/task/PipeTask.java | 9 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 60 +-
.../apache/iotdb/db/pipe/task/PipeTaskManager.java | 96 +
.../task/queue/EventSupplier.java} | 18 +-
.../ListenableBlockingPendingQueue.java} | 15 +-
.../db/pipe/task/queue/ListenablePendingQueue.java | 159 ++
.../ListenableUnblockingPendingQueue.java} | 15 +-
.../PendingQueueEmptyToNotEmptyListener.java} | 14 +-
.../queue/PendingQueueFullToNotFullListener.java | 8 +-
.../PendingQueueNotEmptyToEmptyListener.java} | 8 +-
.../queue/PendingQueueNotFullToFullListener.java} | 8 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 82 +-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 45 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 109 +-
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 101 +-
.../task/{callable => subtask}/DecoratingLock.java | 2 +-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 148 +
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 99 +
.../task/{callable => subtask}/PipeSubtask.java | 14 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 5 +-
.../rest/v1/handler/ExecuteStatementHandler.java | 4 +-
.../rest/v1/handler/QueryDataSetHandler.java | 5 +-
.../v1/handler/StatementConstructionHandler.java | 5 +-
.../rest/v2/handler/ExecuteStatementHandler.java | 4 +-
.../rest/v2/handler/QueryDataSetHandler.java | 5 +-
.../v2/handler/StatementConstructionHandler.java | 5 +-
.../iotdb/db/query/context/QueryContext.java | 2 +
.../db/query/control/QueryResourceManager.java | 7 +-
.../iotdb/db/query/control/SessionManager.java | 7 +
.../query/control/clientsession/ClientSession.java | 3 +-
.../iotdb/db/query/reader/chunk/MemPageReader.java | 12 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 33 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 6 +
.../db/service/TemporaryQueryDataFileService.java | 14 +-
.../db/service/metrics/CompactionMetrics.java | 12 +-
.../iotdb/db/service/metrics/FileMetrics.java | 26 +-
.../metrics/recorder/CompactionMetricsManager.java | 87 +-
.../db/service/thrift/ProcessorWithMetrics.java | 2 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 44 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 244 +-
.../service/thrift/impl/MLNodeRPCServiceImpl.java | 15 +-
.../org/apache/iotdb/db/tools/DiskSpiller.java | 166 ++
.../apache/iotdb/db/tools/FileSpillerReader.java | 143 +
.../MemoryReader.java} | 38 +-
.../apache/iotdb/db/tools/SortBufferManager.java | 73 +
.../java/org/apache/iotdb/db/tools/SortReader.java | 27 +-
.../datastructure/FixedPriorityBlockingQueue.java | 10 +
.../db/utils/datastructure/MergeSortHeap.java | 4 +-
.../iotdb/db/utils/datastructure/MergeSortKey.java | 25 +-
.../{MergeSortKey.java => SortKey.java} | 14 +-
.../java/org/apache/iotdb/db/wal/WALManager.java | 8 +-
.../db/wal/allocation/FirstCreateStrategy.java | 3 +
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 41 +-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 9 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 14 +-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 6 -
.../iotdb/db/auth/AuthorizerManagerTest.java | 26 +-
.../auth/authorizer/LocalFileAuthorizerTest.java | 9 +-
.../iotdb/db/auth/entity/PathPrivilegeTest.java | 8 +-
.../org/apache/iotdb/db/auth/entity/RoleTest.java | 8 +-
.../org/apache/iotdb/db/auth/entity/UserTest.java | 8 +-
.../db/auth/role/LocalFileRoleAccessorTest.java | 6 +-
.../db/auth/role/LocalFileRoleManagerTest.java | 8 +-
.../db/auth/user/LocalFileUserAccessorTest.java | 6 +-
.../db/auth/user/LocalFileUserManagerTest.java | 8 +-
.../engine/compaction/AbstractCompactionTest.java | 9 +-
.../compaction/CompactionTaskComparatorTest.java | 8 +-
.../compaction/FastAlignedCrossCompactionTest.java | 1098 ++++++--
.../FastCrossCompactionPerformerTest.java | 5 +
.../FastInnerCompactionPerformerTest.java | 30 +-
.../FastNonAlignedCrossCompactionTest.java | 1114 ++++++--
.../compaction/ReadChunkInnerCompactionTest.java | 377 +++
...va => ReadPointAlignedCrossCompactionTest.java} | 1151 ++++++--
...=> ReadPointNonAlignedCrossCompactionTest.java} | 1167 ++++++--
.../CrossSpaceCompactionWithFastPerformerTest.java | 7 +-
...eCompactionWithFastPerformerValidationTest.java | 210 +-
...actionWithReadPointPerformerValidationTest.java | 168 +-
.../cross/RewriteCompactionFileSelectorTest.java | 12 +
.../ReadChunkCompactionPerformerNoAlignedTest.java | 274 ++
.../utils/CompactionFileGeneratorUtils.java | 60 +
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../compaction/utils/TsFileGeneratorUtils.java | 116 +-
.../cache/DataNodeDevicePathCacheTest.java | 62 +
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 50 +
.../cache/dualkeycache/DualKeyCacheTest.java | 17 +
.../metadata/mtree/schemafile/SchemaFileTest.java | 137 +-
.../schemaRegion/SchemaRegionBasicTest.java | 103 +-
.../schemaRegion/SchemaRegionManagementTest.java | 4 +-
.../metadata/view/ViewExpressionToStringTest.java | 183 ++
.../apache/iotdb/db/metric/MetricServiceTest.java | 3 -
.../db/mpp/execution/ConfigExecutionTest.java | 2 +-
.../db/mpp/execution/QueryStateMachineTest.java | 2 +
.../execution/operator/MergeSortOperatorTest.java | 61 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 48 +-
.../mpp/execution/operator/SortOperatorTest.java | 273 ++
.../schema/SchemaFetchScanOperatorTest.java | 3 +-
.../schema/SchemaQueryScanOperatorTest.java | 4 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 200 ++
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 11 +-
.../mpp/plan/analyze/QueryTimePartitionTest.java | 14 +-
.../db/mpp/plan/optimization/TestPlanBuilder.java | 6 +-
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 16 +-
.../iotdb/db/mpp/plan/plan/distribution/Util.java | 11 +-
.../node/process/AggregationNodeSerdeTest.java | 7 +-
.../plan/node/process/DeviceViewNodeSerdeTest.java | 6 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 8 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +-
.../plan/plan/node/process/SortNodeSerdeTest.java | 4 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 7 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 15 +-
.../collector/CachedSchemaPatternMatcherTest.java | 175 ++
.../core/collector/PipeRealtimeCollectTest.java | 304 +++
.../executor/PipeAssignerSubtaskExecutorTest.java | 40 -
.../executor/PipeConnectorSubtaskExecutorTest.java | 12 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 14 +-
.../executor/PipeSubtaskExecutorTest.java | 2 +-
.../pipe/resource/PipeFileResourceManagerTest.java | 218 ++
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 36 +-
.../series/SeriesScanLimitOffsetPushDownTest.java | 23 +-
.../udf/datastructure/SerializableListTest.java | 2 +-
.../security/encrypt/MessageDigestEncryptTest.java | 6 +-
.../DataNodeInternalRPCServiceImplTest.java | 31 +-
.../sync/pipedata/BufferedPipeDataQueueTest.java | 658 -----
server/src/test/resources/logback-test.xml | 1 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
site/src/main/.vuepress/sidebar/V0.13.x/en.ts | 3 +-
site/src/main/.vuepress/sidebar/V0.13.x/zh.ts | 3 +-
site/src/main/.vuepress/styles/index.scss | 6 +-
site/src/main/package-lock.json | 2881 +++++++-------------
.../src/main/thrift/confignode.thrift | 54 +-
thrift/src/main/thrift/client.thrift | 3 +
thrift/src/main/thrift/datanode.thrift | 66 +-
.../tsfile/file/metadata/enums/TSDataType.java | 17 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 13 +-
.../read/filter/PredicateRemoveNotRewriter.java | 50 +
.../iotdb/tsfile/read/filter/TimeFilter.java | 158 +-
.../iotdb/tsfile/read/filter/ValueFilter.java | 179 +-
.../tsfile/read/filter/basic/BinaryFilter.java | 5 -
.../iotdb/tsfile/read/filter/basic/Filter.java | 11 +-
.../tsfile/read/filter/factory/FilterFactory.java | 10 +-
.../tsfile/read/filter/operator/AndFilter.java | 11 +
.../iotdb/tsfile/read/filter/operator/Between.java | 127 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 25 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 23 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 23 +-
.../iotdb/tsfile/read/filter/operator/In.java | 34 +-
.../iotdb/tsfile/read/filter/operator/Like.java | 55 +-
.../iotdb/tsfile/read/filter/operator/Lt.java | 23 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 23 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 25 +-
.../tsfile/read/filter/operator/NotFilter.java | 26 +-
.../tsfile/read/filter/operator/OrFilter.java | 11 +
.../iotdb/tsfile/read/filter/operator/Regexp.java | 57 +-
.../read/reader/chunk/AlignedChunkReader.java | 40 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 4 +
.../iotdb/tsfile/read/reader/page/PageReader.java | 12 +-
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 42 +-
.../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 4 -
.../tsfile/write/schema/IMeasurementSchema.java | 4 +
.../tsfile/write/schema/MeasurementSchema.java | 10 +
.../tsfile/write/schema/MeasurementSchemaType.java | 20 +-
.../write/schema/VectorMeasurementSchema.java | 10 +
.../tsfile/read/filter/FilterSerializeTest.java | 21 +-
.../read/filter/MinTimeMaxTimeFilterTest.java | 42 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 4 +-
.../filter/PredicateRemoveNotRewriterTest.java | 121 +
.../tsfile/read/filter/StatisticsFilterTest.java | 113 +-
832 files changed, 38659 insertions(+), 13534 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
index 074753bb60c,a214522ca65..0abc2db4642
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java
@@@ -382,17 -374,13 +385,17 @@@ public abstract class Driver implement
Throwable inFlightException = null;
try {
+ long startTime = System.nanoTime();
root.close();
- long endTime = System.nanoTime();
- driverContext
- .getFragmentInstanceContext()
- .addOperationTime(DRIVER_CLOSE, endTime - startTime);
- startTime = endTime;
+
+ if (driverContext.mayHaveTmpFile()) {
+ cleanTmpFile();
+ }
+
sink.setNoMoreTsBlocks();
+ driverContext
+ .getFragmentInstanceContext()
+ .addOperationTime(SET_NO_MORE_TSBLOCK, System.nanoTime() - startTime);
// record operator execution statistics to metrics
List<OperatorContext> operatorContexts = driverContext.getOperatorContexts();
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/sink/LocalSinkChannel.java
index c094ef93161,aae1f8f37c7..a2098376be4
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/sink/LocalSinkChannel.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/sink/LocalSinkChannel.java
@@@ -33,11 -32,9 +33,12 @@@ import org.slf4j.LoggerFactory
import java.util.Optional;
+ import static com.google.common.util.concurrent.Futures.immediateVoidFuture;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SINK_HANDLE_SEND_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CHECK_AND_INVOKE_ON_FINISHED;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.SINK_HANDLE_END_LISTENER;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.SINK_HANDLE_FINISH_LISTENER;
public class LocalSinkChannel implements ISinkChannel {
@@@ -108,9 -108,10 +114,12 @@@
synchronized (queue) {
if (isFinished()) {
synchronized (this) {
- long start = System.nanoTime();
- sinkListener.onFinish(this);
- QUERY_STATISTICS.addCost(SINK_HANDLE_FINISH_LISTENER, System.nanoTime() - start);
+ if (!invokedOnFinished) {
++ long start = System.nanoTime();
+ sinkListener.onFinish(this);
+ invokedOnFinished = true;
++ QUERY_STATISTICS.addCost(SINK_HANDLE_FINISH_LISTENER, System.nanoTime() - start);
+ }
}
}
}
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
index b7fd82d964f,01882493a5d..29c142d6c3f
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
@@@ -386,7 -378,11 +388,15 @@@ public class FragmentInstanceContext ex
sharedQueryDataSource = null;
}
+ public void setMayHaveTmpFile(boolean mayHaveTmpFile) {
+ this.mayHaveTmpFile = mayHaveTmpFile;
+ }
+
+ public boolean mayHaveTmpFile() {
+ return mayHaveTmpFile;
+ }
++
+ public void addOperationTime(String key, long costTimeInNanos) {
+ QUERY_STATISTICS.addCost(key, costTimeInNanos);
+ }
}
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
index 2bebeef7349,d3988be3667..3ba70b00209
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
@@@ -300,27 -285,20 +300,25 @@@ public class SeriesScanUtil
}
protected void filterFirstChunkMetadata() throws IOException {
+ long startTime = System.nanoTime();
+ try {
- if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) {
- Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstChunkMetadata.getStatistics())) {
- skipCurrentChunk();
- }
- // TODO implement allSatisfied interface for filter, then we can still skip offset.
- } else {
- long rowCount = firstChunkMetadata.getStatistics().getCount();
- if (paginationController.hasCurOffset(rowCount)) {
- skipCurrentChunk();
- paginationController.consumeOffset(rowCount);
- }
+ if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) {
+ Filter queryFilter = scanOptions.getQueryFilter();
+ Statistics statistics = firstChunkMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ skipCurrentChunk();
+ paginationController.consumeOffset(rowCount);
}
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentChunk();
}
+ }
+ } finally {
- QueryStatistics.getInstance()
- .addCost(QueryStatistics.FILTER_FIRST_CHUNK_METADATA, System.nanoTime() - startTime);
++ QueryStatistics.getInstance()
++ .addCost(QueryStatistics.FILTER_FIRST_CHUNK_METADATA, System.nanoTime() - startTime);
+ }
}
/** construct first chunk metadata */
@@@ -1084,29 -1035,21 +1082,27 @@@
}
protected void filterFirstTimeSeriesMetadata() throws IOException {
+ long startTime = System.nanoTime();
+ try {
- if (firstTimeSeriesMetadata != null
- && !isFileOverlapped()
- && !firstTimeSeriesMetadata.isModified()) {
- Filter queryFilter = scanOptions.getQueryFilter();
- if (queryFilter != null) {
- if (!queryFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) {
- skipCurrentFile();
- }
- // TODO implement allSatisfied interface for filter, then we can still skip offset.
- } else {
- long rowCount = firstTimeSeriesMetadata.getStatistics().getCount();
- if (paginationController.hasCurOffset(rowCount)) {
- skipCurrentFile();
- paginationController.consumeOffset(rowCount);
- }
+ if (firstTimeSeriesMetadata != null
+ && !isFileOverlapped()
+ && !firstTimeSeriesMetadata.isModified()) {
+ Filter queryFilter = scanOptions.getQueryFilter();
+ Statistics statistics = firstTimeSeriesMetadata.getStatistics();
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ long rowCount = statistics.getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ skipCurrentFile();
+ paginationController.consumeOffset(rowCount);
}
+ } else if (!queryFilter.satisfy(statistics)) {
+ skipCurrentFile();
}
+ }
+ } finally {
+ QueryStatistics.getInstance()
- .addCost(QueryStatistics.FILTER_FIRST_TIMESERIES_METADATA, System.nanoTime() - startTime);
++ .addCost(QueryStatistics.FILTER_FIRST_TIMESERIES_METADATA, System.nanoTime() - startTime);
+ }
}
protected void unpackAllOverlappedTsFilesToTimeSeriesMetadata(long endpointTime)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
index f3f3a0be68b,715cf29c716..eafa3a6e2b8
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
@@@ -61,8 -63,8 +63,9 @@@ import org.apache.iotdb.db.mpp.plan.sta
import org.apache.iotdb.db.mpp.plan.statement.crud.InsertMultiTabletsStatement;
import org.apache.iotdb.db.mpp.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.mpp.plan.statement.crud.LoadTsFileStatement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.utils.SetThreadName;
+ import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
diff --cc server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
index 551f790cf9e,5d1456935b3..b695b29c920
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
@@@ -206,8 -198,8 +206,9 @@@ public class ClientRPCServiceImpl imple
}
long startTime = System.currentTimeMillis();
+ long startTimeInNano = System.nanoTime();
StatementType statementType = null;
+ Throwable t = null;
try {
Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId());
@@@ -606,10 -605,9 +631,10 @@@
statementType,
COORDINATOR.getTotalExecutionTime(req.queryId));
}
- COORDINATOR.cleanupQueryExecution(req.queryId);
+ COORDINATOR.cleanupQueryExecution(req.queryId, t);
}
SESSION_MANAGER.updateIdleTime();
+ QueryStatistics.getInstance().addCost(SERVER_RPC_RT, System.nanoTime() - startTimeNanos);
}
}
@@@ -1089,9 -1082,9 +1114,10 @@@
@Override
public TSFetchResultsResp fetchResults(TSFetchResultsReq req) {
boolean finished = false;
+ long startTimeNanos = System.nanoTime();
long startTime = System.currentTimeMillis();
StatementType statementType = null;
+ Throwable t = null;
try {
IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime();
if (!SESSION_MANAGER.checkLogin(clientSession)) {
@@@ -1132,10 -1129,9 +1162,10 @@@
statementType,
COORDINATOR.getTotalExecutionTime(req.queryId));
}
- COORDINATOR.cleanupQueryExecution(req.queryId);
+ COORDINATOR.cleanupQueryExecution(req.queryId, t);
}
SESSION_MANAGER.updateIdleTime();
+ QueryStatistics.getInstance().addCost(SERVER_RPC_RT, System.nanoTime() - startTimeNanos);
}
}