You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/05/18 02:35:33 UTC
[iotdb] 01/02: Merge branch 'master' into xingtanzjr/distribution_plan_0516
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/distribution_plan_0516
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 79b3d0825ef521e09d07125a05688bc73ce369dd
Merge: 083e724053 0378bbe78a
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Wed May 18 10:34:15 2022 +0800
Merge branch 'master' into xingtanzjr/distribution_plan_0516
antlr/pom.xml | 1 +
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 179 ++++++++
.../apache/iotdb/db/qp/sql/InfluxDBSqlParser.g4 | 14 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 9 +-
.../org/apache/iotdb/db/qp/sql/PathParser.g4 | 36 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +-
cli/src/assembly/resources/sbin/start-cli.bat | 2 +-
cli/src/assembly/resources/sbin/start-cli.sh | 8 +-
client-py/tests/test_dataframe.py | 10 +-
.../cluster/client/sync/SyncClientAdaptorTest.java | 2 +-
.../resources/conf/iotdb-confignode.properties | 4 +-
.../assembly/resources/sbin/start-confignode.bat | 2 +-
.../assembly/resources/sbin/start-confignode.sh | 8 +-
.../assembly/resources/sbin/stop-confignode.bat | 2 +-
.../confignode/client/AsyncDataNodeClientPool.java | 128 +++++-
.../iotdb/confignode/conf/ConfigNodeConf.java | 4 +-
.../consensus/request/write/CreateRegionsReq.java | 32 +-
.../request/write/RegisterDataNodeReq.java | 22 +-
...deLocationsResp.java => DataNodeInfosResp.java} | 18 +-
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../iotdb/confignode/manager/NodeManager.java | 40 +-
.../iotdb/confignode/manager/PartitionManager.java | 64 ++-
.../iotdb/confignode/manager/load/LoadManager.java | 267 ++++-------
.../manager/load/balancer/RegionBalancer.java | 94 +++-
.../allocator/CopySetRegionAllocator.java | 25 +-
.../{ => balancer}/allocator/IRegionAllocator.java | 6 +-
.../confignode/persistence/ClusterSchemaInfo.java | 27 +-
.../iotdb/confignode/persistence/NodeInfo.java | 79 ++--
.../confignode/persistence/PartitionInfo.java | 107 ++++-
.../impl/DeleteStorageGroupProcedure.java | 13 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 13 +-
.../consensus/request/ConfigRequestSerDeTest.java | 9 +-
.../persistence/ClusterSchemaInfoTest.java | 2 +-
.../iotdb/confignode/persistence/NodeInfoTest.java | 12 +-
.../confignode/persistence/PartitionInfoTest.java | 54 ++-
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 37 +-
.../ratis/ApplicationStateMachineProxy.java | 9 +
distribution/pom.xml | 22 -
distribution/src/assembly/cluster.xml | 81 ----
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 41 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 42 +-
.../db/integration/IoTDBContinuousQueryIT.java | 42 ++
.../db/integration/IoTDBCreateTimeseriesIT.java | 2 +-
.../iotdb/db/integration/IoTDBQuotedPathIT.java | 2 +-
.../iotdb/db/integration/IoTDBRestartIT.java | 101 ++--
.../iotdb/db/integration/IoTDBSelectIntoIT.java | 2 +-
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 4 +-
.../IoTDBSyntaxConventionIdentifierIT.java | 160 ++++++-
.../db/integration/sync/IoTDBSyncReceiverIT.java | 32 +-
.../IoTDBWithoutNullAnyFilterIT.java | 10 +-
.../iotdb/session/IoTDBSessionComplexIT.java | 8 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 8 +-
.../session/IoTDBSessionSyntaxConventionIT.java | 18 +-
.../java/org/apache/iotdb/session/SessionTest.java | 28 +-
.../apache/iotdb/session/template/TemplateUT.java | 14 +-
.../main/assembly/resources/conf/iotdb-metric.yml | 3 +
.../iotdb/metrics/DoNothingMetricService.java | 10 +
.../org/apache/iotdb/metrics/MetricService.java | 13 +
.../apache/iotdb/metrics/config/MetricConfig.java | 11 +
node-commons/pom.xml | 5 +
.../org/apache/iotdb/commons/path/PartialPath.java | 51 +-
.../org/apache/iotdb/commons/utils/PathUtils.java | 78 +---
.../commons/utils/ThriftCommonsSerDeUtils.java | 19 +
.../apache/iotdb/commons/path/PartialPathTest.java | 511 +++++++++++++++++++++
.../apache/iotdb/commons/utils/PathUtilsTest.java | 125 -----
.../schemaregion/rocksdb/RSchemaRegion.java | 2 +-
.../schemaregion/rocksdb/RSchemaUtils.java | 2 +-
server/pom.xml | 5 -
.../resources/conf/iotdb-engine.properties | 4 -
.../assembly/resources/sbin/remove-datanode.bat | 2 +-
.../src/assembly/resources/sbin/remove-datanode.sh | 8 +-
.../src/assembly/resources/sbin/start-datanode.bat | 2 +-
.../src/assembly/resources/sbin/start-datanode.sh | 8 +-
.../src/assembly/resources/sbin/start-server.bat | 2 +-
server/src/assembly/resources/sbin/start-server.sh | 8 +-
.../src/assembly/resources/sbin/stop-datanode.bat | 2 +
server/src/assembly/resources/sbin/stop-server.bat | 2 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 31 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 326 ++++++++++---
.../org/apache/iotdb/db/client/ConfigNodeInfo.java | 160 +++++++
.../iotdb/db/client/DataNodeClientPoolFactory.java | 18 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 13 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 -
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 51 +-
.../compaction/cross/CrossSpaceCompactionTask.java | 9 +-
.../compaction/inner/InnerSpaceCompactionTask.java | 16 +-
.../impl/ReadPointCompactionPerformer.java | 7 +-
.../compaction/task/AbstractCompactionTask.java | 7 +-
.../iotdb/db/engine/cq/ContinuousQueryTask.java | 13 +-
.../db/engine/storagegroup/TsFileProcessor.java | 54 ++-
.../metadata/cache/DataNodeLastCacheManager.java | 81 ++++
.../db/metadata/cache/DataNodeSchemaCache.java | 34 ++
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 19 +
.../db/metadata/idtable/IDTableHashmapImpl.java | 11 +-
.../db/metadata/idtable/entry/SHA256DeviceID.java | 7 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 4 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 4 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 2 +-
.../apache/iotdb/db/metadata/path/AlignedPath.java | 13 +-
.../iotdb/db/metadata/template/Template.java | 20 +-
.../iotdb/db/mpp/aggregation/Accumulator.java | 3 +-
.../iotdb/db/mpp/aggregation/Aggregator.java | 39 +-
.../iotdb/db/mpp/aggregation/AvgAccumulator.java | 38 +-
.../iotdb/db/mpp/aggregation/CountAccumulator.java | 8 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 23 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 25 +-
.../mpp/aggregation/FirstValueDescAccumulator.java | 12 +-
.../db/mpp/aggregation/LastValueAccumulator.java | 32 +-
.../mpp/aggregation/LastValueDescAccumulator.java | 48 +-
.../db/mpp/aggregation/MaxTimeAccumulator.java | 24 +-
.../db/mpp/aggregation/MaxTimeDescAccumulator.java | 17 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 22 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 20 +-
.../db/mpp/aggregation/MinTimeDescAccumulator.java | 2 +-
.../db/mpp/aggregation/MinValueAccumulator.java | 22 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 34 +-
.../timerangeiterator/AggrWindowIterator.java | 172 +++++++
.../timerangeiterator/ITimeRangeIterator.java | 58 +++
.../timerangeiterator/PreAggrWindowIterator.java | 169 +++++++
.../PreAggrWindowWithNaturalMonthIterator.java | 143 ++++++
.../SingleTimeWindowIterator.java | 7 +-
.../TimeRangeIteratorFactory.java | 73 +++
.../operator/process/AggregateOperator.java | 90 +++-
.../execution/operator/process/FilterOperator.java | 25 +-
.../operator/process/RawDataAggregateOperator.java | 194 ++++++++
.../operator/process/TransformOperator.java | 45 +-
.../source/SeriesAggregateScanOperator.java | 107 ++---
.../db/mpp/execution/schedule/DriverScheduler.java | 2 +-
.../org/apache/iotdb/db/mpp/plan/Coordinator.java | 8 +-
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 1 +
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 51 +-
.../execution/config/AuthorizerConfigTask.java | 37 +-
.../mpp/plan/execution/config/ConfigExecution.java | 14 +-
.../execution/config/CountStorageGroupTask.java | 16 +-
.../execution/config/DeleteStorageGroupTask.java | 17 +-
.../db/mpp/plan/execution/config/IConfigTask.java | 8 +-
.../plan/execution/config/SetStorageGroupTask.java | 19 +-
.../db/mpp/plan/execution/config/SetTTLTask.java | 20 +-
.../execution/config/ShowStorageGroupTask.java | 20 +-
.../db/mpp/plan/execution/config/ShowTTLTask.java | 20 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 40 +-
.../db/mpp/plan/parser/StatementGenerator.java | 47 +-
.../db/mpp/plan/planner/DistributionPlanner.java | 46 +-
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 91 +++-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 2 +-
.../planner/plan/node/process/AggregationNode.java | 64 ++-
.../plan/node/process/GroupByLevelNode.java | 6 +-
.../node/source/SeriesAggregationScanNode.java | 12 +-
.../plan/parameter/AggregationDescriptor.java | 61 ++-
.../planner/plan/parameter/AggregationStep.java | 44 +-
.../iotdb/db/qp/physical/sys/SetTemplatePlan.java | 2 +-
.../db/qp/physical/sys/UnsetTemplatePlan.java | 2 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 27 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 8 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 8 +-
.../query/dataset/groupby/GroupByLevelDataSet.java | 9 +-
.../iotdb/db/query/expression/Expression.java | 19 +-
.../query/expression/binary/BinaryExpression.java | 64 ++-
.../db/query/expression/leaf/ConstantOperand.java | 26 ++
.../query/expression/leaf/TimeSeriesOperand.java | 38 ++
.../db/query/expression/leaf/TimestampOperand.java | 33 ++
.../query/expression/multi/FunctionExpression.java | 115 +++++
.../db/query/expression/unary/UnaryExpression.java | 50 ++
.../db/query/udf/core/executor/UDTFExecutor.java | 27 ++
.../query/udf/core/layer/EvaluationDAGBuilder.java | 25 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 24 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 7 +-
.../apache/iotdb/db/service/metrics/Metric.java | 20 +-
.../iotdb/db/service/metrics/MetricsService.java | 19 +
.../db/service/metrics/ProcessMetricsMonitor.java | 148 ++++++
.../db/service/metrics/SysRunMetricsMonitor.java | 134 ++++++
.../thrift/impl/DataNodeTSIServiceImpl.java | 60 ++-
.../db/service/thrift/impl/TSServiceImpl.java | 3 +-
.../apache/iotdb/db/sync/conf/SyncConstant.java | 2 +-
.../iotdb/db/sync/receiver/ReceiverService.java | 16 +-
.../db/sync/receiver/collector/Collector.java | 2 +-
.../receiver/recovery/ReceiverLogAnalyzer.java | 6 +-
.../db/sync/sender/service/TransportHandler.java | 5 +-
.../engine/compaction/CompactionSchedulerTest.java | 2 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 4 +-
.../apache/iotdb/db/metadata/PartialPathTest.java | 158 -------
.../apache/iotdb/db/metadata/SchemaBasicTest.java | 152 ++----
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 69 +++
.../iotdb/db/metadata/mtree/MTreeAboveSGTest.java | 23 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 4 +-
.../db/metadata/upgrade/MetadataUpgradeTest.java | 2 +-
.../iotdb/db/mpp/aggregation/AccumulatorTest.java | 116 ++++-
.../db/mpp/aggregation/TimeRangeIteratorTest.java | 298 ++++++++++++
.../db/mpp/execution/ConfigExecutionTest.java | 16 +-
.../execution/operator/AggregateOperatorTest.java | 321 +++++++++++++
.../operator/RawDataAggregateOperatorTest.java | 368 +++++++++++++++
.../operator/SeriesAggregateScanOperatorTest.java | 25 +-
.../read/DeviceSchemaScanNodeSerdeTest.java | 2 +-
.../metadata/read/SchemaCountNodeSerdeTest.java | 4 +-
.../read/TimeSeriesSchemaScanNodeSerdeTest.java | 2 +-
.../node/process/AggregationNodeSerdeTest.java | 228 +++++++++
.../plan/node/sink/FragmentSinkNodeSerdeTest.java | 7 +-
.../db/qp/physical/PhysicalPlanSerializeTest.java | 6 +-
.../receiver/recovery/ReceiverLogAnalyzerTest.java | 2 +-
thrift-commons/src/main/thrift/common.thrift | 5 +-
.../src/main/thrift/confignode.thrift | 11 +-
tsfile/pom.xml | 10 +
.../tsfile/common/constant/TsFileConstant.java | 7 +
.../iotdb/tsfile/exception/PathParseException.java | 11 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 78 ++--
.../apache/iotdb/tsfile/read/common/TimeRange.java | 20 +-
.../read/common/parser/PathNodesGenerator.java | 94 ++++
.../tsfile/read/common/parser/PathParseError.java | 32 +-
.../tsfile/read/common/parser/PathVisitor.java | 75 +++
.../write/writer/RestorableTsFileIOWriter.java | 37 +-
.../apache/iotdb/tsfile/read/common/PathTest.java | 163 ++++++-
211 files changed, 7135 insertions(+), 1958 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/DistributionPlanner.java
index 5e002ac83b,f0eb1fb387..93f9718e02
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/DistributionPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/DistributionPlanner.java
@@@ -43,10 -43,10 +44,12 @@@ import org.apache.iotdb.db.mpp.plan.pla
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.ExchangeNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.TimeJoinNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.FragmentSinkNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SourceNode;
+ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
+ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep;
import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement;
import java.util.ArrayList;
@@@ -237,26 -234,51 +240,65 @@@ public class DistributionPlanner
return timeJoinNode;
}
- @Override
+ public PlanNode visitSeriesAggregationScan(
+ SeriesAggregationScanNode node, DistributionPlanContext context) {
+ List<TRegionReplicaSet> dataDistribution =
+ analysis.getPartitionInfo(node.getSeriesPath(), node.getTimeFilter());
- if (dataDistribution.size() == 1) {
- node.setRegionReplicaSet(dataDistribution.get(0));
- return node;
- }
-
+ List<AggregationDescriptor> leafAggDescriptorList = new ArrayList<>();
+ node.getAggregationDescriptorList()
+ .forEach(
+ descriptor -> {
+ leafAggDescriptorList.add(
+ new AggregationDescriptor(
+ descriptor.getAggregationType(),
+ AggregationStep.PARTIAL,
+ descriptor.getInputExpressions()));
+ });
+
+ List<AggregationDescriptor> rootAggDescriptorList = new ArrayList<>();
+ node.getAggregationDescriptorList()
+ .forEach(
+ descriptor -> {
+ rootAggDescriptorList.add(
+ new AggregationDescriptor(
+ descriptor.getAggregationType(),
+ AggregationStep.FINAL,
+ descriptor.getInputExpressions()));
+ });
+
+ AggregationNode aggregationNode =
+ new AggregationNode(
+ context.queryContext.getQueryId().genPlanNodeId(), rootAggDescriptorList);
+ for (TRegionReplicaSet dataRegion : dataDistribution) {
+ SeriesAggregationScanNode split = (SeriesAggregationScanNode) node.clone();
+ split.setAggregationDescriptorList(leafAggDescriptorList);
+ split.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId());
+ split.setRegionReplicaSet(dataRegion);
+ aggregationNode.addChild(split);
+ }
+ return aggregationNode;
+ }
+
+ @Override
+ public PlanNode visitAlignedSeriesScan(
+ AlignedSeriesScanNode node, DistributionPlanContext context) {
+ List<TRegionReplicaSet> dataDistribution =
+ analysis.getPartitionInfo(node.getAlignedPath(), node.getTimeFilter());
+ if (dataDistribution.size() == 1) {
+ node.setRegionReplicaSet(dataDistribution.get(0));
+ return node;
+ }
+ TimeJoinNode timeJoinNode =
+ new TimeJoinNode(context.queryContext.getQueryId().genPlanNodeId(), node.getScanOrder());
+ for (TRegionReplicaSet dataRegion : dataDistribution) {
+ AlignedSeriesScanNode split = (AlignedSeriesScanNode) node.clone();
+ split.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId());
+ split.setRegionReplicaSet(dataRegion);
+ timeJoinNode.addChild(split);
+ }
+ return timeJoinNode;
+ }
+
@Override
public PlanNode visitSchemaFetchMerge(
SchemaFetchMergeNode node, DistributionPlanContext context) {
@@@ -471,15 -481,8 +513,15 @@@
}
@Override
+ public PlanNode visitAlignedSeriesScan(AlignedSeriesScanNode node, NodeGroupContext context) {
+ context.putNodeDistribution(
+ node.getPlanNodeId(),
+ new NodeDistribution(NodeDistributionType.NO_CHILD, node.getRegionReplicaSet()));
+ return node.clone();
+ }
+
- @Override
- public PlanNode visitSeriesAggregate(SeriesAggregationScanNode node, NodeGroupContext context) {
+ public PlanNode visitSeriesAggregationScan(
+ SeriesAggregationScanNode node, NodeGroupContext context) {
context.putNodeDistribution(
node.getPlanNodeId(),
new NodeDistribution(NodeDistributionType.NO_CHILD, node.getRegionReplicaSet()));