You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/03/23 11:13:28 UTC
[iotdb] 01/02: Merge branch 'add_mpp_interface' into xingtanzjr/distribution_planner
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/distribution_planner
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 41c2b7c63a2b3b76a695787208e28b276c9e848c
Merge: 5f0aa7f 054badb
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Wed Mar 23 10:03:52 2022 +0800
Merge branch 'add_mpp_interface' into xingtanzjr/distribution_planner
docs/Download/README.md | 4 +-
docs/UserGuide/Library-UDF/Anomaly-Detection.md | 523 +++++++
docs/UserGuide/Process-Data/Continuous-Query.md | 4 +-
docs/zh/Download/README.md | 4 +-
docs/zh/UserGuide/Library-UDF/Anomaly Detection.md | 509 +++++++
docs/zh/UserGuide/Process-Data/Continuous-Query.md | 4 +-
library-udf/pom.xml | 18 +
.../org/apache/iotdb/library/anomaly/UDTFIQR.java | 111 ++
.../apache/iotdb/library/anomaly/UDTFKSigma.java | 127 ++
.../org/apache/iotdb/library/anomaly/UDTFLOF.java | 215 +++
.../iotdb/library/anomaly/UDTFMissDetect.java | 74 +
.../apache/iotdb/library/anomaly/UDTFRange.java | 97 ++
.../iotdb/library/anomaly/UDTFTwoSidedFilter.java | 92 ++
.../iotdb/library/anomaly/util/MissDetector.java | 240 +++
.../library/anomaly/util/StreamMissDetector.java | 189 +++
.../iotdb/library/anomaly/util/WindowDetect.java | 96 ++
.../iotdb/library/util/BooleanCircularQueue.java | 138 ++
.../apache/iotdb/library/util/CircularQueue.java | 138 ++
.../iotdb/library/util/DoubleCircularQueue.java | 138 ++
.../iotdb/library/util/LinearRegression.java | 85 ++
.../iotdb/library/util/LongCircularQueue.java | 138 ++
.../iotdb/library/util/NoNumberException.java | 19 +-
.../java/org/apache/iotdb/library/util/Util.java | 306 ++++
.../apache/iotdb/library/anomaly/AnomalyTests.java | 1533 ++++++++++++++++++++
.../main/assembly/resources/conf/iotdb-metric.yml | 2 +-
.../iotdb/commons/concurrent/ThreadName.java | 6 +-
.../iotdb/commons/partition/DataPartitionInfo.java | 27 +-
.../commons/partition/DataPartitionQueryParam.java | 27 +-
.../iotdb/commons/partition/DataRegionId.java | 14 +-
.../commons/partition/DataRegionPlaceInfo.java | 26 +-
.../iotdb/commons/partition/DeviceGroupId.java | 14 +-
.../iotdb/commons/partition/PartitionInfo.java | 25 +-
.../commons/partition/SchemaPartitionInfo.java | 19 +-
.../iotdb/commons/partition/SchemaRegionId.java | 15 +-
.../commons/partition/SchemaRegionPlaceInfo.java | 33 +-
.../iotdb/commons/partition/TimePartitionId.java | 14 +-
.../apache/iotdb/commons/service/ServiceType.java | 4 +-
pom.xml | 2 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 13 +-
.../sql/SemanticException.java} | 11 +-
.../exception/sql/StatementAnalyzeException.java | 2 +-
.../micrometer/registry/IoTDBRegistryConfig.java | 18 +-
.../iotdb/db/mpp/buffer/DataBlockManager.java | 116 ++
.../db/mpp/buffer/DataBlockManagerService.java | 90 ++
.../DataBlockManagerServiceThriftHandler.java} | 37 +-
.../mpp/buffer/DataBlockServiceClientFactory.java | 44 +
.../DataBlockServiceImpl.java} | 39 +-
.../buffer/{SinkHandle.java => ISinkHandle.java} | 35 +-
.../{SourceHandle.java => ISourceHandle.java} | 11 +-
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 70 +-
.../iotdb/db/mpp/common/MPPQueryContext.java | 18 +-
.../common/{QuerySession.java => SessionInfo.java} | 7 +-
.../common}/filter/BasicFilterType.java | 6 +-
.../common}/filter/BasicFunctionFilter.java | 6 +-
.../common}/filter/FunctionFilter.java | 4 +-
.../statement => mpp/common}/filter/InFilter.java | 4 +-
.../common}/filter/LikeFilter.java | 4 +-
.../common}/filter/QueryFilter.java | 6 +-
.../common}/filter/RegexpFilter.java | 4 +-
.../apache/iotdb/db/mpp/execution/Coordinator.java | 24 +-
.../execution/ExecutionResult.java} | 17 +-
.../iotdb/db/mpp/execution/QueryExecution.java | 47 +-
.../execution/scheduler/StandaloneScheduler.java | 4 +-
.../LocalMemoryManager.java} | 33 +-
.../org/apache/iotdb/db/mpp/memory/MemoryPool.java | 93 ++
.../iotdb/db/mpp/schedule/AbstractExecutor.java | 72 +
.../schedule/ExecutionContext.java} | 31 +-
.../db/mpp/schedule/FragmentInstanceManager.java | 292 ++++
.../mpp/schedule/FragmentInstanceTaskExecutor.java | 84 ++
.../schedule/FragmentInstanceTimeoutSentinel.java | 55 +
.../IFragmentInstanceManager.java} | 39 +-
.../iotdb/db/mpp/schedule/ITaskScheduler.java | 77 +
.../OrderBy.java => mpp/schedule/queue/ID.java} | 10 +-
.../queue/IDIndexedAccessible.java} | 45 +-
.../mpp/schedule/queue/IndexedBlockingQueue.java | 188 +++
.../db/mpp/schedule/queue/L1PriorityQueue.java | 86 ++
.../db/mpp/schedule/queue/L2PriorityQueue.java | 103 ++
.../db/mpp/schedule/task/FragmentInstanceID.java | 68 +
.../db/mpp/schedule/task/FragmentInstanceTask.java | 199 +++
.../schedule/task/FragmentInstanceTaskStatus.java} | 28 +-
.../db/mpp/{common => sql/analyze}/Analysis.java | 47 +-
.../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 144 ++
.../ClusterPartitionFetcher.java} | 31 +-
.../sql/analyze/ClusterSchemaFetcher.java} | 14 +-
.../IPartitionFetcher.java} | 39 +-
.../sql/analyze/ISchemaFetcher.java} | 20 +-
.../analyze/QueryType.java} | 6 +-
.../sql/analyze/StandalonePartitionFetcher.java | 65 +
.../sql/analyze/StandaloneSchemaFetcher.java} | 20 +-
.../db/{ => mpp}/sql/constant/FilterConstant.java | 2 +-
.../db/{ => mpp}/sql/constant/StatementType.java | 2 +-
.../{planner => }/optimization/PlanOptimizer.java | 2 +-
.../iotdb/db/{ => mpp}/sql/parser/ASTVisitor.java | 370 ++++-
.../{ => mpp}/sql/parser/StatementGenerator.java | 65 +-
.../planner/{plan => }/DistributionPlanner.java | 6 +-
.../iotdb/db/mpp/sql/planner/LogicalPlanner.java | 98 ++
.../db/mpp/sql/planner/plan/LogicalQueryPlan.java | 5 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 127 +-
.../planner/plan/node/process/DeviceMergeNode.java | 2 +-
.../sql/planner/plan/node/process/SortNode.java | 2 +-
.../planner/plan/node/process/TimeJoinNode.java | 2 +-
.../planner/plan/node/source/SeriesScanNode.java | 2 +-
.../plan/node/write/InsertMultiTabletNode.java | 2 +-
.../sql/planner/plan/node/write/InsertNode.java | 3 +-
.../sql/planner/plan/node/write/InsertRowNode.java | 2 +-
.../planner/plan/node/write/InsertRowsNode.java | 2 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 35 +-
.../db/mpp/sql/rewriter/ConcatPathRewriter.java | 395 +++++
.../db/mpp/sql/rewriter/DnfFilterOptimizer.java | 163 +++
.../sql/rewriter/IFilterOptimizer.java} | 12 +-
.../IStatementRewriter.java} | 17 +-
.../sql/rewriter/MergeSingleFilterOptimizer.java | 179 +++
.../db/mpp/sql/rewriter/RemoveNotOptimizer.java | 103 ++
.../db/mpp/sql/rewriter/WildcardsRemover.java | 142 ++
.../db/{ => mpp}/sql/statement/Statement.java | 8 +-
.../sql/statement/StatementNode.java} | 20 +-
.../sql/statement/component/FillComponent.java | 5 +-
.../statement/component/FilterNullComponent.java} | 21 +-
.../sql/statement/component/FromComponent.java | 5 +-
.../statement/component/GroupByLevelComponent.java | 6 +-
.../component/GroupByLevelController.java | 221 +++
.../statement/component/GroupByTimeComponent.java | 6 +-
.../{ => mpp}/sql/statement/component/OrderBy.java | 2 +-
.../sql/statement/component/ResultColumn.java | 25 +-
.../sql/statement/component/ResultSetFormat.java | 2 +-
.../sql/statement/component/SelectComponent.java | 45 +-
.../sql/statement/component/WhereCondition.java | 7 +-
.../statement/crud/AggregationQueryStatement.java | 90 ++
.../mpp/sql/statement/crud/FillQueryStatement.java | 73 +
.../statement/crud}/GroupByFillQueryStatement.java | 4 +-
.../sql/statement/crud}/GroupByQueryStatement.java | 4 +-
.../crud/InsertBaseStatement.java} | 25 +-
.../db/mpp/sql/statement/crud/InsertStatement.java | 89 ++
.../sql/statement/crud/InsertTabletStatement.java | 52 +
.../sql/statement/crud}/LastQueryStatement.java | 27 +-
.../sql/statement/crud}/QueryStatement.java | 96 +-
.../mpp/sql/statement/crud/UDAFQueryStatement.java | 91 ++
.../sql/statement/crud}/UDTFQueryStatement.java | 2 +-
.../metadata/CreateAlignedTimeSeriesStatement.java | 167 +++
.../metadata/CreateTimeSeriesStatement.java | 124 ++
.../statement/metadata/ShowDevicesStatement.java} | 33 +-
.../sql/statement/metadata/ShowStatement.java} | 22 +-
.../metadata/ShowTimeSeriesStatement.java | 81 ++
.../iotdb/db/mpp/sql/tree/StatementVisitor.java | 78 +
.../iotdb/db/query/control/SessionManager.java | 9 +
.../iotdb/db/query/expression/Expression.java | 9 +-
.../query/expression/binary/BinaryExpression.java | 17 +-
.../db/query/expression/unary/ConstantOperand.java | 9 +-
.../query/expression/unary/FunctionExpression.java | 14 +-
.../query/expression/unary/NegationExpression.java | 15 +-
.../query/expression/unary/TimeSeriesOperand.java | 13 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +-
.../iotdb/db/service/basic/ServiceProvider.java | 6 +-
.../db/service/thrift/impl/TSServiceImpl.java | 51 +-
.../sql/statement/AggregationQueryStatement.java | 49 -
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 13 +-
.../org/apache/iotdb/db/utils/stats/CpuTimer.java | 156 ++
.../apache/iotdb/db/mpp/memory/MemoryPoolTest.java | 150 ++
.../db/mpp/schedule/queue/L1PriorityQueueTest.java | 134 ++
.../db/mpp/schedule/queue/L2PriorityQueueTest.java | 143 ++
.../iotdb/db/mpp/schedule/queue/QueueElement.java | 80 +
.../iotdb/db/mpp/sql/analyze/AnalyzerTest.java | 51 +
.../{qp/sql => mpp/sql/parser}/ASTVisitorTest.java | 4 +-
.../sql/parser}/StatementGeneratorTest.java | 7 +-
.../db/mpp/sql/plan/DistributionPlannerTest.java | 18 +-
.../iotdb/db/mpp/sql/plan/LogicalPlannerTest.java | 96 ++
.../org/apache/iotdb/db/qp/sql/ASTVisitorTest.java | 2 +-
thrift-datanode/pom.xml | 67 +
thrift-datanode/src/main/thrift/datanode.thrift | 92 ++
.../src/main/thrift/mpp.thrift | 56 +-
171 files changed, 11305 insertions(+), 687 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
index f4c3bd6,ead15b2..e58b755
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
@@@ -41,17 -48,18 +48,17 @@@ public class QueryExecution
private List<PlanOptimizer> planOptimizers;
- private Analysis analysis;
+ private final Analysis analysis;
private LogicalQueryPlan logicalPlan;
private DistributedQueryPlan distributedPlan;
- private List<PlanFragment> fragments;
private List<FragmentInstance> fragmentInstances;
- public QueryExecution(MPPQueryContext context) {
+ public QueryExecution(Statement statement, MPPQueryContext context) {
this.context = context;
+ this.analysis = analyze(statement, context);
}
- public void plan() {
- analyze();
+ public void start() {
doLogicalPlan();
doDistributedPlan();
planFragmentInstances();
@@@ -82,10 -92,8 +91,11 @@@
// Convert fragment to detailed instance
// And for parallel-able fragment, clone it into several instances with different params.
- private void planFragmentInstances() {}
+ public void planFragmentInstances() {
+
+ }
+
+
/**
* This method will be called by the request thread from client connection. This method will block
* until one of these conditions occurs: 1. There is a batch of result 2. There is no more result
diff --cc server/src/test/java/org/apache/iotdb/db/mpp/sql/plan/DistributionPlannerTest.java
index ec90301,90d5b7c..6d98e93
--- a/server/src/test/java/org/apache/iotdb/db/mpp/sql/plan/DistributionPlannerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/sql/plan/DistributionPlannerTest.java
@@@ -22,13 -23,11 +23,14 @@@ import org.apache.iotdb.commons.partiti
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.mpp.common.*;
- import org.apache.iotdb.db.mpp.sql.planner.plan.DistributionPlanner;
+ import org.apache.iotdb.db.mpp.sql.analyze.Analysis;
+ import org.apache.iotdb.db.mpp.sql.planner.DistributionPlanner;
import org.apache.iotdb.db.mpp.sql.planner.plan.LogicalQueryPlan;
+import org.apache.iotdb.db.mpp.sql.planner.plan.PlanFragment;
+import org.apache.iotdb.db.mpp.sql.planner.plan.SubPlan;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.PlanNodeIdAllocator;
+import org.apache.iotdb.db.mpp.sql.planner.plan.node.PlanNodeUtil;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.process.ExchangeNode;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.process.LimitNode;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.process.TimeJoinNode;
@@@ -107,41 -106,9 +109,41 @@@ public class DistributionPlannerTest
rootWithExchange.getChildren().get(0).getChildren().get(2).getChildren().size(), 1);
}
+ @Test
+ public void TestSplitFragment() throws IllegalPathException {
+ TimeJoinNode timeJoinNode =
+ new TimeJoinNode(
+ PlanNodeIdAllocator.generateId(), OrderBy.TIMESTAMP_ASC, FilterNullPolicy.NO_FILTER);
+
+ timeJoinNode.addChild(
+ new SeriesScanNode(PlanNodeIdAllocator.generateId(), new PartialPath("root.sg.d1.s1")));
+ timeJoinNode.addChild(
+ new SeriesScanNode(PlanNodeIdAllocator.generateId(), new PartialPath("root.sg.d1.s2")));
+ timeJoinNode.addChild(
+ new SeriesScanNode(PlanNodeIdAllocator.generateId(), new PartialPath("root.sg.d2.s1")));
+
+ LimitNode root = new LimitNode(PlanNodeIdAllocator.generateId(), 10, timeJoinNode);
+
+ Analysis analysis = constructAnalysis();
+
+ DistributionPlanner planner =
+ new DistributionPlanner(analysis, new LogicalQueryPlan(new MPPQueryContext(), root));
+ PlanNode rootAfterRewrite = planner.rewriteSource();
+ PlanNode rootWithExchange = planner.addExchangeNode(rootAfterRewrite);
+ PlanNodeUtil.printPlanNode(rootWithExchange);
+ SubPlan subPlan = planner.splitFragment(rootWithExchange);
+ System.out.println(subPlan);
+ List<PlanFragment> fragments = subPlan.getPlanFragmentList();
+ fragments.forEach(
+ f -> {
+ System.out.println(f);
+ PlanNodeUtil.printPlanNode(f.getRoot());
+ });
+ }
+
private Analysis constructAnalysis() {
Analysis analysis = new Analysis();
- Map<String, Map<DataRegionTimeSlice, List<DataRegion>>> dataPartitionInfo = new HashMap<>();
+ Map<String, Map<TimePartitionId, List<DataRegion>>> dataPartitionInfo = new HashMap<>();
List<DataRegion> d1DataRegions = new ArrayList<>();
d1DataRegions.add(new DataRegion(1, "192.0.0.1"));
d1DataRegions.add(new DataRegion(2, "192.0.0.1"));