You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/03/27 01:58:06 UTC
[iotdb] branch lmh/fileScan created (now 8cb7027ed0)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 8cb7027ed0 use agg instead of last
This branch includes the following new commits:
new 1d63214b3c (tmp save)
new fdc10a8ffb rename
new 25d366754f Merge remote-tracking branch 'origin/master' into lmh/fileScan
new ac917e467f finish
new 7dc381bd51 add updateAggregationResult
new 4f70135562 finish FE
new 79e4a1cb5a fix bugs & can run
new d53bd3d3bf Merge remote-tracking branch 'origin/master' into lmh/fileScan
new cfbe60c920 merge master
new 76eff499bf fix aligned bug
new 5787d0f858 fix deserialize bug
new 9630dee026 return tsblock only once
new 8cb7027ed0 use agg instead of last
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 07/13: fix bugs & can run
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 79e4a1cb5af7945926d0b88b652b636a38c2682a
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Mar 20 21:54:51 2023 +0800
fix bugs & can run
---
.../iotdb/commons/partition/DataPartition.java | 16 +++++++++++
.../db/engine/querycontext/QueryDataSource.java | 8 ++++++
.../operator/source/FileAggregationScanUtil.java | 12 +++++++--
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 4 +++
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 7 +++--
.../mpp/plan/analyze/GroupByLevelController.java | 8 ++++--
.../planner/distribution/ExchangeNodeAdder.java | 12 +++++----
.../plan/planner/distribution/SourceRewriter.java | 3 +--
.../plan/node/source/FileAggregationScanNode.java | 31 +++++++++++++++++++---
.../materializer/TsFileResourceMaterializer.java | 2 +-
10 files changed, 84 insertions(+), 19 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
index f25f0a928f..b5e851e3ca 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
@@ -82,6 +82,22 @@ public class DataPartition extends Partition {
.collect(Collectors.toList());
}
+ public List<TRegionReplicaSet> getAllDataRegionReplicaSet() {
+ List<TRegionReplicaSet> regionReplicaSet = new ArrayList<>();
+ dataPartitionMap
+ .values()
+ .forEach(
+ tSeriesPartitionSlotMapMap -> {
+ tSeriesPartitionSlotMapMap
+ .values()
+ .forEach(
+ tTimePartitionSlotListMap -> {
+ tTimePartitionSlotListMap.values().forEach(regionReplicaSet::addAll);
+ });
+ });
+ return regionReplicaSet;
+ }
+
public List<TRegionReplicaSet> getDataRegionReplicaSetForWriting(
String deviceName, List<TTimePartitionSlot> timePartitionSlotList) {
// A list of data region replica sets will store data in a same time partition.
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
index 8793b2c135..012b243960 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
@@ -121,4 +121,12 @@ public class QueryDataSource {
}
this.unSeqFileOrderIndex = unSeqFileOrderIndex;
}
+
+ public void constructOrderIndexes() {
+ int[] unSeqFileOrderIndex = new int[unseqResources.size()];
+ for (int i = 0; i < unseqResources.size(); i++) {
+ unSeqFileOrderIndex[i] = i;
+ }
+ this.unSeqFileOrderIndex = unSeqFileOrderIndex;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index 5a4ef5cb38..12290b4ebc 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -50,6 +50,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
+
public class FileAggregationScanUtil {
private final PartialPath pathPattern;
@@ -68,6 +70,8 @@ public class FileAggregationScanUtil {
private final SeriesScanOptions scanOptions;
+ private boolean isCountStar;
+
public FileAggregationScanUtil(
PartialPath pathPattern,
AggregationDescriptor aggregationDescriptor,
@@ -80,9 +84,13 @@ public class FileAggregationScanUtil {
this.partialPathPool = new PartialPathPool();
this.levels = levels;
this.scanOptions = scanOptions;
+ this.isCountStar =
+ pathPattern.getMeasurement().equals(ONE_LEVEL_PATH_WILDCARD)
+ && aggregationDescriptor.getAggregationType() == TAggregationType.COUNT;
}
public void initQueryDataSource(QueryDataSource dataSource) {
+ dataSource.constructOrderIndexes();
this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
}
@@ -166,7 +174,7 @@ public class FileAggregationScanUtil {
PartialPath groupedPath =
partialPathPool.getGroupedPath(devicePath, timeseriesMetadata.getMeasurementId());
- if (pathToAggregatorMap.containsKey(groupedPath)) {
+ if (!pathToAggregatorMap.containsKey(groupedPath)) {
pathToAggregatorMap.put(
groupedPath,
new Aggregator(
@@ -248,7 +256,7 @@ public class FileAggregationScanUtil {
return rawPathToGroupedPathMap.get(rawPathStr);
}
PartialPath groupedPath =
- GroupByLevelController.groupPathByLevel(devicePath, measurementId, levels);
+ GroupByLevelController.groupPathByLevel(devicePath, measurementId, levels, isCountStar);
rawPathToGroupedPathMap.put(rawPathStr, groupedPath);
return groupedPath;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
index be471ba3b5..952541dd1b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
@@ -227,6 +227,10 @@ public class Analysis {
return dataPartition.getDataRegionReplicaSet(deviceName, null);
}
+ public List<TRegionReplicaSet> getPartitionInfo() {
+ return dataPartition.getAllDataRegionReplicaSet();
+ }
+
public Statement getStatement() {
return statement;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index f922ce89dd..5c03ff0763 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -230,6 +230,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
queryStatement.semanticCheck();
if (queryStatement.isGroupByLevel()) {
+ analysis.setStatement(queryStatement);
+
analysis.setLevels(queryStatement.getGroupByLevelComponent().getLevels());
PartialPath pathPrefix = queryStatement.getFromComponent().getPrefixPaths().get(0);
@@ -258,10 +260,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
new DatasetHeader(ColumnHeaderConstant.lastQueryColumnHeaders, true));
Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>();
- sgNameToQueryParamsMap.put(
- "root.iov",
- Collections.singletonList(
- new DataPartitionQueryParam("root.iov.**", Collections.emptyList(), true, true)));
+ sgNameToQueryParamsMap.put("root.iov", Collections.emptyList());
analysis.setDataPartitionInfo(partitionFetcher.getDataPartition(sgNameToQueryParamsMap));
return analysis;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
index 632a5c776e..da57edac25 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
@@ -174,7 +174,7 @@ public class GroupByLevelController {
}
public static PartialPath groupPathByLevel(
- PartialPath rawDevicePath, String measurement, int[] levels) {
+ PartialPath rawDevicePath, String measurement, int[] levels, boolean isCountStar) {
String[] nodes = Arrays.copyOf(rawDevicePath.getNodes(), rawDevicePath.getNodes().length + 1);
nodes[nodes.length - 1] = measurement;
@@ -193,7 +193,11 @@ public class GroupByLevelController {
transformedNodes.add(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD);
}
}
- transformedNodes.add(nodes[nodes.length - 1]);
+ if (isCountStar) {
+ transformedNodes.add(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD);
+ } else {
+ transformedNodes.add(nodes[nodes.length - 1]);
+ }
return new PartialPath(transformedNodes.toArray(new String[0]));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/ExchangeNodeAdder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/ExchangeNodeAdder.java
index 56dfda1cd7..b7c243661f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/ExchangeNodeAdder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/ExchangeNodeAdder.java
@@ -52,6 +52,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.last.LastQueryNode
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
@@ -186,6 +187,11 @@ public class ExchangeNodeAdder extends PlanVisitor<PlanNode, NodeGroupContext> {
return processNoChildSourceNode(node, context);
}
+ @Override
+ public PlanNode visitFileAggregationScan(FileAggregationScanNode node, NodeGroupContext context) {
+ return processNoChildSourceNode(node, context);
+ }
+
private PlanNode processNoChildSourceNode(SourceNode node, NodeGroupContext context) {
context.putNodeDistribution(
node.getPlanNodeId(),
@@ -276,11 +282,7 @@ public class ExchangeNodeAdder extends PlanVisitor<PlanNode, NodeGroupContext> {
MultiChildProcessNode newNode = (MultiChildProcessNode) node.clone();
List<PlanNode> visitedChildren = new ArrayList<>();
- node.getChildren()
- .forEach(
- child -> {
- visitedChildren.add(visit(child, context));
- });
+ node.getChildren().forEach(child -> visitedChildren.add(visit(child, context)));
TRegionReplicaSet dataRegion;
NodeDistributionType distributionType;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
index f61891e59c..c9cfeb1102 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
@@ -448,8 +448,7 @@ public class SourceRewriter extends SimplePlanNodeRewriter<DistributionPlanConte
@Override
public List<PlanNode> visitFileAggregationScan(
FileAggregationScanNode node, DistributionPlanContext context) {
- List<TRegionReplicaSet> dataDistribution =
- analysis.getPartitionInfo(node.getPartitionPath(), node.getPartitionTimeFilter());
+ List<TRegionReplicaSet> dataDistribution = analysis.getPartitionInfo();
if (dataDistribution.size() == 1) {
node.setRegionReplicaSet(dataDistribution.get(0));
return Collections.singletonList(node);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
index 63158d132a..9c1eeaa20c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -45,6 +46,9 @@ public class FileAggregationScanNode extends SeriesSourceNode {
private final int[] levels;
+ // The id of DataRegion where the node will run
+ private TRegionReplicaSet regionReplicaSet;
+
public FileAggregationScanNode(
PlanNodeId id,
PartialPath pathPattern,
@@ -56,6 +60,16 @@ public class FileAggregationScanNode extends SeriesSourceNode {
this.levels = levels;
}
+ public FileAggregationScanNode(
+ PlanNodeId id,
+ PartialPath pathPattern,
+ AggregationDescriptor aggregationDescriptor,
+ int[] levels,
+ TRegionReplicaSet regionReplicaSet) {
+ this(id, pathPattern, aggregationDescriptor, levels);
+ this.regionReplicaSet = regionReplicaSet;
+ }
+
public PartialPath getPathPattern() {
return pathPattern;
}
@@ -83,11 +97,13 @@ public class FileAggregationScanNode extends SeriesSourceNode {
}
@Override
- public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) {}
+ public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) {
+ this.regionReplicaSet = regionReplicaSet;
+ }
@Override
public TRegionReplicaSet getRegionReplicaSet() {
- return null;
+ return regionReplicaSet;
}
@Override
@@ -114,7 +130,11 @@ public class FileAggregationScanNode extends SeriesSourceNode {
@Override
public PlanNode clone() {
return new FileAggregationScanNode(
- getPlanNodeId(), getPathPattern(), getAggregationDescriptor(), getLevels());
+ getPlanNodeId(),
+ getPathPattern(),
+ getAggregationDescriptor(),
+ getLevels(),
+ getRegionReplicaSet());
}
@Override
@@ -155,4 +175,9 @@ public class FileAggregationScanNode extends SeriesSourceNode {
PlanNodeId planNodeId = PlanNodeId.deserialize(buffer);
return new FileAggregationScanNode(planNodeId, pathPattern, aggregationDescriptor, levels);
}
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitFileAggregationScan(this, context);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java b/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java
index e5e1c0bc77..efdec760ae 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java
@@ -42,7 +42,7 @@ public class TsFileResourceMaterializer {
public TsFileResource next() {
if (tsFileResourceSource.hasNextSeqResource(curSeqFileIndex, true)) {
- TsFileResource nextFile = tsFileResourceSource.getSeqResourceByIndex(curUnSeqFileIndex);
+ TsFileResource nextFile = tsFileResourceSource.getSeqResourceByIndex(curSeqFileIndex);
curSeqFileIndex++;
return nextFile;
} else if (tsFileResourceSource.hasNextUnseqResource(curUnSeqFileIndex)) {
[iotdb] 08/13: Merge remote-tracking branch 'origin/master' into lmh/fileScan
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d53bd3d3bffc0aadc97689ab23f9efc6d72e3f68
Merge: 79e4a1cb5a 94ac27b57f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 09:43:14 2023 +0800
Merge remote-tracking branch 'origin/master' into lmh/fileScan
README.md | 4 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 1 +
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../client/async/AsyncDataNodeClientPool.java | 14 +
.../confignode/conf/ConfigNodeDescriptor.java | 2 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 40 +-
.../consensus/request/ConfigPhysicalPlan.java | 16 +
.../write/pipe/plugin/CreatePipePluginPlan.java | 4 +
.../statemachine/ConfigRegionStateMachine.java | 3 +-
.../iotdb/confignode/manager/ConfigManager.java | 5 +
.../apache/iotdb/confignode/manager/IManager.java | 8 +
.../iotdb/confignode/manager/ProcedureManager.java | 46 +
.../manager/consensus/ConsensusManager.java | 63 +-
.../iotdb/confignode/manager/node/NodeManager.java | 9 +
.../manager/pipe/PipePluginCoordinator.java | 120 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 31 +
.../pipe/plugin/CreatePipePluginProcedure.java | 306 +
.../impl/pipe/plugin/DropPipePluginProcedure.java | 251 +
.../state/pipe/plugin/CreatePipePluginState.java | 13 +-
.../state/pipe/plugin/DropPipePluginState.java | 13 +-
.../procedure/store/ProcedureFactory.java | 12 +
.../confignode/procedure/store/ProcedureType.java | 6 +-
.../iotdb/confignode/service/ConfigNode.java | 17 +-
.../confignode/service/ConfigNodeCommandLine.java | 7 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 5 +
.../impl/pipe/CreatePipePluginProcedureTest.java | 60 +
.../impl/pipe/DropPipePluginProcedureTest.java | 53 +
.../confignode1conf/iotdb-common.properties | 1 +
.../confignode2conf/iotdb-common.properties | 1 +
.../confignode3conf/iotdb-common.properties | 1 +
.../iotdb/consensus/config/ConsensusConfig.java | 15 -
.../ratis/ApplicationStateMachineProxy.java | 35 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 31 +-
.../org/apache/iotdb/consensus/ratis/Utils.java | 15 +
.../ratis/metrics/IoTDBMetricRegistry.java | 65 +-
.../ratis/metrics/MetricRegistryManager.java | 13 +-
.../consensus/ratis/metrics/RatisMetricSet.java | 79 +-
.../ratis/metrics/RatisMetricsManager.java | 89 +
.../iotdb/consensus/simple/SimpleConsensus.java | 26 +-
.../apache/iotdb/consensus/iot/ReplicateTest.java | 2 -
.../apache/iotdb/consensus/iot/StabilityTest.java | 2 -
.../iotdb/consensus/ratis/RatisConsensusTest.java | 2 -
.../apache/iotdb/consensus/ratis/SnapshotTest.java | 11 +-
.../iotdb/consensus/simple/RecoveryTest.java | 2 -
docs/Community/Materials.md | 329 +-
.../Apache-IoTDB-ConfigNode-Dashboard.json | 4184 ++++++-------
.../Apache-IoTDB-DataNode-Dashboard.json | 6615 +++++++++++---------
...rd.json => Apache-IoTDB-Disk-IO-Dashboard.json} | 486 +-
.../Apache-IoTDB-Network-Dashboard.json | 7 +-
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 99 +-
docs/UserGuide/Operators-Functions/Aggregation.md | 32 +-
docs/UserGuide/Operators-Functions/Mathematical.md | 65 +-
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 79 +-
.../UserGuide/Operators-Functions/Aggregation.md | 2 +-
.../UserGuide/Operators-Functions/Mathematical.md | 63 +-
docs/zh/UserGuide/Operators-Functions/Overview.md | 44 +-
grafana-plugin/yarn.lock | 6 +-
.../itbase/constant/BuiltinScalarFunctionEnum.java | 1 +
.../BuiltinTimeSeriesGeneratingFunctionEnum.java | 1 -
.../apache/iotdb/itbase/constant/TestConstant.java | 4 +
.../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 47 +
.../iotdb/db/it/aggregation/IoTDBModeIT.java | 160 +
.../scalar/IoTDBRoundFunctionIT.java | 327 +
.../iotdb/libudf/it/dprofile/DProfileIT.java | 3 +-
library-udf/src/assembly/tools/register-UDF.bat | 1 -
library-udf/src/assembly/tools/register-UDF.sh | 1 -
.../apache/iotdb/library/dprofile/UDAFMode.java | 178 -
.../apache/iotdb/metrics/config/MetricConfig.java | 7 +-
.../iotdb/metrics/metricsets/disk/DiskMetrics.java | 30 +-
.../resources/conf/iotdb-common.properties | 13 +
.../service/PipePluginClassLoaderManager.java | 2 +-
.../task/meta/PipeTaskMetaAccessor.java} | 10 +-
.../iotdb/commons/service/StartupChecks.java | 33 +-
.../iotdb/commons/service/metric/enums/Metric.java | 2 +
.../udf/builtin/BuiltinAggregationFunction.java | 5 +-
.../commons/udf/builtin/BuiltinScalarFunction.java | 1 +
.../BuiltinTimeSeriesGeneratingFunction.java | 1 -
.../commons/udf/service/UDFManagementService.java | 2 +-
server/src/main/codegen/dataModel/AllDataType.tdd | 24 +-
.../src/main/codegen/templates/ModeAccumulator.ftl | 179 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 16 +
.../apache/iotdb/db/conf/DataNodeStartupCheck.java | 69 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 -
.../db/consensus/SchemaRegionConsensusImpl.java | 2 -
.../statemachine/DataRegionStateMachine.java | 3 +-
.../org/apache/iotdb/db/constant/SqlConstant.java | 5 +-
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 264 +-
.../readchunk/AlignedSeriesCompactionExecutor.java | 3 +-
.../db/metadata/metric/SchemaEngineMemMetric.java | 16 +-
.../db/metadata/metric/SchemaRegionMemMetric.java | 34 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 28 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 15 +-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 12 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 4 +-
.../metadata/rescon/ISchemaEngineStatistics.java | 2 +
.../metadata/rescon/ISchemaRegionStatistics.java | 4 +
.../metadata/rescon/MemSchemaEngineStatistics.java | 26 +-
.../metadata/rescon/MemSchemaRegionStatistics.java | 36 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 5 +
.../schemaregion/SchemaRegionSchemaFileImpl.java | 5 +
.../metadata/template/ClusterTemplateManager.java | 7 +
.../iotdb/db/metadata/template/Template.java | 4 +
.../db/mpp/aggregation/AccumulatorFactory.java | 21 +
.../SlidingWindowAggregatorFactory.java | 2 +
.../iotdb/db/mpp/execution/driver/Driver.java | 13 +-
.../execution/exchange/MPPDataExchangeManager.java | 67 +-
.../mpp/execution/exchange/SharedTsBlockQueue.java | 6 +-
.../db/mpp/execution/exchange/sink/ISink.java | 4 +
.../mpp/execution/exchange/sink/ISinkHandle.java | 3 +
.../execution/exchange/sink/LocalSinkChannel.java | 7 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 24 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 11 +-
.../execution/exchange/source/SourceHandle.java | 59 +-
.../execution/executor/RegionWriteExecutor.java | 164 +-
.../iotdb/db/mpp/execution/operator/Operator.java | 10 +-
.../process/AbstractConsumeAllOperator.java | 4 +-
.../operator/process/AbstractIntoOperator.java | 7 +-
.../operator/process/AggregationOperator.java | 6 +-
.../operator/process/DeviceMergeOperator.java | 6 +-
.../operator/process/DeviceViewOperator.java | 17 +-
.../execution/operator/process/FillOperator.java | 6 +-
.../operator/process/FilterAndProjectOperator.java | 6 +-
.../execution/operator/process/LimitOperator.java | 6 +-
.../operator/process/LinearFillOperator.java | 8 +-
.../operator/process/MergeSortOperator.java | 8 +-
.../execution/operator/process/OffsetOperator.java | 6 +-
.../process/RawDataAggregationOperator.java | 9 +-
.../operator/process/SingleDeviceViewOperator.java | 6 +-
.../process/SingleInputAggregationOperator.java | 6 +-
.../process/SlidingWindowAggregationOperator.java | 4 +-
.../execution/operator/process/SortOperator.java | 6 +-
.../operator/process/TagAggregationOperator.java | 8 +-
.../operator/process/TransformOperator.java | 16 +-
.../process/join/HorizontallyConcatOperator.java | 8 +-
.../process/join/RowBasedTimeJoinOperator.java | 11 +-
.../operator/process/join/TimeJoinOperator.java | 9 +-
.../last/AbstractUpdateLastCacheOperator.java | 4 +-
.../last/AlignedUpdateLastCacheOperator.java | 2 +-
.../process/last/LastQueryCollectOperator.java | 6 +-
.../process/last/LastQueryMergeOperator.java | 12 +-
.../operator/process/last/LastQueryOperator.java | 7 +-
.../process/last/LastQuerySortOperator.java | 7 +-
.../process/last/UpdateLastCacheOperator.java | 2 +-
.../schema/CountGroupByLevelMergeOperator.java | 6 +-
.../schema/CountGroupByLevelScanOperator.java | 6 +-
.../operator/schema/CountMergeOperator.java | 7 +-
.../schema/NodeManageMemoryMergeOperator.java | 6 +-
.../operator/schema/NodePathsConvertOperator.java | 6 +-
.../operator/schema/NodePathsCountOperator.java | 6 +-
.../operator/schema/SchemaCountOperator.java | 6 +-
.../operator/schema/SchemaFetchMergeOperator.java | 6 +-
.../operator/schema/SchemaFetchScanOperator.java | 6 +-
.../operator/schema/SchemaQueryMergeOperator.java | 6 +-
.../schema/SchemaQueryOrderByHeatOperator.java | 6 +-
.../operator/schema/SchemaQueryScanOperator.java | 6 +-
.../operator/sink/IdentitySinkOperator.java | 35 +-
.../operator/sink/ShuffleHelperOperator.java | 39 +-
.../AbstractSeriesAggregationScanOperator.java | 6 +-
.../operator/source/AlignedSeriesScanOperator.java | 6 +-
.../operator/source/ExchangeOperator.java | 6 +-
.../operator/source/LastCacheScanOperator.java | 6 +-
.../operator/source/SeriesScanOperator.java | 6 +-
.../operator/source/ShowQueriesOperator.java | 6 +-
.../plan/expression/multi/FunctionExpression.java | 2 +-
.../multi/builtin/BuiltInScalarFunctionHelper.java | 4 +-
.../BuiltInScalarFunctionHelperFactory.java | 3 +
.../multi/builtin/helper/RoundFunctionHelper.java | 84 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 15 +
.../planner/distribution/DistributionPlanner.java | 21 +
.../metedata/write/CreateMultiTimeSeriesNode.java | 6 +-
.../plan/node/metedata/write/MeasurementGroup.java | 36 +-
.../db/mpp/plan/scheduler/AsyncPlanNodeSender.java | 34 +
.../scheduler/FragmentInstanceDispatcherImpl.java | 32 +-
.../db/mpp/transformation/api/YieldableReader.java | 6 +-
.../scalar/RoundFunctionColumnTransformer.java | 72 +
.../transformation/dag/input/IUDFInputDataSet.java | 2 +-
.../dag/input/QueryDataSetInputLayer.java | 4 +-
.../dag/input/TsBlockInputDataSet.java | 2 +-
.../MultiInputColumnIntermediateLayer.java | 10 +-
...InputColumnMultiReferenceIntermediateLayer.java | 12 +-
...nputColumnSingleReferenceIntermediateLayer.java | 10 +-
.../dag/transformer/Transformer.java | 4 +-
.../dag/transformer/binary/BinaryTransformer.java | 4 +-
.../transformer/binary/LogicBinaryTransformer.java | 2 +-
.../multi/MappableUDFQueryRowTransformer.java | 2 +-
.../transformer/multi/UDFQueryRowTransformer.java | 2 +-
.../multi/UDFQueryRowWindowTransformer.java | 2 +-
.../multi/UniversalUDFQueryTransformer.java | 4 +-
.../transformer/ternary/TernaryTransformer.java | 4 +-
.../dag/transformer/unary/IsNullTransformer.java | 2 +-
.../dag/transformer/unary/UnaryTransformer.java | 2 +-
.../unary/scalar/DiffFunctionTransformer.java | 2 +-
.../unary/scalar/RoundFunctionTransformer.java | 72 +
.../db/mpp/transformation/dag/udf/UDTFContext.java | 3 +
.../transformation/dag/util/LayerCacheUtils.java | 10 +-
.../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 74 +
.../pipe/agent/{ => plugin}/PipePluginAgent.java | 29 +-
.../db/pipe/agent/runtime/HeartbeatScheduler.java | 11 +-
.../db/pipe/agent/runtime/MetaSyncScheduler.java | 10 +-
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 23 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 23 +-
.../db/pipe/agent/task/PipeTaskRegionAgent.java | 10 +-
.../collector/PipeCollectorEventPendingQueue.java | 10 +-
.../core/collector/PipeCollectorEventSelector.java | 10 +-
.../historical/PipeHistoricalCollector.java | 10 +-
.../collector/realtime/PipeRealtimeCollector.java | 10 +-
.../realtime/cache/PipeRealtimeEventCache.java | 10 +-
.../realtime/listener/IoTLogListerner.java | 10 +-
.../realtime/listener/RatisLogListener.java | 10 +-
.../realtime/listener/SimpleLogListener.java | 10 +-
.../listener/TsFileGenerationListener.java | 10 +-
.../pipe/core/collector/realtime/matcher/Rule.java | 10 +-
.../realtime/matcher/RulePrefixMatchTree.java | 10 +-
.../collector/realtime/recorder/TsFileEpoch.java | 10 +-
.../realtime/recorder/TsFileEpochRecorder.java | 10 +-
.../core/connector/PipeConnectorContainer.java | 10 +-
.../pipe/core/connector/PipeConnectorManager.java | 10 +-
.../PipeConnectorPluginRuntimeWrapper.java | 13 +-
.../pipe/core/event/PipeTabletInsertionEvent.java | 26 +-
.../pipe/core/event/PipeTsFileInsertionEvent.java | 16 +-
.../iotdb/db/pipe/core/event/access/PipeRow.java | 102 +
.../core/event/access/PipeRowIterator.java} | 39 +-
.../core/event/collector/PipeEventCollector.java} | 18 +-
.../core/event/collector/PipeRowCollector.java} | 10 +-
.../pipe/core/event/indexer/PipeEventIndexer.java | 10 +-
.../core/event/indexer/PipeIoTEventIndexer.java | 10 +-
.../core/event/indexer/PipeRatisEventIndexer.java | 10 +-
.../core/event/indexer/PipeSimpleEventIndexer.java | 10 +-
.../core/event/indexer/PipeTsFileEventIndexer.java | 10 +-
.../PipeProcessorPluginRuntimeWrapper.java | 13 +-
.../executor/PipeAssignerSubtaskExecutor.java | 10 +-
.../executor/PipeConnectorSubtaskExecutor.java | 10 +-
.../executor/PipeProcessorSubtaskExecutor.java | 10 +-
.../execution/executor/PipeSubtaskExecutor.java | 10 +-
.../pipe/execution/executor/PipeTaskExecutor.java | 49 +
.../scheduler/PipeAssignerSubtaskScheduler.java | 18 +-
.../scheduler/PipeConnectorSubtaskScheduler.java | 18 +-
.../scheduler/PipeProcessorSubtaskScheduler.java | 18 +-
.../execution/scheduler/PipeSubtaskScheduler.java} | 14 +-
.../execution/scheduler/PipeTaskScheduler.java | 60 +
.../iotdb/db/pipe/resource/PipeFileManager.java | 10 +-
.../iotdb/db/pipe/resource/PipeRaftlogHolder.java | 10 +-
.../iotdb/db/pipe/resource/PipeTsFileHolder.java | 10 +-
.../iotdb/db/pipe/resource/PipeWALHolder.java | 10 +-
.../task/PipeTask.java} | 30 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 11 +-
.../pipe/task/metrics/PipeTaskRuntimeRecorder.java | 10 +-
.../db/pipe/task/runnable/PipeAssignerSubtask.java | 12 +-
.../pipe/task/runnable/PipeConnectorSubtask.java | 12 +-
.../pipe/task/runnable/PipeProcessorSubtask.java | 12 +-
.../iotdb/db/pipe/task/runnable/PipeSubtask.java | 21 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 19 +-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 19 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 19 +-
.../task/stage/PipeTaskStage.java} | 37 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 154 +-
.../db/service/ResourcesInformationHolder.java | 12 +
.../metrics/IoTDBInternalLocalReporter.java | 37 +
.../impl/DataNodeInternalRPCServiceImpl.java | 6 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 2 +
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 3 +
.../schemaRegion/SchemaStatisticsTest.java | 86 +
.../iotdb/db/mpp/execution/exchange/StubSink.java | 5 +
.../operator/AggregationOperatorTest.java | 26 +-
.../AlignedSeriesAggregationScanOperatorTest.java | 38 +-
.../operator/AlignedSeriesScanOperatorTest.java | 6 +-
.../operator/DeviceMergeOperatorTest.java | 565 --
.../execution/operator/DeviceViewOperatorTest.java | 3 +-
.../mpp/execution/operator/FillOperatorTest.java | 16 +-
.../operator/HorizontallyConcatOperatorTest.java | 2 +-
.../operator/LastQueryMergeOperatorTest.java | 38 +-
.../execution/operator/LastQueryOperatorTest.java | 4 +-
.../operator/LastQuerySortOperatorTest.java | 5 +-
.../mpp/execution/operator/LimitOperatorTest.java | 2 +-
.../execution/operator/LinearFillOperatorTest.java | 56 +-
.../execution/operator/MergeSortOperatorTest.java | 38 +-
.../mpp/execution/operator/OffsetOperatorTest.java | 6 +-
.../operator/RawDataAggregationOperatorTest.java | 54 +-
.../SeriesAggregationScanOperatorTest.java | 65 +-
.../execution/operator/SeriesScanOperatorTest.java | 2 +-
.../operator/SingleDeviceViewOperatorTest.java | 3 +-
.../SlidingWindowAggregationOperatorTest.java | 3 +-
.../execution/operator/TimeJoinOperatorTest.java | 6 +-
.../operator/UpdateLastCacheOperatorTest.java | 6 +-
.../operator/schema/SchemaCountOperatorTest.java | 4 +-
.../schema/SchemaQueryScanOperatorTest.java | 4 +-
.../datanode1conf/iotdb-common.properties | 3 +-
.../datanode2conf/iotdb-common.properties | 3 +-
.../datanode3conf/iotdb-common.properties | 3 +-
site/src/main/.vuepress/styles/config.scss | 1 +
thrift-commons/src/main/thrift/common.thrift | 3 +-
.../src/main/thrift/confignode.thrift | 6 +
thrift/src/main/thrift/datanode.thrift | 12 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 7 +
298 files changed, 11274 insertions(+), 7512 deletions(-)
[iotdb] 04/13: finish
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ac917e467f041fd72f1a5afb9b6953bd7461c2bb
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Mar 16 16:35:56 2023 +0800
finish
---
.../operator/source/FileAggregationScanUtil.java | 173 +++++++++++++++++++--
.../mpp/plan/analyze/GroupByLevelController.java | 25 +++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../iotdb/tsfile/read/filter/basic/Filter.java | 8 +
4 files changed, 194 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index be136d531c..2f34285f7c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -19,43 +19,190 @@
package org.apache.iotdb.db.mpp.execution.operator.source;
+import org.apache.iotdb.common.rpc.thrift.TAggregationType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.mpp.aggregation.AccumulatorFactory;
import org.apache.iotdb.db.mpp.aggregation.Aggregator;
+import org.apache.iotdb.db.mpp.plan.analyze.GroupByLevelController;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.materializer.TsFileResourceMaterializer;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.reader.IPageReader;
-import java.util.Comparator;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeSet;
public class FileAggregationScanUtil {
- private final Map<PartialPath, List<Aggregator>> pathToAggregatorsMap;
+ private final Map<PartialPath, Aggregator> pathToAggregatorMap;
private final TsFileResourceMaterializer fileResourceMaterializer;
- private final TreeSet<ChunkMetadata> chunkMetadataList =
- new TreeSet<>(
- Comparator.comparingLong(ChunkMetadata::getVersion)
- .thenComparingLong(ChunkMetadata::getOffsetOfChunkHeader));
+ private final Map<PartialPath, List<IChunkMetadata>> chunkMetadataMap;
+
+ private final PartialPathPool partialPathPool;
+
+ private final int[] levels;
+
+ private final SeriesScanOptions scanOptions;
public FileAggregationScanUtil(
- Map<PartialPath, List<Aggregator>> pathToAggregatorsMap, QueryDataSource dataSource) {
- this.pathToAggregatorsMap = pathToAggregatorsMap;
+ Map<PartialPath, Aggregator> pathToAggregatorMap,
+ QueryDataSource dataSource,
+ int[] levels,
+ SeriesScanOptions scanOptions) {
+ this.pathToAggregatorMap = pathToAggregatorMap;
this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
+ this.chunkMetadataMap = new HashMap<>();
+ this.partialPathPool = new PartialPathPool();
+ this.levels = levels;
+ this.scanOptions = scanOptions;
}
public boolean hasNextFile() {
return fileResourceMaterializer.hasNext();
}
- public void consume() {
+ public void consume() throws IOException {
TsFileResource nextFile = fileResourceMaterializer.next();
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance().get(nextFile.getTsFilePath(), nextFile.isClosed());
+ List<String> allDevices = reader.getAllDevices();
+ for (String device : allDevices) {
+ PartialPath devicePath = partialPathPool.get(device);
+ List<TimeseriesMetadata> timeseriesMetadataMap = reader.getDeviceTimeseriesMetadata(device);
+ for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataMap) {
+ consumeTimeseriesMetadata(devicePath, timeseriesMetadata);
+ }
+ }
+
+ for (Map.Entry<PartialPath, List<IChunkMetadata>> entry : chunkMetadataMap.entrySet()) {
+ PartialPath device = entry.getKey();
+ for (IChunkMetadata chunkMetadata : entry.getValue()) {
+ unpackChunkMetadata(device, chunkMetadata);
+ }
+ }
+ }
+
+ private void unpackChunkMetadata(PartialPath devicePath, IChunkMetadata chunkMetadata)
+ throws IOException {
+ PartialPath groupedPath =
+ partialPathPool.getGroupedPath(devicePath, chunkMetadata.getMeasurementUid());
+
+ List<IPageReader> pageReaderList =
+ FileLoaderUtils.loadPageReaderList(chunkMetadata, scanOptions.getGlobalTimeFilter());
+ for (IPageReader pageReader : pageReaderList) {
+ Filter queryFilter = scanOptions.getQueryFilter();
+ Statistics statistics = pageReader.getStatistics();
+
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ updateAggregationResult(groupedPath, statistics);
+ } else if (queryFilter.allNotSatisfy(statistics)) {
+ // skip
+ } else {
+ pageReader.setFilter(scanOptions.getQueryFilter());
+ updateAggregationResult(groupedPath, pageReader.getAllSatisfiedData());
+ }
+ }
+ }
+
+ private void consumeTimeseriesMetadata(
+ PartialPath devicePath, TimeseriesMetadata timeseriesMetadata) {
+ PartialPath groupedPath =
+ partialPathPool.getGroupedPath(devicePath, timeseriesMetadata.getMeasurementId());
+ if (pathToAggregatorMap.containsKey(groupedPath)) {
+ pathToAggregatorMap.put(
+ groupedPath,
+ new Aggregator(
+ AccumulatorFactory.createAccumulator(
+ TAggregationType.COUNT,
+ timeseriesMetadata.getTSDataType(),
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ true),
+ AggregationStep.SINGLE));
+ }
+
+ Filter queryFilter = scanOptions.getQueryFilter();
+ Statistics statistics = timeseriesMetadata.getStatistics();
+
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ updateAggregationResult(groupedPath, statistics);
+ } else if (queryFilter.allNotSatisfy(statistics)) {
+ // skip
+ } else {
+ consumeChunkMetadataList(devicePath, groupedPath, timeseriesMetadata.getChunkMetadataList());
+ }
+ }
+
+ private void consumeChunkMetadataList(
+ PartialPath devicePath, PartialPath groupedPath, List<IChunkMetadata> chunkMetadataList) {
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ Filter queryFilter = scanOptions.getQueryFilter();
+ Statistics statistics = chunkMetadata.getStatistics();
+
+ if (queryFilter == null || queryFilter.allSatisfy(statistics)) {
+ updateAggregationResult(groupedPath, statistics);
+ } else if (queryFilter.allNotSatisfy(statistics)) {
+ // skip
+ } else {
+ chunkMetadataMap.computeIfAbsent(devicePath, key -> new ArrayList<>()).add(chunkMetadata);
+ }
+ }
+ }
+
+ private void updateAggregationResult(PartialPath groupedPath, Statistics statistics) {}
+
+ private void updateAggregationResult(PartialPath groupedPath, TsBlock tsBlock) {}
+
+ private class PartialPathPool {
+ Map<String, PartialPath> pool;
+ Map<String, PartialPath> rawPathToGroupedPathMap;
+
+ public PartialPathPool() {
+ this.pool = new HashMap<>();
+ this.rawPathToGroupedPathMap = new HashMap<>();
+ }
+
+ public PartialPath get(String pathStr) {
+ if (pool.containsKey(pathStr)) {
+ return pool.get(pathStr);
+ } else {
+ PartialPath path = null;
+ try {
+ path = new PartialPath(pathStr);
+ } catch (IllegalPathException ignored) {
+
+ }
+ pool.put(pathStr, path);
+ return path;
+ }
+ }
- List<ITimeSeriesMetadata> timeSeriesMetadata;
+ public PartialPath getGroupedPath(PartialPath devicePath, String measurementId) {
+ String rawPathStr = devicePath.getDevice().concat(measurementId);
+ if (rawPathToGroupedPathMap.containsKey(rawPathStr)) {
+ return rawPathToGroupedPathMap.get(rawPathStr);
+ }
+ PartialPath groupedPath =
+ GroupByLevelController.groupPathByLevel(devicePath, measurementId, levels);
+ rawPathToGroupedPathMap.put(rawPathStr, groupedPath);
+ return groupedPath;
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
index 8641e3542e..632a5c776e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/GroupByLevelController.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.tsfile.utils.Pair;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -172,6 +173,30 @@ public class GroupByLevelController {
return groupedPath;
}
+ public static PartialPath groupPathByLevel(
+ PartialPath rawDevicePath, String measurement, int[] levels) {
+ String[] nodes = Arrays.copyOf(rawDevicePath.getNodes(), rawDevicePath.getNodes().length + 1);
+ nodes[nodes.length - 1] = measurement;
+
+ Set<Integer> levelSet = new HashSet<>();
+ for (int level : levels) {
+ levelSet.add(level);
+ }
+
+ List<String> transformedNodes = new ArrayList<>(nodes.length);
+
+ transformedNodes.add(nodes[0]);
+ for (int k = 1; k < nodes.length - 1; k++) {
+ if (levelSet.contains(k)) {
+ transformedNodes.add(nodes[k]);
+ } else {
+ transformedNodes.add(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD);
+ }
+ }
+ transformedNodes.add(nodes[nodes.length - 1]);
+ return new PartialPath(transformedNodes.toArray(new String[0]));
+ }
+
public Map<Expression, Set<Expression>> getGroupedExpressionToRawExpressionsMap() {
return groupedExpressionToRawExpressionsMap;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index de57b49e1e..531e41f045 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -1129,7 +1129,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
/* This method will not only deserialize the TimeseriesMetadata, but also all the chunk metadata list meanwhile. */
- private List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
+ public List<TimeseriesMetadata> getDeviceTimeseriesMetadata(String device) throws IOException {
MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair =
getMetadataAndEndOffset(metadataIndexNode, device, true, true);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
index dc0d479c83..9bee46c445 100755
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java
@@ -38,6 +38,14 @@ public interface Filter {
*/
boolean satisfy(Statistics statistics);
+ default boolean allSatisfy(Statistics statistics) {
+ return true;
+ }
+
+ default boolean allNotSatisfy(Statistics statistics) {
+ return false;
+ }
+
/**
* To examine whether the single point(with time and value) is satisfied with the filter.
*
[iotdb] 05/13: add updateAggregationResult
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7dc381bd51cd51ce626f4eef7d9854469a224538
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Fri Mar 17 10:16:20 2023 +0800
add updateAggregationResult
---
.../mpp/execution/operator/source/FileAggregationScanUtil.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index 2f34285f7c..ff12114388 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -166,9 +166,15 @@ public class FileAggregationScanUtil {
}
}
- private void updateAggregationResult(PartialPath groupedPath, Statistics statistics) {}
+ private void updateAggregationResult(PartialPath groupedPath, Statistics statistics) {
+ Statistics[] statisticsList = new Statistics[1];
+ statisticsList[0] = statistics;
+ pathToAggregatorMap.get(groupedPath).processStatistics(statisticsList);
+ }
- private void updateAggregationResult(PartialPath groupedPath, TsBlock tsBlock) {}
+ private void updateAggregationResult(PartialPath groupedPath, TsBlock tsBlock) {
+ pathToAggregatorMap.get(groupedPath).processTsBlock(tsBlock, null, tsBlock.getPositionCount());
+ }
private class PartialPathPool {
Map<String, PartialPath> pool;
[iotdb] 03/13: Merge remote-tracking branch 'origin/master' into lmh/fileScan
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 25d366754f71d0e3c7295d4feb9126fa261c67c2
Merge: fdc10a8ffb 279b775b85
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Mar 16 14:31:18 2023 +0800
Merge remote-tracking branch 'origin/master' into lmh/fileScan
.github/workflows/iotdb-ml.yml | 60 +
.github/workflows/main-unix.yml | 8 +-
.github/workflows/main-win.yml | 6 +-
.github/workflows/site-build.yml | 6 +-
LICENSE-binary | 5 +
antlr/pom.xml | 2 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 4 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 63 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 44 +-
cli/pom.xml | 2 +-
cli/src/assembly/resources/sbin/start-cli.sh | 2 +-
cli/src/assembly/resources/tools/export-csv.sh | 2 +-
cli/src/assembly/resources/tools/export-tsfile.sh | 2 +-
cli/src/assembly/resources/tools/import-csv.sh | 2 +-
cli/src/assembly/resources/tools/load-tsfile.sh | 2 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 54 +-
.../org/apache/iotdb/tool/AbstractCsvTool.java | 22 +-
.../java/org/apache/iotdb/tool/ExportTsFile.java | 10 +
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 12 +-
.../java/org/apache/iotdb/cli/AbstractCliIT.java | 3 +-
client-cpp/pom.xml | 2 +-
client-cpp/src/main/Session.h | 3 +-
client-py/SessionExample.py | 2 +-
client-py/iotdb/Session.py | 11 +-
client-py/iotdb/utils/IoTDBConstants.py | 1 +
client-py/iotdb/utils/IoTDBRpcDataSet.py | 31 +-
client-py/iotdb/utils/NumpyTablet.py | 5 +-
client-py/iotdb/utils/Tablet.py | 5 +-
client-py/pom.xml | 2 +-
client-py/tests/test_session.py | 2 +-
code-coverage/pom.xml | 2 +-
compile-tools/pom.xml | 2 +-
compile-tools/thrift/pom.xml | 2 +-
confignode/pom.xml | 7 +-
.../confignode/client/DataNodeRequestType.java | 4 +
.../async/AsyncConfigNodeHeartbeatClientPool.java | 6 +-
.../client/async/AsyncDataNodeClientPool.java | 4 +-
.../async/AsyncDataNodeHeartbeatClientPool.java | 6 +-
.../client/sync/SyncConfigNodeClientPool.java | 2 +
.../client/sync/SyncDataNodeClientPool.java | 4 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 36 +-
.../confignode/conf/ConfigNodeDescriptor.java | 2 +
.../confignode/conf/ConfigNodeStartupCheck.java | 58 +-
.../confignode/conf/SystemPropertiesUtils.java | 32 +-
.../consensus/request/ConfigPhysicalPlan.java | 16 +-
.../consensus/request/ConfigPhysicalPlanType.java | 8 +-
.../request/read/database/CountDatabasePlan.java | 2 +-
.../read/partition/GetSeriesSlotListPlan.java | 2 +-
.../read/partition/GetTimeSlotListPlan.java | 2 +-
.../read/pipe/plugin/GetPipePluginJarPlan.java | 66 +
.../read/pipe/plugin/GetPipePluginTablePlan.java | 42 +
.../request/read/region/GetRegionIdPlan.java | 2 +-
.../database/AdjustMaxRegionGroupNumPlan.java | 94 +
.../request/write/database/DatabaseSchemaPlan.java | 77 +
.../request/write/database/DeleteDatabasePlan.java | 75 +
.../write/database/PreDeleteDatabasePlan.java | 104 +
.../database/SetDataReplicationFactorPlan.java | 85 +
.../database/SetSchemaReplicationFactorPlan.java | 86 +
.../request/write/database/SetTTLPlan.java | 96 +
.../database/SetTimePartitionIntervalPlan.java | 86 +
.../write/pipe/plugin/CreatePipePluginPlan.java | 72 +
.../write/pipe/plugin/DropPipePluginPlan.java | 56 +
.../storagegroup/AdjustMaxRegionGroupNumPlan.java | 89 -
.../write/storagegroup/DatabaseSchemaPlan.java | 77 -
.../write/storagegroup/DeleteDatabasePlan.java | 75 -
.../write/storagegroup/PreDeleteDatabasePlan.java | 104 -
.../storagegroup/SetDataReplicationFactorPlan.java | 81 -
.../SetSchemaReplicationFactorPlan.java | 81 -
.../request/write/storagegroup/SetTTLPlan.java | 91 -
.../storagegroup/SetTimePartitionIntervalPlan.java | 81 -
.../response/pipe/plugin/PipePluginTableResp.java | 49 +
.../statemachine/ConfigNodeRegionStateMachine.java | 394 -
.../statemachine/ConfigRegionStateMachine.java | 381 +
.../confignode/manager/ClusterSchemaManager.java | 196 +-
.../iotdb/confignode/manager/ConfigManager.java | 215 +-
.../apache/iotdb/confignode/manager/IManager.java | 32 +-
.../iotdb/confignode/manager/ProcedureManager.java | 7 +-
.../manager/consensus/ConsensusManager.java | 20 +-
.../manager/partition/PartitionManager.java | 75 +-
.../iotdb/confignode/manager/pipe/PipeManager.java | 42 +
.../manager/pipe/PipePluginCoordinator.java | 209 +
.../persistence/executor/ConfigPlanExecutor.java | 38 +-
.../partition/DatabasePartitionTable.java | 17 +-
.../persistence/partition/PartitionInfo.java | 110 +-
.../confignode/persistence/pipe/PipeInfo.java | 48 +
.../persistence/pipe/PipePluginInfo.java | 214 +
.../persistence/schema/ClusterSchemaInfo.java | 191 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 19 +-
.../procedure/env/DataNodeRemoveHandler.java | 5 +-
.../procedure/impl/schema/DataNodeRegionTask.java | 195 -
.../impl/schema/DataNodeRegionTaskExecutor.java | 210 +
.../impl/schema/DeactivateTemplateProcedure.java | 242 +-
.../impl/schema/DeleteDatabaseProcedure.java | 99 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 340 +-
.../impl/schema/UnsetTemplateProcedure.java | 121 +-
.../state/schema/DeleteStorageGroupState.java | 5 +-
.../iotdb/confignode/service/ConfigNode.java | 21 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 46 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 42 +-
.../persistence/ClusterSchemaInfoTest.java | 23 +-
.../confignode/persistence/PartitionInfoTest.java | 2 +-
.../confignode/persistence/TemplateTableTest.java | 36 +-
.../impl/DeactivateTemplateProcedureTest.java | 4 +-
.../procedure/impl/OperatePipeProcedureTest.java | 174 -
.../procedure/impl/UnsetTemplateProcedureTest.java | 2 +-
consensus/README.md | 2 +-
consensus/pom.xml | 2 +-
.../common/request/IndexedConsensusRequest.java | 19 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 18 +
.../consensus/iot/IoTConsensusServerImpl.java | 16 +-
.../iot/client/AsyncIoTConsensusServiceClient.java | 19 +-
.../iot/client/IoTConsensusClientPool.java | 4 +
.../iot/client/SyncIoTConsensusServiceClient.java | 32 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 33 +-
.../ratis/ApplicationStateMachineProxy.java | 19 +
.../iotdb/consensus/simple/SimpleConsensus.java | 15 +
.../consensus/iot/util/FakeConsensusReqReader.java | 4 +-
.../apache/iotdb/consensus/ratis/UtilsTest.java | 4 +-
.../consensus/simple/SimpleConsensusTest.java | 4 +-
distribution/pom.xml | 2 +-
.../DockerCompose/docker-compose-cluster-1c2d.yml | 22 +-
.../DockerCompose/docker-compose-cluster-3c3d.yml | 60 -
.../DockerCompose/docker-compose-host-3c3d.yml | 27 +-
.../DockerCompose/docker-compose-standalone.yml | 10 +-
docker/src/main/Dockerfile-1.0.0-confignode | 4 +-
docker/src/main/Dockerfile-1.0.0-datanode | 4 +-
...unity-Powered By.md => Community-Powered-By.md} | 0
docs/Community/Community-Project Committers.md | 22 -
docs/Community/Community-Project-Committers.md | 25 +
docs/Community/Materials.md | 320 +-
docs/Development/VoteRelease.md | 4 +-
docs/README.md | 5 +
docs/UserGuide/API/InfluxDB-Protocol.md | 10 +-
docs/UserGuide/API/Programming-JDBC.md | 2 +-
docs/UserGuide/API/Programming-MQTT.md | 6 +-
.../UserGuide/API/Programming-Python-Native-API.md | 2 +-
docs/UserGuide/Cluster/Cluster-Concept.md | 4 +-
docs/UserGuide/Cluster/Cluster-Maintenance.md | 271 +-
docs/UserGuide/Data-Concept/Compression.md | 2 +
.../Data-Concept/Data-Model-and-Terminology.md | 4 +-
docs/UserGuide/Data-Concept/Schema-Template.md | 6 +-
docs/UserGuide/Data-Concept/Time-Partition.md | 2 +-
docs/UserGuide/Ecosystem-Integration/DBeaver.md | 16 +-
.../Ecosystem-Integration/Grafana-Connector.md | 6 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 58 +-
docs/UserGuide/Ecosystem-Integration/NiFi-IoTDB.md | 4 +-
.../UserGuide/Ecosystem-Integration/Spark-IoTDB.md | 2 +-
.../Ecosystem-Integration/Spark-TsFile.md | 4 +-
.../Ecosystem-Integration/Writing-Data-on-HDFS.md | 2 +-
.../Ecosystem-Integration/Zeppelin-IoTDB.md | 8 +-
.../Edge-Cloud-Collaboration/Sync-Tool.md | 4 +-
.../Integration-Test-refactoring-tutorial.md | 4 +-
docs/UserGuide/IoTDB-Introduction/Architecture.md | 2 +-
docs/UserGuide/IoTDB-Introduction/Scenario.md | 14 +-
docs/UserGuide/Maintenance-Tools/JMX-Tool.md | 4 +-
docs/UserGuide/Maintenance-Tools/Log-Tool.md | 6 +-
.../Maintenance-Tools/Maintenance-Command.md | 15 +-
.../Maintenance-Tools/TsFile-Settle-Tool.md | 41 +
docs/UserGuide/Monitor-Alert/Alerting.md | 2 +-
.../Apache-IoTDB-ConfigNode-Dashboard.json | 2545 ++--
.../Apache-IoTDB-DataNode-Dashboard.json | 7101 ++++++----
.../Apache-IoTDB-Network-Dashboard.json | 534 +
...pache-IoTDB-Performance-Overview-Dashboard.json | 1518 ++-
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 120 +-
.../Operate-Metadata/Auto-Create-MetaData.md | 2 +-
docs/UserGuide/Operate-Metadata/Node.md | 2 +-
docs/UserGuide/Operate-Metadata/Timeseries.md | 2 +-
docs/UserGuide/Operators-Functions/Aggregation.md | 80 +-
.../Operators-Functions/Continuous-Interval.md | 8 +-
docs/UserGuide/Operators-Functions/Conversion.md | 151 +-
.../Operators-Functions/Machine-Learning.md | 123 +
docs/UserGuide/Operators-Functions/Sample.md | 10 +-
docs/UserGuide/Operators-Functions/String.md | 8 +-
.../Operators-Functions/User-Defined-Function.md | 10 +-
docs/UserGuide/Query-Data/Continuous-Query.md | 8 +-
docs/UserGuide/Query-Data/Group-By.md | 73 +-
docs/UserGuide/Query-Data/Overview.md | 11 +-
docs/UserGuide/Query-Data/Pagination.md | 6 +-
docs/UserGuide/Query-Data/Select-Into.md | 24 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 28 +-
docs/UserGuide/QuickStart/QuickStart.md | 2 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 62 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 25 +-
docs/UserGuide/Reference/SQL-Reference.md | 4 +-
docs/UserGuide/Reference/TSDB-Comparison.md | 16 +-
docs/UserGuide/Syntax-Conventions/Identifier.md | 2 +-
docs/UserGuide/Syntax-Conventions/KeyValue-Pair.md | 4 +-
.../UserGuide/Syntax-Conventions/Literal-Values.md | 10 +-
.../Syntax-Conventions/NodeName-In-Path.md | 12 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 18 +-
docs/UserGuide/UDF-Library/M4.md | 26 -
docs/UserGuide/UserGuideReadme.md | 5 +-
docs/UserGuide/Write-Data/Batch-Load-Tool.md | 2 +-
...unity-Powered By.md => Community-Powered-By.md} | 0
docs/zh/Community/Community-Project Committers.md | 22 -
docs/zh/Community/Community-Project-Committers.md | 25 +
docs/zh/README.md | 6 +
docs/zh/UserGuide/API/InfluxDB-Protocol.md | 10 +-
docs/zh/UserGuide/API/Interface-Comparison.md | 2 +-
.../zh/UserGuide/API/Programming-Cpp-Native-API.md | 40 +-
docs/zh/UserGuide/API/Programming-JDBC.md | 2 +-
.../UserGuide/API/Programming-Java-Native-API.md | 48 +-
docs/zh/UserGuide/API/Programming-MQTT.md | 4 +-
.../UserGuide/API/Programming-Python-Native-API.md | 78 +-
docs/zh/UserGuide/Cluster/Cluster-Concept.md | 6 +-
docs/zh/UserGuide/Cluster/Cluster-Maintenance.md | 300 +-
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 58 +-
.../UserGuide/Cluster/Deployment-Recommendation.md | 26 +-
docs/zh/UserGuide/Data-Concept/Compression.md | 7 +-
.../Data-Concept/Data-Model-and-Terminology.md | 4 +-
docs/zh/UserGuide/Data-Concept/Data-Type.md | 14 +-
docs/zh/UserGuide/Data-Concept/Deadband-Process.md | 4 +-
docs/zh/UserGuide/Data-Concept/Encoding.md | 10 +-
docs/zh/UserGuide/Data-Concept/Schema-Template.md | 16 +-
docs/zh/UserGuide/Data-Concept/Time-Partition.md | 10 +-
docs/zh/UserGuide/Data-Concept/Time-zone.md | 4 +-
docs/zh/UserGuide/Data-Modeling/DataRegion.md | 8 +-
.../Data-Modeling/SchemaRegion-rocksdb.md | 8 +-
docs/zh/UserGuide/Delete-Data/TTL.md | 8 +-
docs/zh/UserGuide/Ecosystem-Integration/DBeaver.md | 16 +-
.../Ecosystem-Integration/Grafana-Connector.md | 6 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 98 +-
.../UserGuide/Ecosystem-Integration/NiFi-IoTDB.md | 24 +-
.../UserGuide/Ecosystem-Integration/Workbench.md | 420 +
.../Ecosystem-Integration/Writing-Data-on-HDFS.md | 2 +-
.../Ecosystem-Integration/Zeppelin-IoTDB.md | 8 +-
.../Edge-Cloud-Collaboration/Sync-Tool.md | 2 +-
docs/zh/UserGuide/FAQ/FAQ-for-cluster-setup.md | 36 +-
.../Integration-Test-refactoring-tutorial.md | 4 +-
.../UserGuide/IoTDB-Introduction/Architecture.md | 2 +-
docs/zh/UserGuide/IoTDB-Introduction/Scenario.md | 14 +-
docs/zh/UserGuide/Maintenance-Tools/CSV-Tool.md | 24 +-
.../IoTDB-Data-Dir-Overview-Tool.md | 6 +-
docs/zh/UserGuide/Maintenance-Tools/JMX-Tool.md | 8 +-
docs/zh/UserGuide/Maintenance-Tools/Load-Tsfile.md | 14 +-
docs/zh/UserGuide/Maintenance-Tools/Log-Tool.md | 6 +-
.../Maintenance-Tools/Maintenance-Command.md | 7 +-
.../Maintenance-Tools/TsFile-Load-Export-Tool.md | 24 +-
.../TsFile-Resource-Sketch-Tool.md | 6 +-
.../Maintenance-Tools/TsFile-Settle-Tool.md | 42 +
.../Maintenance-Tools/TsFile-Sketch-Tool.md | 6 +-
.../Maintenance-Tools/TsFile-Split-Tool.md | 2 +-
.../Maintenance-Tools/TsFileSelfCheck-Tool.md | 4 +-
docs/zh/UserGuide/Monitor-Alert/Alerting.md | 26 +-
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 122 +-
.../Operate-Metadata/Auto-Create-MetaData.md | 12 +-
docs/zh/UserGuide/Operate-Metadata/Database.md | 11 +-
docs/zh/UserGuide/Operate-Metadata/Node.md | 14 +-
docs/zh/UserGuide/Operate-Metadata/Template.md | 16 +-
docs/zh/UserGuide/Operate-Metadata/Timeseries.md | 16 +-
.../UserGuide/Operators-Functions/Aggregation.md | 100 +-
.../Operators-Functions/Anomaly-Detection.md | 56 +-
.../zh/UserGuide/Operators-Functions/Comparison.md | 18 +-
docs/zh/UserGuide/Operators-Functions/Constant.md | 2 +-
.../Operators-Functions/Continuous-Interval.md | 12 +-
.../zh/UserGuide/Operators-Functions/Conversion.md | 161 +-
.../UserGuide/Operators-Functions/Data-Matching.md | 32 +-
.../Operators-Functions/Data-Profiling.md | 170 +-
.../UserGuide/Operators-Functions/Data-Quality.md | 48 +-
.../Operators-Functions/Data-Repairing.md | 48 +-
.../Operators-Functions/Frequency-Domain.md | 54 +-
docs/zh/UserGuide/Operators-Functions/Lambda.md | 8 +-
docs/zh/UserGuide/Operators-Functions/Logical.md | 6 +-
.../Operators-Functions/Machine-Learning.md | 133 +-
.../UserGuide/Operators-Functions/Mathematical.md | 12 +-
docs/zh/UserGuide/Operators-Functions/Overview.md | 18 +-
docs/zh/UserGuide/Operators-Functions/Sample.md | 50 +-
docs/zh/UserGuide/Operators-Functions/Selection.md | 2 +-
.../Operators-Functions/Series-Discovery.md | 18 +-
docs/zh/UserGuide/Operators-Functions/String.md | 106 +-
.../UserGuide/Operators-Functions/Time-Series.md | 8 +-
.../Operators-Functions/User-Defined-Function.md | 62 +-
.../Operators-Functions/Variation-Trend.md | 10 +-
docs/zh/UserGuide/Query-Data/Align-By.md | 6 +-
docs/zh/UserGuide/Query-Data/Continuous-Query.md | 60 +-
docs/zh/UserGuide/Query-Data/Fill.md | 14 +-
docs/zh/UserGuide/Query-Data/Group-By.md | 114 +-
docs/zh/UserGuide/Query-Data/Having-Condition.md | 2 +-
docs/zh/UserGuide/Query-Data/Last-Query.md | 2 +-
docs/zh/UserGuide/Query-Data/Order-By.md | 6 +-
docs/zh/UserGuide/Query-Data/Overview.md | 56 +-
docs/zh/UserGuide/Query-Data/Pagination.md | 10 +-
docs/zh/UserGuide/Query-Data/Select-Expression.md | 22 +-
docs/zh/UserGuide/Query-Data/Select-Into.md | 70 +-
docs/zh/UserGuide/Query-Data/Where-Condition.md | 12 +-
docs/zh/UserGuide/QuickStart/ClusterQuickStart.md | 20 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 26 +-
docs/zh/UserGuide/QuickStart/Files.md | 32 +-
docs/zh/UserGuide/QuickStart/QuickStart.md | 2 +-
docs/zh/UserGuide/QuickStart/ServerFileList.md | 30 +-
docs/zh/UserGuide/QuickStart/WayToGetIoTDB.md | 50 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 64 +-
.../Reference/ConfigNode-Config-Manual.md | 18 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 20 +-
docs/zh/UserGuide/Reference/Keywords.md | 2 +-
docs/zh/UserGuide/Reference/SQL-Reference.md | 26 +-
docs/zh/UserGuide/Reference/Status-Codes.md | 2 +-
docs/zh/UserGuide/Reference/TSDB-Comparison.md | 44 +-
.../Syntax-Conventions/Detailed-Grammar.md | 2 +-
docs/zh/UserGuide/Syntax-Conventions/Identifier.md | 10 +-
.../UserGuide/Syntax-Conventions/KeyValue-Pair.md | 2 +-
.../Keywords-And-Reserved-Words.md | 2 +-
.../UserGuide/Syntax-Conventions/Literal-Values.md | 17 +-
.../Syntax-Conventions/NodeName-In-Path.md | 6 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 2 +-
.../UserGuide/Trigger/Configuration-Parameters.md | 2 +-
docs/zh/UserGuide/Trigger/Implement-Trigger.md | 20 +-
docs/zh/UserGuide/Trigger/Instructions.md | 10 +-
docs/zh/UserGuide/Trigger/Notes.md | 2 +-
docs/zh/UserGuide/Trigger/Trigger-Management.md | 10 +-
docs/zh/UserGuide/UserGuideReadme.md | 7 +-
docs/zh/UserGuide/Write-Data/Batch-Load-Tool.md | 6 +-
docs/zh/UserGuide/Write-Data/MQTT.md | 2 +-
docs/zh/UserGuide/Write-Data/REST-API.md | 2 +-
docs/zh/UserGuide/Write-Data/Session.md | 4 +-
example/client-cpp-example/pom.xml | 2 +-
example/ext-pipe-plugin-example/pom.xml | 4 +-
example/flink/pom.xml | 2 +-
example/hadoop/pom.xml | 2 +-
example/influxdb-protocol-example/pom.xml | 2 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 2 +-
example/mqtt-customize/pom.xml | 2 +-
example/mqtt/pom.xml | 2 +-
example/pom.xml | 2 +-
example/pulsar/pom.xml | 2 +-
example/rabbitmq/pom.xml | 2 +-
example/rest-java-example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
example/session/pom.xml | 2 +-
example/trigger/pom.xml | 2 +-
example/tsfile/pom.xml | 2 +-
example/udf/pom.xml | 2 +-
external-api/pom.xml | 2 +-
external-pipe-api/pom.xml | 2 +-
flink-iotdb-connector/pom.xml | 2 +-
flink-tsfile-connector/pom.xml | 2 +-
.../iotdb/flink/tsfile/TsFileOutputFormat.java | 2 +-
grafana-connector/pom.xml | 2 +-
grafana-plugin/pkg/main.go | 2 +-
.../pkg/plugin/iotdb_resource_handler.go | 26 +-
grafana-plugin/pkg/plugin/plugin.go | 228 +-
grafana-plugin/pom.xml | 2 +-
grafana-plugin/src/ConfigEditor.tsx | 50 +-
grafana-plugin/src/QueryEditor.tsx | 12 +-
grafana-plugin/src/datasource.ts | 19 +-
grafana-plugin/src/types.ts | 1 +
grafana-plugin/yarn.lock | 11545 +++++++----------
hadoop/pom.xml | 2 +-
hive-connector/pom.xml | 2 +-
influxdb-protocol/pom.xml | 2 +-
integration-test/pom.xml | 2 +-
.../iotdb/db/query/udf/example/ConstValue.java | 42 +
.../iotdb/it/env/cluster/MppCommonConfig.java | 8 +-
.../it/env/cluster/MppSharedCommonConfig.java | 13 +-
.../iotdb/it/env/remote/RemoteCommonConfig.java | 7 +-
.../itbase/constant/BuiltinScalarFunctionEnum.java | 37 +
.../BuiltinTimeSeriesGeneratingFunctionEnum.java | 1 -
.../apache/iotdb/itbase/constant/TestConstant.java | 4 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +-
.../confignode/it/IoTDBSnapshotTransferIT.java | 4 +-
.../it/load/IoTDBConfigNodeSwitchLeaderIT.java | 2 +-
.../it/partition/IoTDBPartitionGetterIT.java | 6 +-
.../iotdb/db/it/IoTDBInsertMultiPartitionIT.java | 71 +
.../it/IoTDBSyntaxConventionStringLiteralIT.java | 7 +-
.../db/it/aggregation/IoTDBAggregationIT.java | 9 +
.../it/aggregation/IoTDBAggregationOptimizeIT.java | 8 +-
.../db/it/aggregation/IoTDBTagAggregationIT.java | 55 +-
.../db/it/aggregation/IoTDBTimeDurationIT.java | 368 +
.../scalar/IoTDBCastFunctionIT.java | 1032 ++
.../scalar/IoTDBReplaceFunctionIT.java | 171 +
.../db/it/groupby/IoTDBGroupByConditionIT.java | 2 +-
.../iotdb/db/it/groupby/IoTDBGroupByCountIT.java | 407 +
.../iotdb/db/it/groupby/IoTDBGroupBySessionIT.java | 18 +-
.../it/last/IoTDBLastQueryWithLimitOffsetIT.java | 193 +
.../IoTDBNoSelectExpressionAfterAnalyzedIT.java | 93 +
.../iotdb/db/it/schema/AbstractSchemaIT.java | 2 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 19 +-
.../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 2 +-
.../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 2 +-
.../org/apache/iotdb/db/it/sync/IoTDBPipeIT.java | 200 -
.../apache/iotdb/db/it/sync/IoTDBPipeSinkIT.java | 115 -
.../apache/iotdb/db/it/sync/SyncTransportTest.java | 323 -
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 14 +-
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 1424 --
.../it/udf/IoTDBUDTFBuiltinScalarFunctionIT.java | 1424 ++
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 12 +-
integration/pom.xml | 2 +-
.../db/integration/IoTDBSchemaTemplateIT.java | 357 -
.../db/integration/sync/IoTDBSyncReceiverIT.java | 164 -
.../sync/IoTDBSyncReceiverLoaderIT.java | 146 -
.../db/integration/sync/IoTDBSyncSenderIT.java | 587 -
.../iotdb/db/integration/sync/MockSyncClient.java | 50 -
.../iotdb/db/integration/sync/SyncTestUtil.java | 238 -
isession/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 1 +
library-udf/pom.xml | 2 +-
metrics/dropwizard-metrics/pom.xml | 2 +-
metrics/interface/pom.xml | 2 +-
.../iotdb/metrics/metricsets/disk/DiskMetrics.java | 26 +-
.../metricsets/disk/LinuxDiskMetricsManager.java | 66 +-
.../metrics/metricsets/net/INetMetricManager.java | 62 +
.../metricsets/net/LinuxNetMetricManager.java | 234 +
.../metricsets/net/MacNetMetricManager.java | 22 +
.../iotdb/metrics/metricsets/net/NetMetrics.java | 114 +
.../metricsets/net/WindowsNetMetricManager.java | 22 +
.../reporter/prometheus/PrometheusReporter.java | 2 -
.../iotdb/metrics/type/HistogramSnapshot.java | 1 -
metrics/micrometer-metrics/pom.xml | 4 +-
.../micrometer/type/MicrometerAutoGauge.java | 2 -
metrics/pom.xml | 2 +-
mlnode/iotdb/mlnode/serde.py | 439 +
mlnode/pom.xml | 2 +-
mlnode/requirements.txt | 22 +
mlnode/requirements_dev.txt | 21 +
mlnode/test/test_serde.py | 129 +
node-commons/pom.xml | 7 +-
.../resources/conf/iotdb-common.properties | 8 +-
.../src/assembly/resources/sbin/iotdb-common.sh | 2 +-
.../assembly/resources/sbin/start-standalone.sh | 2 +-
.../src/assembly/resources/sbin/stop-standalone.sh | 2 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 11 +-
.../iotdb/commons/client/ClientPoolFactory.java | 24 +-
.../iotdb/commons/client/IClientManager.java | 8 +-
.../iotdb/commons/client/IClientPoolFactory.java | 3 +
.../apache/iotdb/commons/client/ThriftClient.java | 29 +-
.../AsyncConfigNodeHeartbeatServiceClient.java | 144 -
.../async/AsyncConfigNodeIServiceClient.java | 23 +-
.../async/AsyncDataNodeHeartbeatServiceClient.java | 144 -
.../async/AsyncDataNodeInternalServiceClient.java | 24 +-
.../AsyncDataNodeMPPDataExchangeServiceClient.java | 23 +-
.../client/property/ThriftClientProperty.java | 32 +-
.../client/sync/SyncConfigNodeIServiceClient.java | 32 +-
.../sync/SyncDataNodeInternalServiceClient.java | 32 +-
.../SyncDataNodeMPPDataExchangeServiceClient.java | 32 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 10 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 2 +
.../commons/consensus/ConfigNodeRegionId.java | 34 -
.../iotdb/commons/consensus/ConfigRegionId.java | 34 +
.../iotdb/commons/consensus/ConsensusGroupId.java | 8 +-
.../commons/exception/ConfigurationException.java | 27 +-
.../meta/ConfigNodePipePluginMetaKeeper.java | 108 +
.../plugin/meta/DataNodePipePluginMetaKeeper.java | 53 +
.../commons/pipe/plugin/meta/PipePluginMeta.java | 130 +
.../pipe/plugin/meta/PipePluginMetaKeeper.java | 52 +
.../pipe/plugin/service/PipePluginClassLoader.java | 90 +
.../service/PipePluginClassLoaderManager.java | 103 +
.../service/PipePluginExecutableManager.java | 94 +
.../apache/iotdb/commons/schema/node/IMNode.java | 88 +
.../iotdb/commons/schema/node/MNodeType.java | 82 +
.../node/common/AbstractAboveDatabaseMNode.java | 191 +
.../node/common/AbstractDatabaseDeviceMNode.java | 290 +
.../schema/node/common/AbstractDatabaseMNode.java | 215 +
.../schema/node/common/AbstractDeviceMNode.java | 280 +
.../node/common/AbstractMeasurementMNode.java | 256 +
.../schema/node/info/IDatabaseDeviceInfo.java | 24 +
.../commons/schema/node/info/IDatabaseInfo.java | 33 +
.../commons/schema/node/info/IDeviceInfo.java | 68 +
.../commons/schema/node/info/IMeasurementInfo.java | 48 +
.../commons/schema/node/role/IDatabaseMNode.java | 28 +
.../commons/schema/node/role/IDeviceMNode.java | 57 +
.../schema/node/role/IMeasurementMNode.java | 48 +
.../commons/schema/node/utils/IMNodeContainer.java | 25 +
.../commons/schema/node/utils/IMNodeFactory.java | 42 +
.../commons/schema/node/utils/IMNodeIterator.java | 28 +
.../commons/schema/node/visitor/MNodeVisitor.java | 42 +
.../apache/iotdb/commons/service/ServiceType.java | 5 +-
.../iotdb/commons/service/metric/enums/Metric.java | 12 +
.../metric/enums/PerformanceOverviewMetrics.java | 180 +
.../iotdb/commons/sync/pipe/TsFilePipeInfo.java | 12 +-
.../udf/builtin/BuiltinAggregationFunction.java | 4 +-
.../iotdb/commons/udf/builtin/BuiltinFunction.java | 70 -
.../commons/udf/builtin/BuiltinScalarFunction.java | 65 +
.../BuiltinTimeSeriesGeneratingFunction.java | 1 -
.../apache/iotdb/commons/udf/builtin/UDTFCast.java | 253 -
.../commons/utils/ThriftCommonsSerDeUtils.java | 13 +-
.../commons/utils/ThriftConfigNodeSerDeUtils.java | 8 +-
.../commons/utils/ThriftCommonsSerDeUtilsTest.java | 2 +-
.../utils/ThriftConfigNodeSerDeUtilsTest.java | 4 +-
openapi/pom.xml | 2 +-
pipe-api/pom.xml | 2 +-
.../org/apache/iotdb/pipe/api/PipeConnector.java | 2 +-
.../java/org/apache/iotdb/pipe/api/PipePlugin.java | 22 +
.../org/apache/iotdb/pipe/api/PipeProcessor.java | 2 +-
.../api/exception/PipeManagementException.java | 32 +
pom.xml | 20 +-
rewrite-tsfile-tool/pom.xml | 2 +-
.../src/assembly/resources/sbin/rewrite-tsfile.sh | 2 +-
schema-engine-rocksdb/pom.xml | 2 +-
.../rocksdb/RSchemaReadWriteHandler.java | 4 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 22 +-
.../schemaregion/rocksdb/RSchemaUtils.java | 10 +-
.../schemaregion/rocksdb/mnode/RDatabaseMNode.java | 117 +
.../schemaregion/rocksdb/mnode/RDeviceMNode.java | 171 +
.../schemaregion/rocksdb/mnode/REntityMNode.java | 134 -
.../schemaregion/rocksdb/mnode/RInternalMNode.java | 30 +-
.../schemaregion/rocksdb/mnode/RMNode.java | 101 +-
.../rocksdb/mnode/RMeasurementMNode.java | 47 +-
.../rocksdb/mnode/RStorageGroupMNode.java | 118 -
schema-engine-tag/pom.xml | 2 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 6 +-
server/pom.xml | 2 +-
.../src/assembly/resources/sbin/remove-datanode.sh | 2 +-
.../resources/tools/schema/print-schema-file.sh | 2 +-
.../resources/tools/schema/print-schema-log.sh | 2 +-
.../resources/tools/tsfile/print-iotdb-data-dir.sh | 2 +-
.../resources/tools/tsfile/print-tsfile.sh | 2 +-
.../resources/tools/tsfile/settle-tsfile.sh | 2 +-
.../resources/tools/tsfile/split-tsfile-tool.sh | 2 +-
.../resources/tools/tsfile/validate-tsfile.sh | 2 +-
.../codegen/templates/abstractVariationWindow.ftl | 84 +
.../src/main/codegen/templates/evEventWindow.ftl | 67 -
.../codegen/templates/evEventWindowManager.ftl | 117 -
server/src/main/codegen/templates/eventWindow.ftl | 84 -
.../main/codegen/templates/eventWindowManager.ftl | 51 -
.../src/main/codegen/templates/variationWindow.ftl | 67 +
.../codegen/templates/variationWindowManager.ftl | 117 +
.../org/apache/iotdb/db/audit/AuditLogger.java | 4 +
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 18 +-
.../iotdb/db/auth/ClusterAuthorityFetcher.java | 14 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 113 +-
.../iotdb/db/client/ConfigNodeClientManager.java | 8 +-
.../org/apache/iotdb/db/client/ConfigNodeInfo.java | 8 +-
.../iotdb/db/client/DataNodeClientPoolFactory.java | 14 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 1 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 55 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 5 +-
.../statemachine/DataRegionStateMachine.java | 11 +-
.../IoTConsensusDataRegionStateMachine.java | 8 +-
.../org/apache/iotdb/db/constant/SqlConstant.java | 7 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 6 +
.../execute/task/InnerSpaceCompactionTask.java | 9 +-
.../compaction/schedule/CompactionScheduler.java | 39 +-
.../compaction/schedule/CompactionTaskManager.java | 3 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 47 +-
.../impl/SizeTieredCompactionSelector.java | 25 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 40 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 129 +-
.../engine/storagegroup/TimePartitionManager.java | 9 +-
.../db/engine/storagegroup/TsFileProcessor.java | 81 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 14 +-
.../storagegroup/TsFileProcessorInfoMetrics.java | 4 +-
.../metadata/DatabaseAlreadySetException.java | 60 +
.../metadata/DatabaseNotSetException.java | 45 +
.../metadata/StorageGroupAlreadySetException.java | 60 -
.../metadata/StorageGroupNotSetException.java | 45 -
.../apache/iotdb/db/metadata/MetadataConstant.java | 2 +
.../idtable/entry/InsertMeasurementMNode.java | 287 -
.../db/metadata/mnode/AboveDatabaseMNode.java | 37 -
.../iotdb/db/metadata/mnode/EntityMNode.java | 188 -
.../iotdb/db/metadata/mnode/IEntityMNode.java | 49 -
.../org/apache/iotdb/db/metadata/mnode/IMNode.java | 106 -
.../iotdb/db/metadata/mnode/IMeasurementMNode.java | 48 -
.../db/metadata/mnode/IStorageGroupMNode.java | 39 -
.../iotdb/db/metadata/mnode/InternalMNode.java | 259 -
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 200 -
.../apache/iotdb/db/metadata/mnode/MNodeType.java | 82 -
.../apache/iotdb/db/metadata/mnode/MNodeUtils.java | 77 -
.../iotdb/db/metadata/mnode/MeasurementMNode.java | 233 -
.../db/metadata/mnode/StorageGroupEntityMNode.java | 90 -
.../iotdb/db/metadata/mnode/StorageGroupMNode.java | 102 -
.../db/metadata/mnode/config/IConfigMNode.java | 45 +
.../mnode/config/basic/ConfigBasicMNode.java | 279 +
.../config/container/ConfigMNodeContainer.java | 93 +
.../mnode/config/factory/ConfigMNodeFactory.java | 81 +
.../config/impl/ConfigBasicInternalMNode.java | 181 +
.../mnode/config/impl/ConfigDatabaseMNode.java | 80 +
.../mnode/config/info/ConfigDatabaseInfo.java | 70 +
.../mnode/config/info/ConfigMNodeInfo.java | 82 +
.../metadata/mnode/container/IMNodeContainer.java | 25 -
.../mnode/container/MNodeContainerMapImpl.java | 26 -
.../metadata/mnode/container/MNodeContainers.java | 110 -
.../mnode/estimator/BasicMNodSizeEstimator.java | 114 -
.../mnode/estimator/IMNodeSizeEstimator.java | 38 -
.../mnode/iterator/AbstractTraverserIterator.java | 115 -
.../mnode/iterator/CachedTraverserIterator.java | 45 -
.../db/metadata/mnode/iterator/IMNodeIterator.java | 28 -
.../db/metadata/mnode/iterator/MNodeIterator.java | 48 -
.../mnode/iterator/MemoryTraverserIterator.java | 34 -
.../iotdb/db/metadata/mnode/mem/IMemMNode.java | 23 +
.../db/metadata/mnode/mem/basic/BasicMNode.java | 252 +
.../mnode/mem/container/MemMNodeContainer.java | 94 +
.../mnode/mem/estimator/MNodeSizeEstimator.java | 28 +
.../mnode/mem/factory/MemMNodeFactory.java | 84 +
.../mnode/mem/impl/AboveDatabaseMNode.java | 35 +
.../mnode/mem/impl/BasicInternalMNode.java | 181 +
.../mnode/mem/impl/DatabaseDeviceMNode.java | 37 +
.../db/metadata/mnode/mem/impl/DatabaseMNode.java | 43 +
.../db/metadata/mnode/mem/impl/DeviceMNode.java | 36 +
.../metadata/mnode/mem/impl/MeasurementMNode.java | 49 +
.../db/metadata/mnode/mem/info/BasicMNodeInfo.java | 41 +
.../mnode/mem/info/DatabaseDeviceInfo.java | 59 +
.../db/metadata/mnode/mem/info/DatabaseInfo.java | 58 +
.../db/metadata/mnode/mem/info/DeviceInfo.java | 195 +
.../metadata/mnode/mem/info/MeasurementInfo.java | 113 +
.../mem/iterator/AbstractTraverserIterator.java | 121 +
.../metadata/mnode/mem/iterator/MNodeIterator.java | 49 +
.../mem/iterator/MemoryTraverserIterator.java | 40 +
.../db/metadata/mnode/schemafile/ICachedMNode.java | 28 +
.../mnode/schemafile/basic/CachedBasicMNode.java | 264 +
.../schemafile/container/CachedMNodeContainer.java | 507 +
.../container/ICachedMNodeContainer.java | 79 +
.../schemafile/factory/CacheMNodeFactory.java | 84 +
.../schemafile/impl/CachedAboveDatabaseMNode.java | 47 +
.../schemafile/impl/CachedBasicInternalMNode.java | 191 +
.../schemafile/impl/CachedDatabaseDeviceMNode.java | 49 +
.../mnode/schemafile/impl/CachedDatabaseMNode.java | 54 +
.../mnode/schemafile/impl/CachedDeviceMNode.java | 48 +
.../schemafile/impl/CachedMeasurementMNode.java | 60 +
.../mnode/schemafile/info/CacheMNodeInfo.java | 45 +
.../iterator/CachedTraverserIterator.java | 51 +
.../iotdb/db/metadata/mnode/utils/MNodeUtils.java | 123 +
.../db/metadata/mnode/visitor/MNodeVisitor.java | 39 -
.../iotdb/db/metadata/mtree/ConfigMTree.java | 324 +-
.../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 195 -
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 419 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 378 +-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 142 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 169 +-
.../db/metadata/mtree/store/ConfigMTreeStore.java | 143 +
.../iotdb/db/metadata/mtree/store/IMTreeStore.java | 44 +-
.../db/metadata/mtree/store/MemMTreeStore.java | 147 +-
.../store/ReentrantReadOnlyCachedMTreeStore.java | 44 +-
.../mtree/store/disk/CachedMNodeContainer.java | 443 -
.../mtree/store/disk/ICachedMNodeContainer.java | 80 -
.../mtree/store/disk/cache/CacheManager.java | 139 +-
.../mtree/store/disk/cache/CacheMemoryManager.java | 11 +
.../mtree/store/disk/cache/ICacheManager.java | 28 +-
.../mtree/store/disk/cache/LRUCacheManager.java | 24 +-
.../mtree/store/disk/cache/PlainCacheManager.java | 10 +-
.../disk/memcontrol/CachedMNodeSizeEstimator.java | 72 -
.../mtree/store/disk/memcontrol/MemManager.java | 26 +-
.../mtree/store/disk/schemafile/ISchemaFile.java | 17 +-
.../store/disk/schemafile/ISegmentedPage.java | 8 +-
.../store/disk/schemafile/MockSchemaFile.java | 128 +-
.../mtree/store/disk/schemafile/RecordUtils.java | 84 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 57 +-
.../mtree/store/disk/schemafile/SegmentedPage.java | 41 +-
.../store/disk/schemafile/WrappedSegment.java | 20 +-
.../disk/schemafile/pagemgr/BTreePageManager.java | 30 +-
.../disk/schemafile/pagemgr/IPageManager.java | 13 +-
.../store/disk/schemafile/pagemgr/PageManager.java | 34 +-
.../db/metadata/mtree/traverser/Traverser.java | 49 +-
.../traverser/TraverserWithLimitOffsetWrapper.java | 18 +-
.../mtree/traverser/basic/DatabaseTraverser.java | 22 +-
.../mtree/traverser/basic/EntityTraverser.java | 19 +-
.../mtree/traverser/basic/MNodeTraverser.java | 21 +-
.../traverser/basic/MeasurementTraverser.java | 14 +-
.../traverser/collector/DatabaseCollector.java | 14 +-
.../mtree/traverser/collector/EntityCollector.java | 14 +-
.../traverser/collector/MNodeAboveDBCollector.java | 62 +
.../traverser/collector/MNodeAboveSGCollector.java | 62 -
.../mtree/traverser/collector/MNodeCollector.java | 10 +-
.../traverser/collector/MeasurementCollector.java | 22 +-
.../mtree/traverser/counter/DatabaseCounter.java | 10 +-
.../mtree/traverser/counter/EntityCounter.java | 9 +-
.../traverser/counter/MeasurementCounter.java | 9 +-
.../mtree/traverser/updater/EntityUpdater.java | 15 +-
.../traverser/updater/MeasurementUpdater.java | 13 +-
.../plan/schemaregion/result/ShowNodesResult.java | 2 +-
.../db/metadata/query/info/INodeSchemaInfo.java | 2 +-
.../metadata/schemaregion/ISchemaRegionParams.java | 36 +
.../metadata/schemaregion/RSchemaRegionLoader.java | 24 +-
.../db/metadata/schemaregion/SchemaEngine.java | 72 +-
.../db/metadata/schemaregion/SchemaRegion.java | 31 +
.../metadata/schemaregion/SchemaRegionLoader.java | 124 +
.../schemaregion/SchemaRegionMemoryImpl.java | 71 +-
.../metadata/schemaregion/SchemaRegionParams.java | 67 +
.../schemaregion/SchemaRegionSchemaFileImpl.java | 119 +-
.../apache/iotdb/db/metadata/tag/TagManager.java | 55 +-
.../metadata/template/ClusterTemplateManager.java | 18 +-
.../iotdb/db/metadata/template/Template.java | 352 +-
.../iotdb/db/metadata/utils/MetaFormatUtils.java | 9 +-
.../apache/iotdb/db/metadata/utils/MetaUtils.java | 2 +-
.../iotdb/db/mpp/aggregation/Accumulator.java | 16 +-
.../db/mpp/aggregation/AccumulatorFactory.java | 2 +
.../iotdb/db/mpp/aggregation/Aggregator.java | 35 +-
.../iotdb/db/mpp/aggregation/AvgAccumulator.java | 90 +-
.../iotdb/db/mpp/aggregation/CountAccumulator.java | 23 +-
.../db/mpp/aggregation/CountIfAccumulator.java | 25 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 90 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 150 +-
.../mpp/aggregation/FirstValueDescAccumulator.java | 110 +-
.../db/mpp/aggregation/LastValueAccumulator.java | 132 +-
.../mpp/aggregation/LastValueDescAccumulator.java | 122 +-
.../db/mpp/aggregation/MaxTimeAccumulator.java | 22 +-
.../db/mpp/aggregation/MaxTimeDescAccumulator.java | 25 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 94 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 25 +-
.../db/mpp/aggregation/MinTimeDescAccumulator.java | 20 +-
.../db/mpp/aggregation/MinValueAccumulator.java | 93 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 92 +-
.../mpp/aggregation/TimeDurationAccumulator.java | 124 +
.../slidingwindow/SlidingWindowAggregator.java | 5 +-
.../SlidingWindowAggregatorFactory.java | 2 +
.../db/mpp/common/header/ColumnHeaderConstant.java | 34 +-
.../db/mpp/common/header/DatasetHeaderFactory.java | 4 +
.../iotdb/db/mpp/execution/driver/DataDriver.java | 6 +
.../execution/exchange/sink/ShuffleSinkHandle.java | 20 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 6 +-
.../execution/exchange/source/SourceHandle.java | 7 +-
.../execution/executor/RegionWriteExecutor.java | 10 +-
.../fragment/FragmentInstanceContext.java | 8 +-
.../fragment/FragmentInstanceExecution.java | 4 +
.../fragment/FragmentInstanceManager.java | 21 +-
.../iotdb/db/mpp/execution/memory/MemoryPool.java | 237 +-
.../db/mpp/execution/operator/AggregationUtil.java | 18 +-
.../operator/process/AggregationOperator.java | 2 +-
.../operator/process/MergeSortOperator.java | 9 +
.../process/RawDataAggregationOperator.java | 48 +-
.../process/SlidingWindowAggregationOperator.java | 5 +-
.../schema/NodeManageMemoryMergeOperator.java | 2 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/window/AbstractVariationWindow.java | 91 +
.../execution/operator/window/ConditionWindow.java | 108 +
.../operator/window/ConditionWindowManager.java | 171 +
.../operator/window/ConditionWindowParameter.java | 53 +
.../mpp/execution/operator/window/CountWindow.java | 101 +
.../operator/window/CountWindowManager.java | 141 +
.../operator/window/CountWindowParameter.java | 46 +
.../mpp/execution/operator/window/EventWindow.java | 87 -
.../operator/window/EventWindowManager.java | 122 -
.../operator/window/EventWindowParameter.java | 61 -
.../db/mpp/execution/operator/window/IWindow.java | 21 +-
.../execution/operator/window/IWindowManager.java | 52 +-
.../execution/operator/window/SeriesWindow.java | 113 -
.../operator/window/SeriesWindowManager.java | 196 -
.../operator/window/SeriesWindowParameter.java | 53 -
.../execution/operator/window/SessionWindow.java | 11 +-
.../operator/window/SessionWindowManager.java | 33 +-
.../mpp/execution/operator/window/TimeWindow.java | 8 -
.../operator/window/TimeWindowManager.java | 24 +-
.../operator/window/VariationWindowManager.java | 100 +
.../operator/window/VariationWindowParameter.java | 61 +
.../operator/window/WindowManagerFactory.java | 42 +-
.../mpp/execution/operator/window/WindowType.java | 7 +-
.../db/mpp/metric/PerformanceOverviewMetrics.java | 142 -
.../metric/PerformanceOverviewMetricsManager.java | 323 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 120 +-
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 18 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 697 +-
.../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 7 +-
.../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 93 +-
.../db/mpp/plan/analyze/cache/PartitionCache.java | 10 +-
.../db/mpp/plan/execution/QueryExecution.java | 43 +-
.../plan/execution/config/ConfigTaskVisitor.java | 66 +-
.../config/executor/ClusterConfigTaskExecutor.java | 390 +-
.../config/executor/IConfigTaskExecutor.java | 32 +-
.../config/metadata/CountDatabaseTask.java | 63 +
.../config/metadata/CountStorageGroupTask.java | 63 -
.../config/metadata/CreatePipePluginTask.java | 42 +
.../config/metadata/DatabaseSchemaTask.java | 23 +-
.../config/metadata/DeleteStorageGroupTask.java | 10 +-
.../config/metadata/DropPipePluginTask.java | 42 +
.../config/metadata/ShowDatabaseTask.java | 42 +
.../config/metadata/ShowFunctionsTask.java | 15 +
.../config/metadata/ShowPipePluginsTask.java | 80 +
.../config/metadata/ShowStorageGroupTask.java | 42 -
.../execution/config/sys/pipe/CreatePipeTask.java | 42 +
.../execution/config/sys/pipe/DropPipeTask.java | 42 +
.../execution/config/sys/pipe/ShowPipeTask.java | 80 +
.../execution/config/sys/pipe/StartPipeTask.java | 42 +
.../execution/config/sys/pipe/StopPipeTask.java | 42 +
.../execution/config/sys/sync/CreatePipeTask.java | 42 -
.../execution/config/sys/sync/DropPipeTask.java | 42 -
.../execution/config/sys/sync/ShowPipeTask.java | 80 -
.../execution/config/sys/sync/StartPipeTask.java | 42 -
.../execution/config/sys/sync/StopPipeTask.java | 42 -
.../memory/StatementMemorySourceVisitor.java | 2 +-
.../iotdb/db/mpp/plan/expression/Expression.java | 1 -
.../plan/expression/multi/FunctionExpression.java | 65 +-
.../db/mpp/plan/expression/multi/FunctionType.java | 2 +-
.../multi/builtin/BuiltInFunctionHelper.java | 22 +
.../multi/builtin/BuiltInScalarFunctionHelper.java | 93 +
.../BuiltInScalarFunctionHelperFactory.java | 41 +
.../multi/builtin/helper/CastFunctionHelper.java | 150 +
.../multi/builtin/helper/DiffFunctionHelper.java | 68 +
.../builtin/helper/ReplaceFunctionHelper.java | 91 +
.../BindTypeForTimeSeriesOperandVisitor.java | 71 +
.../visitor/CartesianProductVisitor.java | 57 +
.../CollectAggregationExpressionsVisitor.java | 43 +
.../visitor/CollectSourceExpressionsVisitor.java | 47 +
.../plan/expression/visitor/CollectVisitor.java | 56 +
.../visitor/ColumnTransformerVisitor.java | 23 +-
.../ConcatDeviceAndRemoveWildcardVisitor.java | 112 +
.../ConcatExpressionWithSuffixPathsVisitor.java | 113 +
.../visitor/ExpressionAnalyzeVisitor.java | 39 +
.../visitor/GetMeasurementExpressionVisitor.java | 56 +
.../visitor/IntermediateLayerVisitor.java | 22 +-
.../expression/visitor/ReconstructVisitor.java | 63 +
.../visitor/RemoveAliasFromExpressionVisitor.java | 54 +
.../visitor/RemoveWildcardInExpressionVisitor.java | 99 +
.../RemoveWildcardInFilterByDeviceVisitor.java | 103 +
.../visitor/RemoveWildcardInFilterVisitor.java | 149 +
.../ReplaceRawPathWithGroupedPathVisitor.java | 80 +
.../mpp/plan/optimization/LimitOffsetPushDown.java | 3 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 292 +-
.../db/mpp/plan/parser/StatementGenerator.java | 80 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 2 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 74 +-
.../db/mpp/plan/planner/SubPlanTypeExtractor.java | 16 +-
.../planner/distribution/DistributionPlanner.java | 73 +-
.../planner/distribution/ExchangeNodeAdder.java | 15 +-
.../planner/distribution/NodeGroupContext.java | 25 +-
.../SimpleFragmentParallelPlanner.java | 16 +-
.../db/mpp/plan/planner/plan/PlanFragment.java | 18 -
.../db/mpp/plan/planner/plan/node/PlanNode.java | 10 -
.../planner/plan/node/process/GroupByTagNode.java | 27 +
.../plan/node/sink/MultiChildrenSinkNode.java | 4 +
.../plan/parameter/AggregationDescriptor.java | 4 +
.../plan/parameter/GroupByConditionParameter.java | 87 +
.../plan/parameter/GroupByCountParameter.java | 85 +
.../planner/plan/parameter/GroupByParameter.java | 6 +-
.../plan/parameter/GroupBySeriesParameter.java | 87 -
.../plan/parameter/GroupByVariationParameter.java | 2 +-
.../db/mpp/plan/scheduler/AsyncPlanNodeSender.java | 16 +-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 28 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 4 +
.../iotdb/db/mpp/plan/statement/StatementType.java | 8 +-
.../db/mpp/plan/statement/StatementVisitor.java | 46 +-
.../component/GroupByConditionComponent.java | 41 +
.../statement/component/GroupByCountComponent.java | 34 +
.../component/GroupBySeriesComponent.java | 41 -
.../component/GroupByVariationComponent.java | 2 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 17 +-
.../statement/metadata/CountDatabaseStatement.java | 42 +
.../metadata/CountStorageGroupStatement.java | 42 -
.../metadata/CreatePipePluginStatement.java | 72 +
.../metadata/DatabaseSchemaStatement.java | 16 +-
.../metadata/DeleteDatabaseStatement.java | 77 +
.../metadata/DeleteStorageGroupStatement.java | 77 -
.../metadata/DropPipePluginStatement.java | 60 +
.../plan/statement/metadata/SetTTLStatement.java | 14 +-
.../statement/metadata/ShowDatabaseStatement.java | 130 +
.../metadata/ShowPipePluginsStatement.java | 52 +
.../metadata/ShowStorageGroupStatement.java | 130 -
.../template/CreateSchemaTemplateStatement.java | 121 +-
.../statement/sys/pipe/CreatePipeStatement.java | 90 +
.../plan/statement/sys/pipe/DropPipeStatement.java | 62 +
.../plan/statement/sys/pipe/ShowPipeStatement.java | 63 +
.../statement/sys/pipe/StartPipeStatement.java | 62 +
.../plan/statement/sys/pipe/StopPipeStatement.java | 62 +
.../statement/sys/sync/CreatePipeStatement.java | 128 -
.../plan/statement/sys/sync/DropPipeStatement.java | 62 -
.../plan/statement/sys/sync/ShowPipeStatement.java | 50 -
.../statement/sys/sync/StartPipeStatement.java | 62 -
.../plan/statement/sys/sync/StopPipeStatement.java | 62 -
.../unary/DiffFunctionColumnTransformer.java | 67 -
.../scalar/CastFunctionColumnTransformer.java | 230 +
.../scalar/DiffFunctionColumnTransformer.java | 68 +
.../scalar/ReplaceFunctionColumnTransformer.java | 57 +
.../transformer/unary/DiffFunctionTransformer.java | 103 -
.../unary/scalar/CastFunctionTransformer.java | 228 +
.../unary/scalar/DiffFunctionTransformer.java | 104 +
.../unary/scalar/ReplaceFunctionTransformer.java | 50 +
.../iotdb/db/pipe/agent/PipePluginAgent.java | 191 +
.../db/protocol/rest/handler/ExceptionHandler.java | 6 +-
.../iotdb/db/rescon/PrimitiveArrayManager.java | 14 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 16 +-
.../iotdb/db/rescon/TsFileResourceManager.java | 5 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 10 +-
.../db/service/DataNodeServerCommandLine.java | 6 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +-
.../iotdb/db/service/RegionMigrateService.java | 2 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 3 +
.../iotdb/db/service/metrics/WritingMetrics.java | 245 +
.../metrics/recorder/WritingMetricsManager.java | 351 +
.../service/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 377 +-
.../java/org/apache/iotdb/db/sync/SyncService.java | 29 -
.../db/sync/common/ClusterSyncInfoFetcher.java | 12 +-
.../db/sync/transport/server/ReceiverManager.java | 2 +-
.../tools/settle/TsFileSettleByCompactionTool.java | 4 +-
.../db/trigger/executor/TriggerFireVisitor.java | 8 +-
.../trigger/service/TriggerInformationUpdater.java | 8 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 4 +
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 46 +-
.../apache/iotdb/db/utils/sync/SyncPipeUtil.java | 40 +-
.../java/org/apache/iotdb/db/wal/WALManager.java | 7 +
.../iotdb/db/wal/allocation/ElasticStrategy.java | 6 +
.../db/wal/allocation/FirstCreateStrategy.java | 6 +
.../db/wal/allocation/NodeAllocationStrategy.java | 3 +
.../db/wal/allocation/RoundRobinStrategy.java | 6 +
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 22 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 10 +
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 10 +-
.../inner/InnerCompactionSchedulerTest.java | 6 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 4 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 18 +-
.../iotdb/db/metadata/idtable/IDTableTest.java | 2 +-
.../apache/iotdb/db/metadata/mnode/MNodeTest.java | 64 +-
.../iotdb/db/metadata/mtree/ConfigMTreeTest.java | 96 +-
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 257 -
.../mtree/disk/CachedMNodeContainerTest.java | 25 +-
.../mtree/multimode/MTreeDiskModeTest.java | 42 -
.../mtree/multimode/MTreeFullMemoryTest.java | 28 -
.../mtree/multimode/MTreeMemoryModeTest.java | 35 -
.../mtree/multimode/MTreeNonMemoryTest.java | 28 -
.../mtree/multimode/MTreePartialMemoryTest.java | 28 -
.../metadata/mtree/schemafile/RecordUtilTests.java | 32 +-
.../mtree/schemafile/SchemaFileLogTest.java | 23 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 301 +-
.../metadata/mtree/schemafile/SchemaPageTest.java | 26 +-
.../mtree/schemafile/WrappedSegmentTest.java | 48 +-
.../schemaRegion/SchemaRegionBasicTest.java | 2 +-
.../schemaRegion/SchemaRegionManagementTest.java | 8 +-
.../schemaRegion/SchemaRegionTemplateTest.java | 56 +-
.../schemaRegion/SchemaStatisticsTest.java | 97 +-
.../iotdb/db/mpp/aggregation/AccumulatorTest.java | 55 +-
.../iotdb/db/mpp/execution/exchange/Utils.java | 4 +-
.../db/mpp/execution/memory/MemoryPoolTest.java | 26 +-
.../execution/operator/MergeSortOperatorTest.java | 96 +-
.../operator/RawDataAggregationOperatorTest.java | 15 +-
.../NodeManagementMemoryMergeNodeSerdeTest.java | 2 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 2 +-
.../iotdb/db/rescon/ResourceManagerTest.java | 56 +-
.../iotdb/db/tools/SchemaFileSketchTest.java | 43 +-
server/src/test/resources/start-sync-test.sh | 2 +-
server/src/test/resources/stop-sync-test.sh | 2 +-
service-rpc/pom.xml | 2 +-
.../apache/iotdb/rpc/ConfigurableTByteBuffer.java | 100 +
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 1 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 14 +-
session/pom.xml | 2 +-
site/iotdb-doap.rdf | 8 +
site/pom.xml | 107 +-
site/replace-picture-urls.sh | 3 +-
site/src/main/.eslintrc.cjs | 5 +-
site/src/main/.gitignore | 8 +
site/src/main/.vuepress/client.ts | 34 +
site/src/main/.vuepress/components/Contributor.vue | 411 +
site/src/main/.vuepress/components/IoTDB.vue | 308 +
site/src/main/.vuepress/components/IoTDBZH.vue | 302 +
site/src/main/.vuepress/components/PageFooter.vue | 52 +
.../docsearch/client/components/Docsearch.ts | 155 +
.../docsearch/client/components/index.ts | 20 +
.../docsearch/client/composables/index.ts | 20 +
.../client/composables/useDocsearchShim.ts | 81 +
.../components/docsearch/client/config.js | 28 +
.../.vuepress/components/docsearch/client/index.ts | 22 +
.../components/docsearch/client/shims.d.ts | 26 +
.../docsearch/client/styles/docsearch.css | 44 +
.../components/docsearch/node/docsearchPlugin.ts | 56 +
.../.vuepress/components/docsearch/node/index.ts | 24 +
.../.vuepress/components/docsearch/shared/index.ts | 20 +
.../.vuepress/components/docsearch/shared/types.ts | 37 +
site/src/main/.vuepress/config.js | 1538 ---
site/src/main/.vuepress/config.ts | 126 +
site/src/main/.vuepress/enhanceApp.js | 31 -
site/src/main/.vuepress/navbar/en.ts | 75 +
site/src/main/.vuepress/navbar/index.ts | 21 +
site/src/main/.vuepress/navbar/zh.ts | 75 +
site/src/main/.vuepress/public/favicon.ico | Bin 0 -> 2816 bytes
site/src/main/.vuepress/public/img/IOTDB.png | Bin 0 -> 20818 bytes
.../TsFile/TsFile/tsFileVectorIndexCase5.png | Bin 0 -> 37197 bytes
.../TsFile/TsFile/tsFileVectorIndexCase6.png | Bin 0 -> 71207 bytes
.../TsFile/TsFile/tsFileVectorIndexCase7.png | Bin 0 -> 81847 bytes
.../API/IoTDB-InfluxDB/architecture-design.png | Bin 0 -> 145997 bytes
.../UserGuide/API/IoTDB-InfluxDB/class-diagram.png | Bin 0 -> 16881 bytes
.../UserGuide/API/IoTDB-InfluxDB/influxdb-data.png | Bin 0 -> 59411 bytes
.../API/IoTDB-InfluxDB/influxdb-vs-iotdb-data.png | Bin 0 -> 202016 bytes
.../UserGuide/API/IoTDB-InfluxDB/iotdb-data.png | Bin 0 -> 53434 bytes
.../API/IoTDB-SQLAlchemy/sqlalchemy-to-iotdb.png | Bin 0 -> 173461 bytes
.../AdministrationConsole.png | Bin 0 -> 197363 bytes
.../CLI/Command-Line-Interface/add_Realm_1.png | Bin 0 -> 212253 bytes
.../CLI/Command-Line-Interface/add_Realm_2.png | Bin 0 -> 49422 bytes
.../CLI/Command-Line-Interface/add_role1.png | Bin 0 -> 79274 bytes
.../CLI/Command-Line-Interface/add_role2.png | Bin 0 -> 55567 bytes
.../CLI/Command-Line-Interface/add_role3.png | Bin 0 -> 67423 bytes
.../CLI/Command-Line-Interface/add_role4.png | Bin 0 -> 73825 bytes
.../CLI/Command-Line-Interface/add_role5.png | Bin 0 -> 82391 bytes
.../CLI/Command-Line-Interface/client.png | Bin 0 -> 132520 bytes
.../CLI/Command-Line-Interface/login_keycloak.png | Bin 0 -> 219950 bytes
.../UserGuide/CLI/Command-Line-Interface/pwd.png | Bin 0 -> 235839 bytes
.../UserGuide/CLI/Command-Line-Interface/user.png | Bin 0 -> 167194 bytes
.../public/img/UserGuide/Cluster/Architecture.png | Bin 0 -> 55767 bytes
.../img/UserGuide/Cluster/Data-Partition.png | Bin 0 -> 106930 bytes
.../UserGuide/Cluster/DataNode-StateMachine-EN.jpg | Bin 0 -> 70582 bytes
.../UserGuide/Cluster/DataNode-StateMachine-ZH.jpg | Bin 0 -> 66988 bytes
.../img/UserGuide/Cluster/Preview1-Function.png | Bin 0 -> 256908 bytes
.../auto_create_sg_example.png | Bin 0 -> 81521 bytes
.../example_template_lifetime.png | Bin 0 -> 283522 bytes
.../example_template_lifetime_zh.png | Bin 0 -> 308440 bytes
.../Measurement-Template/example_with_template.png | Bin 0 -> 28982 bytes
.../example_without_template.png | Bin 0 -> 58632 bytes
.../Time-Partition/time_partition_example.png | Bin 0 -> 160113 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/01.png | Bin 0 -> 163272 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/02.png | Bin 0 -> 123947 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/03.png | Bin 0 -> 144867 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/04.png | Bin 0 -> 228587 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/05.png | Bin 0 -> 198084 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/06.png | Bin 0 -> 191463 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/07.png | Bin 0 -> 222235 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/08.png | Bin 0 -> 58150 bytes
.../UserGuide/Ecosystem-Integration/DBeaver/09.png | Bin 0 -> 350352 bytes
.../Grafana-plugin/add-empty-panel.png | Bin 0 -> 98208 bytes
.../Grafana-plugin/addvaribles.png | Bin 0 -> 196588 bytes
.../Grafana-plugin/alertPanel5.png | Bin 0 -> 42293 bytes
.../Grafana-plugin/alerting1.png | Bin 0 -> 103200 bytes
.../Grafana-plugin/alerting2.png | Bin 0 -> 44192 bytes
.../Grafana-plugin/alerting3.png | Bin 0 -> 119687 bytes
.../Grafana-plugin/alerting4.png | Bin 0 -> 130345 bytes
.../Grafana-plugin/alertpanel3.png | Bin 0 -> 250072 bytes
.../Grafana-plugin/alertpanel4.png | Bin 0 -> 85865 bytes
.../Grafana-plugin/alertpanel6.png | Bin 0 -> 164736 bytes
.../Grafana-plugin/alertpanle1.png | Bin 0 -> 333611 bytes
.../Grafana-plugin/alertpanle2.jpg | Bin 0 -> 137699 bytes
.../Grafana-plugin/applyvariables.png | Bin 0 -> 314991 bytes
.../Grafana-plugin/datasource_1.png | Bin 0 -> 275723 bytes
.../Grafana-plugin/datasource_2.png | Bin 0 -> 123473 bytes
.../Grafana-plugin/datasource_3.png | Bin 0 -> 167831 bytes
.../Grafana-plugin/distribution.png | Bin 0 -> 95592 bytes
.../Grafana-plugin/grafana-plugin-build.png | Bin 0 -> 48467 bytes
.../Grafana-plugin/grafana_input.png | Bin 0 -> 279162 bytes
.../Grafana-plugin/grafana_input2.png | Bin 0 -> 299017 bytes
.../Grafana-plugin/grafana_input_style.png | Bin 0 -> 268438 bytes
.../Grafana-plugin/manage.png | Bin 0 -> 224614 bytes
.../Grafana-plugin/setconf.png | Bin 0 -> 197642 bytes
.../Grafana-plugin/variablesinput3-1.png | Bin 0 -> 316272 bytes
.../Grafana-plugin/variablesinput3.png | Bin 0 -> 95470 bytes
.../Grafana-plugin/variblesinput.png | Bin 0 -> 92755 bytes
.../Grafana-plugin/variblesinput2-1.png | Bin 0 -> 94878 bytes
.../Grafana-plugin/variblesinput2-2.png | Bin 0 -> 326277 bytes
.../Grafana-plugin/variblesinput2.png | Bin 0 -> 92655 bytes
.../Ecosystem-Integration/Workbench/image10.jpeg | Bin 0 -> 131946 bytes
.../Ecosystem-Integration/Workbench/image11.jpeg | Bin 0 -> 487707 bytes
.../Ecosystem-Integration/Workbench/image12.png | Bin 0 -> 45444 bytes
.../Ecosystem-Integration/Workbench/image13.jpeg | Bin 0 -> 488050 bytes
.../Ecosystem-Integration/Workbench/image14.jpeg | Bin 0 -> 457975 bytes
.../Ecosystem-Integration/Workbench/image15.jpeg | Bin 0 -> 493580 bytes
.../Ecosystem-Integration/Workbench/image16.jpeg | Bin 0 -> 444360 bytes
.../Ecosystem-Integration/Workbench/image17.jpeg | Bin 0 -> 512202 bytes
.../Ecosystem-Integration/Workbench/image18.jpeg | Bin 0 -> 99839 bytes
.../Ecosystem-Integration/Workbench/image19.jpeg | Bin 0 -> 596956 bytes
.../Ecosystem-Integration/Workbench/image20.png | Bin 0 -> 674410 bytes
.../Ecosystem-Integration/Workbench/image21.jpeg | Bin 0 -> 694550 bytes
.../Ecosystem-Integration/Workbench/image22.jpeg | Bin 0 -> 808356 bytes
.../Ecosystem-Integration/Workbench/image23.jpeg | Bin 0 -> 827186 bytes
.../Ecosystem-Integration/Workbench/image24.png | Bin 0 -> 179727 bytes
.../Ecosystem-Integration/Workbench/image25.jpeg | Bin 0 -> 529074 bytes
.../Ecosystem-Integration/Workbench/image26.jpeg | Bin 0 -> 307123 bytes
.../Ecosystem-Integration/Workbench/image27.png | Bin 0 -> 350407 bytes
.../Ecosystem-Integration/Workbench/image28.jpeg | Bin 0 -> 737495 bytes
.../Ecosystem-Integration/Workbench/image29.jpeg | Bin 0 -> 71283 bytes
.../Ecosystem-Integration/Workbench/image30.png | Bin 0 -> 200568 bytes
.../Ecosystem-Integration/Workbench/image31.jpeg | Bin 0 -> 433529 bytes
.../Ecosystem-Integration/Workbench/image32.jpeg | Bin 0 -> 539089 bytes
.../Ecosystem-Integration/Workbench/image33.png | Bin 0 -> 273003 bytes
.../Ecosystem-Integration/Workbench/image34.png | Bin 0 -> 148569 bytes
.../Ecosystem-Integration/Workbench/image35.png | Bin 0 -> 186961 bytes
.../Ecosystem-Integration/Workbench/image36.png | Bin 0 -> 349267 bytes
.../Ecosystem-Integration/Workbench/image37.png | Bin 0 -> 400709 bytes
.../Ecosystem-Integration/Workbench/image38.png | Bin 0 -> 349067 bytes
.../Ecosystem-Integration/Workbench/image39.jpeg | Bin 0 -> 941372 bytes
.../Ecosystem-Integration/Workbench/image40.png | Bin 0 -> 168395 bytes
.../Ecosystem-Integration/Workbench/image41.png | Bin 0 -> 342396 bytes
.../Ecosystem-Integration/Workbench/image42.png | Bin 0 -> 470241 bytes
.../Ecosystem-Integration/Workbench/image43.png | Bin 0 -> 110992 bytes
.../Ecosystem-Integration/Workbench/image44.png | Bin 0 -> 241073 bytes
.../Ecosystem-Integration/Workbench/image45.png | Bin 0 -> 253855 bytes
.../Ecosystem-Integration/Workbench/image5.jpeg | Bin 0 -> 7926 bytes
.../Ecosystem-Integration/Workbench/image6.jpeg | Bin 0 -> 395220 bytes
.../Ecosystem-Integration/Workbench/image7.png | Bin 0 -> 392808 bytes
.../Ecosystem-Integration/Workbench/image8.jpeg | Bin 0 -> 107764 bytes
.../Ecosystem-Integration/Workbench/image9.jpeg | Bin 0 -> 146196 bytes
.../Architecture/Structure-of-Apache-IoTDB-cn.png | Bin 0 -> 126257 bytes
.../Architecture/Structure-of-Apache-IoTDB.png | Bin 0 -> 103140 bytes
.../public/img/UserGuide/OtherMaterial-Sample.Data | 516 +
.../Process-Data/Continuous-Query/pic1.png | Bin 0 -> 28418 bytes
.../Process-Data/Continuous-Query/pic2.png | Bin 0 -> 25023 bytes
.../Process-Data/Continuous-Query/pic3.png | Bin 0 -> 32982 bytes
.../Process-Data/Continuous-Query/pic4.png | Bin 0 -> 36135 bytes
.../Process-Data/GroupBy/groupBySession.jpeg | Bin 0 -> 45630 bytes
.../Process-Data/GroupBy/groupByVariation.jpeg | Bin 0 -> 59469 bytes
.../Process-Data/Triggers/ForwardQueueConsume.png | Bin 0 -> 190973 bytes
.../Process-Data/Triggers/Trigger_Process_Flow.jpg | Bin 0 -> 25591 bytes
.../Triggers/Trigger_Process_Strategy.jpg | Bin 0 -> 70240 bytes
.../UDF-User-Defined-Function/countWindow.png | Bin 0 -> 102766 bytes
.../UDF-User-Defined-Function/sessionWindow.png | Bin 0 -> 109806 bytes
.../UDF-User-Defined-Function/stateWindow.png | Bin 0 -> 94322 bytes
.../UDF-User-Defined-Function/timeWindow.png | Bin 0 -> 100258 bytes
.../UserGuide/System-Tools/Metrics/dashboard.png | Bin 0 -> 203647 bytes
.../Metrics/iotdb_prometheus_grafana.png | Bin 0 -> 304191 bytes
.../System-Tools/Metrics/metrics_demo_1.png | Bin 0 -> 262783 bytes
.../System-Tools/Metrics/metrics_demo_2.png | Bin 0 -> 191559 bytes
.../img/UserGuide/System-Tools/Sync-Tool/pipe2.png | Bin 0 -> 58248 bytes
...752940-50407b00-43a5-11eb-94fb-3e3be222183c.png | Bin 0 -> 169114 bytes
...752945-5171a800-43a5-11eb-8614-53b3276a3ce2.png | Bin 0 -> 87397 bytes
...752947-520a3e80-43a5-11eb-8fb1-8fac471c8c7e.png | Bin 0 -> 173739 bytes
...752948-52a2d500-43a5-11eb-9156-0c55667eb4cd.png | Bin 0 -> 125270 bytes
...251336-cf03c000-624f-11eb-8395-de5e349f47b5.png | Bin 0 -> 99963 bytes
...251353-d32fdd80-624f-11eb-80c1-fdb4197939fe.png | Bin 0 -> 112717 bytes
...251369-d7f49180-624f-11eb-9d19-fc7341582b90.png | Bin 0 -> 89864 bytes
...251377-daef8200-624f-11eb-9678-b1d5440be2de.png | Bin 0 -> 111910 bytes
...251391-df1b9f80-624f-11eb-9f1f-66823839acba.png | Bin 0 -> 94191 bytes
...251411-e5aa1700-624f-11eb-8ca8-00c0627b1e96.png | Bin 0 -> 98958 bytes
...254214-6cacbe80-6253-11eb-8532-d6a1829f8f66.png | Bin 0 -> 95325 bytes
...426760-73e3da80-8d73-11eb-9a8f-9232d1f2033b.png | Bin 0 -> 76603 bytes
...125919-f4850800-9929-11eb-8211-81d4c04af1ec.png | Bin 0 -> 142774 bytes
...957896-a9791080-a537-11eb-9962-541412bdcee6.png | Bin 0 -> 110029 bytes
...790229-23e34900-b8c8-11eb-87da-ac01dd117f28.png | Bin 0 -> 32058 bytes
...833923-182ffc00-bf32-11eb-8b3f-9f95d3729ad2.png | Bin 0 -> 324124 bytes
...668849-b1c69280-d1ec-11eb-83cb-3b73c40bdf72.png | Bin 0 -> 85937 bytes
...542457-5f511d00-d77c-11eb-8006-562d83069baa.png | Bin 0 -> 85488 bytes
...951720-707f1ee8-32ee-4fde-9252-048caebd232e.png | Bin 0 -> 293510 bytes
...178733-a0919d17-0663-4672-9c4f-1efad6f463c2.png | Bin 0 -> 101545 bytes
...181449-00d563c8-7bce-4ecd-a031-ec120ca42c3f.png | Bin 0 -> 66497 bytes
...183015-93b56644-3330-4acf-ae9e-d718a02b5f4c.png | Bin 0 -> 16451 bytes
...018765-6fda9391-ebcf-4c80-98c5-26f34bd74df0.png | Bin 0 -> 350636 bytes
...577195-f94d7500-1ef3-11e9-999a-b4f67055d80e.png | Bin 0 -> 41935 bytes
...577204-fe122900-1ef3-11e9-9e89-2eb1d46e24b8.png | Bin 0 -> 221206 bytes
...577216-09fdeb00-1ef4-11e9-9005-542ad7d9e9e0.png | Bin 0 -> 203454 bytes
...579014-695ef980-1efa-11e9-8cbc-e9e7ee4fa0d8.png | Bin 0 -> 861503 bytes
...579033-7ed42380-1efa-11e9-889f-fb4180291a9e.png | Bin 0 -> 386176 bytes
...579064-8f849980-1efa-11e9-8cd6-a7339cd0540f.jpg | Bin 0 -> 303859 bytes
...579080-96aba780-1efa-11e9-87ac-940c45b19dd7.jpg | Bin 0 -> 234375 bytes
...579095-a4f9c380-1efa-11e9-9f95-17165ec55568.jpg | Bin 0 -> 200091 bytes
...664777-2766ae00-1ff5-11e9-9d2f-7489f8ccbfc2.png | Bin 0 -> 277087 bytes
...664842-554bf280-1ff5-11e9-97d2-54eebe0b2ca1.png | Bin 0 -> 333531 bytes
...664878-6e54a380-1ff5-11e9-9718-4d0e24627fa8.png | Bin 0 -> 276994 bytes
...937461-14296f80-a303-11e9-9602-a7bed624bfb3.png | Bin 0 -> 238093 bytes
...922722-35180400-f05a-11e9-8ff0-7dd51716e4a8.png | Bin 0 -> 16824 bytes
...943956-39c1e800-fc16-11e9-8da2-a662f8246816.png | Bin 0 -> 213188 bytes
...109512-f808bc80-0ab2-11ea-9e4d-b2b2f58fb474.png | Bin 0 -> 16914 bytes
...792176-1718f400-1201-11ea-861a-1a83c07ca144.jpg | Bin 0 -> 14952 bytes
...357432-0c71cf80-75e4-11ea-98aa-c43a54d469ce.png | Bin 0 -> 38174 bytes
...357469-1bf11880-75e4-11ea-978f-a53996667a0d.png | Bin 0 -> 25267 bytes
...351839-bd288900-7f6b-11ea-8d12-feb18c35adad.png | Bin 0 -> 179218 bytes
...414865-5f815480-7fde-11ea-800c-47c7dbad7648.png | Bin 0 -> 101410 bytes
...462909-53a53e80-91e8-11ea-98df-0012380da0b2.png | Bin 0 -> 77209 bytes
...462914-5738c580-91e8-11ea-94d1-4ff6607e7e2c.png | Bin 0 -> 121334 bytes
...464569-725e0200-91f5-11ea-9ff9-49745f4c9ef2.png | Bin 0 -> 115173 bytes
...464639-ed271d00-91f5-11ea-91a0-b4fe9cb8204e.png | Bin 0 -> 160280 bytes
...768477-b874d780-950d-11ea-80ca-8807b9bd0970.png | Bin 0 -> 476193 bytes
...768490-bf034f00-950d-11ea-9b56-fef3edca0958.png | Bin 0 -> 332540 bytes
...197835-99a64980-1f62-11eb-84af-8301b8a6aad5.png | Bin 0 -> 71304 bytes
...197920-be9abc80-1f62-11eb-9efb-027f0590031c.png | Bin 0 -> 71488 bytes
...197948-cf4b3280-1f62-11eb-9c8c-c97d1adf032c.png | Bin 0 -> 116419 bytes
...633970-73671c00-235d-11eb-9913-f38e570fcfc8.png | Bin 0 -> 36452 bytes
...787878-47b51480-2b5b-11eb-8ed3-84088c5c30f7.png | Bin 0 -> 112663 bytes
.../img/zh/development/howtocontributecode/01.png | Bin 0 -> 89974 bytes
.../img/zh/development/howtocontributecode/02.png | Bin 0 -> 121641 bytes
.../img/zh/development/howtocontributecode/03.png | Bin 0 -> 214508 bytes
.../zh/development/howtocontributecode/issue.png | Bin 0 -> 108398 bytes
site/src/main/.vuepress/public/logo.png | Bin 0 -> 21687 bytes
site/src/main/.vuepress/redirects | 3 -
site/src/main/.vuepress/sidebar/V0.13.x/en.ts | 215 +
site/src/main/.vuepress/sidebar/V0.13.x/zh.ts | 215 +
site/src/main/.vuepress/sidebar/V1.0.x/en.ts | 285 +
site/src/main/.vuepress/sidebar/V1.0.x/zh.ts | 286 +
site/src/main/.vuepress/sidebar/V1.1.x/en.ts | 305 +
site/src/main/.vuepress/sidebar/V1.1.x/zh.ts | 306 +
site/src/main/.vuepress/sidebar/en.ts | 313 +
site/src/main/.vuepress/sidebar/index.ts | 21 +
site/src/main/.vuepress/sidebar/zh.ts | 315 +
site/src/main/.vuepress/styles/config.scss | 25 +
site/src/main/.vuepress/styles/index.scss | 47 +
site/src/main/.vuepress/styles/index.styl | 23 -
site/src/main/.vuepress/styles/palette.scss | 23 +
site/src/main/.vuepress/styles/palette.styl | 23 -
site/src/main/.vuepress/theme.ts | 154 +
.../theme/components/AlgoliaSearchBox.vue | 189 -
.../.vuepress/theme/components/DropdownLink.vue | 196 -
.../theme/components/DropdownTransition.vue | 50 -
site/src/main/.vuepress/theme/components/Home.vue | 179 -
.../main/.vuepress/theme/components/NavLink.vue | 65 -
.../main/.vuepress/theme/components/NavLinks.vue | 166 -
.../src/main/.vuepress/theme/components/Navbar.vue | 139 -
site/src/main/.vuepress/theme/components/Page.vue | 338 -
.../main/.vuepress/theme/components/RightMenu.vue | 136 -
.../main/.vuepress/theme/components/Sidebar.vue | 76 -
.../.vuepress/theme/components/SidebarButton.vue | 44 -
.../.vuepress/theme/components/SidebarGroup.vue | 146 -
.../.vuepress/theme/components/SidebarLink.vue | 126 -
.../.vuepress/theme/components/SidebarLinks.vue | 103 -
.../.vuepress/theme/global-components/Badge.vue | 61 -
.../theme/global-components/Contributor.vue | 411 -
.../.vuepress/theme/global-components/IoTDB.vue | 293 -
.../.vuepress/theme/global-components/IoTDBZH.vue | 285 -
site/src/main/.vuepress/theme/index.js | 63 -
site/src/main/.vuepress/theme/layouts/404.vue | 43 -
site/src/main/.vuepress/theme/layouts/Layout.vue | 166 -
site/src/main/.vuepress/theme/styles/arrow.styl | 39 -
site/src/main/.vuepress/theme/styles/code.styl | 166 -
.../main/.vuepress/theme/styles/custom-blocks.styl | 47 -
site/src/main/.vuepress/theme/styles/index.styl | 229 -
site/src/main/.vuepress/theme/styles/mobile.styl | 54 -
site/src/main/.vuepress/theme/styles/palette.styl | 1 -
site/src/main/.vuepress/theme/styles/toc.styl | 20 -
site/src/main/.vuepress/theme/styles/wrapper.styl | 26 -
site/src/main/.vuepress/theme/util/index.js | 256 -
site/src/main/.vuepress/utils/define.ts | 25 +
site/src/main/.vuepress/utils/editLink.ts | 94 +
site/src/main/.vuepress/utils/index.ts | 22 +
site/src/main/.vuepress/utils/typings.ts | 36 +
site/src/main/deploy.js | 4 +-
site/src/main/package-lock.json | 12874 +++++++++++++++++++
site/src/main/package.json | 60 +-
site/src/main/pnpm-lock.yaml | 9892 ++++++++++++++
site/src/main/tsconfig.base.json | 56 +
site/src/main/tsconfig.json | 8 +
spark-iotdb-connector/pom.xml | 2 +-
spark-tsfile/pom.xml | 2 +-
subscription-api/pom.xml | 68 +
.../api/SubscriptionConfiguration.java | 114 +
.../subscription/api/SubscriptionFactory.java | 47 +
.../iotdb/subscription/api/consumer/Consumer.java | 56 +
.../api/consumer/pull/PullConsumer.java | 38 +
.../api/consumer/push/DataArrivalListener.java | 37 +
.../api/consumer/push/ExceptionListener.java | 34 +
.../api/consumer/push/PushConsumer.java | 68 +
.../api/dataset/SubscriptionDataSet.java | 22 +
.../api/exception/SubscriptionException.java | 31 +
.../SubscriptionStrategyNotValidException.java | 31 +
.../api/strategy/SubscriptionStrategy.java | 29 +
.../disorder/DisorderHandlingStrategy.java | 35 +
.../api/strategy/disorder/IntolerableStrategy.java | 32 +
.../api/strategy/disorder/WatermarkStrategy.java | 36 +
.../strategy/topic/MultipleConnectionStrategy.java | 63 +
.../api/strategy/topic/SingleTopicStrategy.java | 52 +
.../api/strategy/topic/TopicsStrategy.java | 24 +
testcontainer/pom.xml | 2 +-
thrift-commons/pom.xml | 2 +-
thrift-commons/src/main/thrift/common.thrift | 5 +-
thrift-confignode/pom.xml | 2 +-
.../src/main/thrift/confignode.thrift | 69 +-
thrift-influxdb/pom.xml | 2 +-
thrift-iot-consensus/pom.xml | 2 +-
thrift-mlnode/pom.xml | 2 +-
thrift/pom.xml | 2 +-
thrift/rpc-changelist.md | 10 +-
thrift/src/main/thrift/datanode.thrift | 24 +
tools/check_sigs.sh | 2 +-
tools/common.sh | 2 +-
tools/download_staged_release.sh | 2 +-
trigger-api/pom.xml | 2 +-
tsfile/pom.xml | 6 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 2 +-
.../apache/iotdb/tsfile/compress/ICompressor.java | 61 +-
.../iotdb/tsfile/compress/IUnCompressor.java | 44 +
.../file/metadata/enums/CompressionType.java | 6 +-
.../tsfile/write/record/datapoint/DataPoint.java | 2 +-
.../tsfile/write/schema/MeasurementSchema.java | 8 +
.../apache/iotdb/tsfile/compress/CompressTest.java | 39 +
.../org/apache/iotdb/tsfile/compress/ZstdTest.java | 88 +
udf-api/pom.xml | 2 +-
zeppelin-interpreter/pom.xml | 2 +-
1248 files changed, 72888 insertions(+), 38427 deletions(-)
[iotdb] 01/13: (tmp save)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1d63214b3c838ee9013746e1ef5404420c22de70
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Fri Mar 3 10:44:47 2023 +0800
(tmp save)
---
.../execution/operator/source/FileScanUtil.java | 61 ++++++++++++++++++++++
.../materializer/TsFileResourceMaterializer.java | 56 ++++++++++++++++++++
2 files changed, 117 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java
new file mode 100644
index 0000000000..bb0bd014d8
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.execution.operator.source;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.mpp.aggregation.Aggregator;
+import org.apache.iotdb.db.query.reader.materializer.TsFileResourceMaterializer;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
+public class FileScanUtil {
+
+ private final Map<PartialPath, List<Aggregator>> pathToAggregatorsMap;
+
+ private final TsFileResourceMaterializer fileResourceMaterializer;
+
+ private final TreeSet<ChunkMetadata> chunkMetadataList =
+ new TreeSet<>(
+ Comparator.comparingLong(ChunkMetadata::getVersion)
+ .thenComparingLong(ChunkMetadata::getOffsetOfChunkHeader));
+
+ public FileScanUtil(
+ Map<PartialPath, List<Aggregator>> pathToAggregatorsMap, QueryDataSource dataSource) {
+ this.pathToAggregatorsMap = pathToAggregatorsMap;
+ this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
+ }
+
+ public boolean hasNextFile() {
+ return fileResourceMaterializer.hasNext();
+ }
+
+ public void consume() {
+ TsFileResource nextFile = fileResourceMaterializer.next();
+
+ List<ITimeSeriesMetadata> timeSeriesMetadata;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java b/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java
new file mode 100644
index 0000000000..e5e1c0bc77
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/materializer/TsFileResourceMaterializer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.materializer;
+
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+
+public class TsFileResourceMaterializer {
+
+ private final QueryDataSource tsFileResourceSource;
+
+ // file index
+ private int curSeqFileIndex;
+ private int curUnSeqFileIndex;
+
+ public TsFileResourceMaterializer(QueryDataSource tsFileResourceSource) {
+ this.tsFileResourceSource = tsFileResourceSource;
+ curSeqFileIndex = curUnSeqFileIndex = 0;
+ }
+
+ public boolean hasNext() {
+ return tsFileResourceSource.hasNextSeqResource(curSeqFileIndex, true)
+ || tsFileResourceSource.hasNextUnseqResource(curUnSeqFileIndex);
+ }
+
+ public TsFileResource next() {
+ if (tsFileResourceSource.hasNextSeqResource(curSeqFileIndex, true)) {
+ TsFileResource nextFile = tsFileResourceSource.getSeqResourceByIndex(curUnSeqFileIndex);
+ curSeqFileIndex++;
+ return nextFile;
+ } else if (tsFileResourceSource.hasNextUnseqResource(curUnSeqFileIndex)) {
+ TsFileResource nextFile = tsFileResourceSource.getUnseqResourceByIndex(curUnSeqFileIndex);
+ curUnSeqFileIndex++;
+ return nextFile;
+ } else {
+ throw new IllegalStateException("");
+ }
+ }
+}
[iotdb] 06/13: finish FE
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4f70135562b83530bc677b3db52ed1a77a3263d4
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Mar 20 16:29:06 2023 +0800
finish FE
---
.../commons/partition/DataPartitionQueryParam.java | 6 -
.../source/FileAggregationScanOperator.java | 107 ++++++++++++++
.../operator/source/FileAggregationScanUtil.java | 58 ++++++--
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 10 ++
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 38 +++++
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 25 ++++
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 9 ++
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 26 ++++
.../plan/planner/distribution/SourceRewriter.java | 24 ++++
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/source/FileAggregationScanNode.java | 158 +++++++++++++++++++++
.../common/block/column/BinaryColumnBuilder.java | 52 +++++++
13 files changed, 509 insertions(+), 15 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java
index 4ed3cd8dc4..6925fb2191 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartitionQueryParam.java
@@ -36,12 +36,6 @@ public class DataPartitionQueryParam {
// (timePartitionSlotList.get(timePartitionSlotList.size() - 1), +oo)
private boolean needRightAll = false;
- public DataPartitionQueryParam(
- String devicePath, List<TTimePartitionSlot> timePartitionSlotList) {
- this.devicePath = devicePath;
- this.timePartitionSlotList = timePartitionSlotList;
- }
-
public DataPartitionQueryParam(
String devicePath,
List<TTimePartitionSlot> timePartitionSlotList,
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
new file mode 100644
index 0000000000..083332ad9c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.execution.operator.source;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.mpp.execution.operator.process.last.LastQueryUtil;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+
+import java.io.IOException;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileAggregationScanOperator extends AbstractSourceOperator
+ implements DataSourceOperator, SourceOperator {
+
+ private final FileAggregationScanUtil aggregationScanUtil;
+
+ private final TsBlockBuilder tsBlockBuilder;
+
+ public FileAggregationScanOperator(
+ OperatorContext context,
+ PlanNodeId sourceId,
+ PartialPath pathPattern,
+ AggregationDescriptor aggregationDescriptor,
+ int[] levels) {
+ this.sourceId = sourceId;
+ this.operatorContext = context;
+ this.aggregationScanUtil =
+ new FileAggregationScanUtil(
+ pathPattern, aggregationDescriptor, levels, new SeriesScanOptions.Builder().build());
+ this.tsBlockBuilder = LastQueryUtil.createTsBlockBuilder();
+ }
+
+ @Override
+ public void initQueryDataSource(QueryDataSource dataSource) {
+ aggregationScanUtil.initQueryDataSource(dataSource);
+ }
+
+ @Override
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ @Override
+ public PlanNodeId getSourceId() {
+ return sourceId;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return aggregationScanUtil.hasNextFile();
+ }
+
+ @Override
+ public TsBlock next() {
+ tsBlockBuilder.reset();
+ try {
+ aggregationScanUtil.consume();
+ return aggregationScanUtil.getAggregationResult(tsBlockBuilder);
+ } catch (IOException e) {
+ throw new RuntimeException("Error happened while scanning the file", e);
+ }
+ }
+
+ @Override
+ public boolean isFinished() {
+ return !hasNextWithTimer();
+ }
+
+ @Override
+ public long calculateMaxPeekMemory() {
+ return DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ @Override
+ public long calculateMaxReturnSize() {
+ return DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ @Override
+ public long calculateRetainedSizeAfterCallingNext() {
+ return 0;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index ff12114388..5a4ef5cb38 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.mpp.aggregation.AccumulatorFactory;
import org.apache.iotdb.db.mpp.aggregation.Aggregator;
import org.apache.iotdb.db.mpp.plan.analyze.GroupByLevelController;
-import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.SeriesScanOptions;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.materializer.TsFileResourceMaterializer;
@@ -37,8 +37,11 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.IPageReader;
+import org.apache.iotdb.tsfile.utils.Binary;
import java.io.IOException;
import java.util.ArrayList;
@@ -49,9 +52,13 @@ import java.util.Map;
public class FileAggregationScanUtil {
+ private final PartialPath pathPattern;
+
+ private final AggregationDescriptor aggregationDescriptor;
+
private final Map<PartialPath, Aggregator> pathToAggregatorMap;
- private final TsFileResourceMaterializer fileResourceMaterializer;
+ private TsFileResourceMaterializer fileResourceMaterializer;
private final Map<PartialPath, List<IChunkMetadata>> chunkMetadataMap;
@@ -62,23 +69,30 @@ public class FileAggregationScanUtil {
private final SeriesScanOptions scanOptions;
public FileAggregationScanUtil(
- Map<PartialPath, Aggregator> pathToAggregatorMap,
- QueryDataSource dataSource,
+ PartialPath pathPattern,
+ AggregationDescriptor aggregationDescriptor,
int[] levels,
SeriesScanOptions scanOptions) {
- this.pathToAggregatorMap = pathToAggregatorMap;
- this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
+ this.pathPattern = pathPattern;
+ this.aggregationDescriptor = aggregationDescriptor;
+ this.pathToAggregatorMap = new HashMap<>();
this.chunkMetadataMap = new HashMap<>();
this.partialPathPool = new PartialPathPool();
this.levels = levels;
this.scanOptions = scanOptions;
}
+ public void initQueryDataSource(QueryDataSource dataSource) {
+ this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
+ }
+
public boolean hasNextFile() {
return fileResourceMaterializer.hasNext();
}
public void consume() throws IOException {
+ pathToAggregatorMap.clear();
+
TsFileResource nextFile = fileResourceMaterializer.next();
TsFileSequenceReader reader =
FileReaderManager.getInstance().get(nextFile.getTsFilePath(), nextFile.isClosed());
@@ -99,6 +113,28 @@ public class FileAggregationScanUtil {
}
}
+ public TsBlock getAggregationResult(TsBlockBuilder builder) {
+ for (Map.Entry<PartialPath, Aggregator> entry : pathToAggregatorMap.entrySet()) {
+ builder.getTimeColumnBuilder().writeLong(0L);
+ builder.getValueColumnBuilders()[0].writeBinary(
+ Binary.valueOf(
+ String.format(
+ "%s(%s)",
+ aggregationDescriptor.getAggregationFuncName(), entry.getKey().toString())));
+
+ Aggregator aggregator = entry.getValue();
+ ColumnBuilder[] columnBuilders = new ColumnBuilder[1];
+ columnBuilders[0] = builder.getValueColumnBuilders()[1];
+ aggregator.outputResult(columnBuilders);
+
+ builder.getValueColumnBuilders()[2].writeBinary(
+ Binary.valueOf(aggregator.getOutputType()[0].toString()));
+
+ builder.declarePosition();
+ }
+ return builder.build();
+ }
+
private void unpackChunkMetadata(PartialPath devicePath, IChunkMetadata chunkMetadata)
throws IOException {
PartialPath groupedPath =
@@ -123,6 +159,11 @@ public class FileAggregationScanUtil {
private void consumeTimeseriesMetadata(
PartialPath devicePath, TimeseriesMetadata timeseriesMetadata) {
+ PartialPath fullPath = devicePath.concatNode(timeseriesMetadata.getMeasurementId());
+ if (!pathPattern.matchFullPath(fullPath)) {
+ return;
+ }
+
PartialPath groupedPath =
partialPathPool.getGroupedPath(devicePath, timeseriesMetadata.getMeasurementId());
if (pathToAggregatorMap.containsKey(groupedPath)) {
@@ -130,12 +171,13 @@ public class FileAggregationScanUtil {
groupedPath,
new Aggregator(
AccumulatorFactory.createAccumulator(
- TAggregationType.COUNT,
+ TAggregationType.valueOf(
+ aggregationDescriptor.getAggregationFuncName().toUpperCase()),
timeseriesMetadata.getTSDataType(),
Collections.emptyList(),
Collections.emptyMap(),
true),
- AggregationStep.SINGLE));
+ aggregationDescriptor.getStep()));
}
Filter queryFilter = scanOptions.getQueryFilter();
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
index bf15b2a826..be471ba3b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java
@@ -83,6 +83,8 @@ public class Analysis {
// Query Analysis (used in ALIGN BY TIME)
/////////////////////////////////////////////////////////////////////////////////////////////////
+ private int[] levels;
+
// map from device name to series/aggregation under this device
private Set<Expression> sourceExpressions;
@@ -586,4 +588,12 @@ public class Analysis {
public Map<NodeRef<Expression>, TSDataType> getExpressionTypes() {
return expressionTypes;
}
+
+ public void setLevels(int[] levels) {
+ this.levels = levels;
+ }
+
+ public int[] getLevels() {
+ return levels;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index a6bc60a90e..f922ce89dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -229,6 +229,44 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
// check for semantic errors
queryStatement.semanticCheck();
+ if (queryStatement.isGroupByLevel()) {
+ analysis.setLevels(queryStatement.getGroupByLevelComponent().getLevels());
+
+ PartialPath pathPrefix = queryStatement.getFromComponent().getPrefixPaths().get(0);
+ FunctionExpression aggregationMeasurementExpression =
+ (FunctionExpression)
+ queryStatement.getSelectComponent().getResultColumns().get(0).getExpression();
+ TimeSeriesOperand sourceMeasurementExpression =
+ (TimeSeriesOperand) aggregationMeasurementExpression.getExpressions().get(0);
+
+ Set<Expression> sourceExpressions = new HashSet<>();
+ TimeSeriesOperand sourceExpression =
+ new TimeSeriesOperand(pathPrefix.concatPath(sourceMeasurementExpression.getPath()));
+ sourceExpressions.add(sourceExpression);
+ analysis.setSourceExpressions(sourceExpressions);
+
+ Set<Expression> aggregationExpressions = new HashSet<>();
+ FunctionExpression aggregationExpression =
+ new FunctionExpression(
+ aggregationMeasurementExpression.getFunctionName(),
+ aggregationMeasurementExpression.getFunctionAttributes(),
+ Collections.singletonList(sourceExpression));
+ aggregationExpressions.add(aggregationExpression);
+ analysis.setAggregationExpressions(aggregationExpressions);
+
+ analysis.setRespDatasetHeader(
+ new DatasetHeader(ColumnHeaderConstant.lastQueryColumnHeaders, true));
+
+ Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>();
+ sgNameToQueryParamsMap.put(
+ "root.iov",
+ Collections.singletonList(
+ new DataPartitionQueryParam("root.iov.**", Collections.emptyList(), true, true)));
+ analysis.setDataPartitionInfo(partitionFetcher.getDataPartition(sgNameToQueryParamsMap));
+
+ return analysis;
+ }
+
// concat path and construct path pattern tree
PathPatternTree patternTree = new PathPatternTree();
queryStatement =
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index f2ba699208..70b470ee42 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -73,6 +73,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.last.LastQueryNode
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationSourceNode;
@@ -1192,4 +1193,28 @@ public class LogicalPlanBuilder {
this.root = new ShowQueriesNode(context.getQueryId().genPlanNodeId(), dataNodeLocation);
return this;
}
+
+ public LogicalPlanBuilder planFileAggregation(
+ Set<Expression> sourceExpressions, Set<Expression> aggregationExpressions, int[] levels) {
+ PartialPath pathPattern =
+ ((TimeSeriesOperand) (new ArrayList<>(sourceExpressions).get(0))).getPath();
+ FunctionExpression functionExpression =
+ (FunctionExpression) (new ArrayList<>(aggregationExpressions).get(0));
+ AggregationDescriptor aggregationDescriptor =
+ new AggregationDescriptor(
+ functionExpression.getFunctionName(),
+ AggregationStep.SINGLE,
+ Collections.emptyList(),
+ Collections.emptyMap());
+ this.root =
+ new FileAggregationScanNode(
+ context.getQueryId().genPlanNodeId(), pathPattern, aggregationDescriptor, levels);
+
+ ColumnHeaderConstant.lastQueryColumnHeaders.forEach(
+ columnHeader ->
+ context
+ .getTypeProvider()
+ .setType(columnHeader.getColumnName(), columnHeader.getColumnType()));
+ return this;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
index 5e503cb6c6..5c0749d634 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanVisitor.java
@@ -109,6 +109,15 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
public PlanNode visitQuery(QueryStatement queryStatement, MPPQueryContext context) {
LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context);
+ if (queryStatement.isGroupByLevel()) {
+ return planBuilder
+ .planFileAggregation(
+ analysis.getSourceExpressions(),
+ analysis.getAggregationExpressions(),
+ analysis.getLevels())
+ .getRoot();
+ }
+
if (queryStatement.isLastQuery()) {
return planBuilder
.planLast(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
index 0e556f45bb..1d3851cb1f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java
@@ -120,6 +120,7 @@ import org.apache.iotdb.db.mpp.execution.operator.sink.ShuffleHelperOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.AlignedSeriesAggregationScanOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.AlignedSeriesScanOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.ExchangeOperator;
+import org.apache.iotdb.db.mpp.execution.operator.source.FileAggregationScanOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.SeriesAggregationScanOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.SeriesScanOperator;
import org.apache.iotdb.db.mpp.execution.operator.source.ShowQueriesOperator;
@@ -182,6 +183,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
@@ -487,6 +489,30 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
return seriesAggregationScanOperator;
}
+ @Override
+ public Operator visitFileAggregationScan(
+ FileAggregationScanNode node, LocalExecutionPlanContext context) {
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ FileAggregationScanOperator.class.getSimpleName());
+ FileAggregationScanOperator fileAggregationScanOperator =
+ new FileAggregationScanOperator(
+ operatorContext,
+ node.getPlanNodeId(),
+ node.getPathPattern(),
+ node.getAggregationDescriptor(),
+ node.getLevels());
+
+ ((DataDriverContext) context.getDriverContext()).addSourceOperator(fileAggregationScanOperator);
+ context.getDriverContext().setInputDriver(true);
+ context.getTimeSliceAllocator().recordExecutionWeight(operatorContext, 1);
+ return fileAggregationScanOperator;
+ }
+
@Override
public Operator visitSchemaQueryOrderByHeat(
SchemaQueryOrderByHeatNode node, LocalExecutionPlanContext context) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
index 22efafdb10..f61891e59c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
@@ -51,6 +51,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.last.LastQueryNode
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationSourceNode;
@@ -444,6 +445,29 @@ public class SourceRewriter extends SimplePlanNodeRewriter<DistributionPlanConte
return processSeriesAggregationSource(node, context);
}
+ @Override
+ public List<PlanNode> visitFileAggregationScan(
+ FileAggregationScanNode node, DistributionPlanContext context) {
+ List<TRegionReplicaSet> dataDistribution =
+ analysis.getPartitionInfo(node.getPartitionPath(), node.getPartitionTimeFilter());
+ if (dataDistribution.size() == 1) {
+ node.setRegionReplicaSet(dataDistribution.get(0));
+ return Collections.singletonList(node);
+ }
+
+ LastQueryCollectNode lastQueryCollectNode =
+ new LastQueryCollectNode(context.queryContext.getQueryId().genPlanNodeId());
+
+ for (TRegionReplicaSet dataRegion : dataDistribution) {
+ FileAggregationScanNode split = (FileAggregationScanNode) node.clone();
+ split.setAggregationDescriptor(node.getAggregationDescriptor());
+ split.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId());
+ split.setRegionReplicaSet(dataRegion);
+ lastQueryCollectNode.addChild(split);
+ }
+ return Collections.singletonList(lastQueryCollectNode);
+ }
+
private List<PlanNode> processSeriesAggregationSource(
SeriesAggregationSourceNode node, DistributionPlanContext context) {
List<TRegionReplicaSet> dataDistribution =
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
index 9b75d9e4a1..20661f7571 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanNodeType.java
@@ -77,6 +77,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
@@ -165,7 +166,8 @@ public enum PlanNodeType {
INTERNAL_BATCH_ACTIVATE_TEMPLATE((short) 68),
INTERNAL_CREATE_MULTI_TIMESERIES((short) 69),
IDENTITY_SINK((short) 70),
- SHUFFLE_SINK((short) 71);
+ SHUFFLE_SINK((short) 71),
+ FILE_AGGREGATION((short) 72);
public static final int BYTES = Short.BYTES;
@@ -356,6 +358,8 @@ public enum PlanNodeType {
return IdentitySinkNode.deserialize(buffer);
case 71:
return ShuffleSinkNode.deserialize(buffer);
+ case 72:
+ return FileAggregationScanNode.deserialize(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
index cf3fbe02af..245b79f955 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/PlanVisitor.java
@@ -77,6 +77,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.AlignedSeriesScanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.FileAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesAggregationScanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.source.SeriesScanNode;
@@ -125,6 +126,10 @@ public abstract class PlanVisitor<R, C> {
return visitSourceNode(node, context);
}
+ public R visitFileAggregationScan(FileAggregationScanNode node, C context) {
+ return visitSourceNode(node, context);
+ }
+
public R visitDeviceView(DeviceViewNode node, C context) {
return visitMultiChildProcess(node, context);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
new file mode 100644
index 0000000000..63158d132a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.planner.plan.node.source;
+
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.collect.ImmutableList;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import static org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode.LAST_QUERY_HEADER_COLUMNS;
+
+public class FileAggregationScanNode extends SeriesSourceNode {
+
+ private final PartialPath pathPattern;
+
+ private AggregationDescriptor aggregationDescriptor;
+
+ private final int[] levels;
+
+ public FileAggregationScanNode(
+ PlanNodeId id,
+ PartialPath pathPattern,
+ AggregationDescriptor aggregationDescriptor,
+ int[] levels) {
+ super(id);
+ this.pathPattern = pathPattern;
+ this.aggregationDescriptor = aggregationDescriptor;
+ this.levels = levels;
+ }
+
+ public PartialPath getPathPattern() {
+ return pathPattern;
+ }
+
+ public AggregationDescriptor getAggregationDescriptor() {
+ return aggregationDescriptor;
+ }
+
+ public void setAggregationDescriptor(AggregationDescriptor aggregationDescriptor) {
+ this.aggregationDescriptor = aggregationDescriptor;
+ }
+
+ public int[] getLevels() {
+ return levels;
+ }
+
+ @Override
+ public PartialPath getPartitionPath() {
+ return null;
+ }
+
+ @Override
+ public Filter getPartitionTimeFilter() {
+ return null;
+ }
+
+ @Override
+ public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) {}
+
+ @Override
+ public TRegionReplicaSet getRegionReplicaSet() {
+ return null;
+ }
+
+ @Override
+ public void open() throws Exception {}
+
+ @Override
+ public void close() throws Exception {}
+
+ @Override
+ public List<PlanNode> getChildren() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public int allowedChildCount() {
+ return NO_CHILD_ALLOWED;
+ }
+
+ @Override
+ public void addChild(PlanNode child) {
+ throw new UnsupportedOperationException("no child is allowed for FileAggregationScanNode");
+ }
+
+ @Override
+ public PlanNode clone() {
+ return new FileAggregationScanNode(
+ getPlanNodeId(), getPathPattern(), getAggregationDescriptor(), getLevels());
+ }
+
+ @Override
+ public List<String> getOutputColumnNames() {
+ return LAST_QUERY_HEADER_COLUMNS;
+ }
+
+ @Override
+ protected void serializeAttributes(ByteBuffer buffer) {
+ PlanNodeType.FILE_AGGREGATION.serialize(buffer);
+ pathPattern.serialize(buffer);
+ aggregationDescriptor.serialize(buffer);
+ ReadWriteIOUtils.write(levels.length, buffer);
+ for (int level : levels) {
+ ReadWriteIOUtils.write(level, buffer);
+ }
+ }
+
+ @Override
+ protected void serializeAttributes(DataOutputStream stream) throws IOException {
+ PlanNodeType.FILE_AGGREGATION.serialize(stream);
+ pathPattern.serialize(stream);
+ aggregationDescriptor.serialize(stream);
+ ReadWriteIOUtils.write(levels.length, stream);
+ for (int level : levels) {
+ ReadWriteIOUtils.write(level, stream);
+ }
+ }
+
+ public static PlanNode deserialize(ByteBuffer buffer) {
+ PartialPath pathPattern = PartialPath.deserialize(buffer);
+ AggregationDescriptor aggregationDescriptor = AggregationDescriptor.deserialize(buffer);
+ int levelsSize = ReadWriteIOUtils.readInt(buffer);
+ int[] levels = new int[levelsSize];
+ for (int i = 0; i < levelsSize; i++) {
+ levels[i] = ReadWriteIOUtils.readInt(buffer);
+ }
+ PlanNodeId planNodeId = PlanNodeId.deserialize(buffer);
+ return new FileAggregationScanNode(planNodeId, pathPattern, aggregationDescriptor, levels);
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
index ad1e05ac61..40bb95a7cd 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column/BinaryColumnBuilder.java
@@ -151,4 +151,56 @@ public class BinaryColumnBuilder implements ColumnBuilder {
private void updateArraysDataSize() {
arraysRetainedSizeInBytes = sizeOf(valueIsNull) + sizeOf(values);
}
+
+ @Override
+ public ColumnBuilder writeInt(int value) {
+ if (values.length <= positionCount) {
+ growCapacity();
+ }
+
+ values[positionCount] = Binary.valueOf(String.valueOf(value));
+
+ hasNonNullValue = true;
+ positionCount++;
+ return this;
+ }
+
+ @Override
+ public ColumnBuilder writeLong(long value) {
+ if (values.length <= positionCount) {
+ growCapacity();
+ }
+
+ values[positionCount] = Binary.valueOf(String.valueOf(value));
+
+ hasNonNullValue = true;
+ positionCount++;
+ return this;
+ }
+
+ @Override
+ public ColumnBuilder writeFloat(float value) {
+ if (values.length <= positionCount) {
+ growCapacity();
+ }
+
+ values[positionCount] = Binary.valueOf(String.valueOf(value));
+
+ hasNonNullValue = true;
+ positionCount++;
+ return this;
+ }
+
+ @Override
+ public ColumnBuilder writeDouble(double value) {
+ if (values.length <= positionCount) {
+ growCapacity();
+ }
+
+ values[positionCount] = Binary.valueOf(String.valueOf(value));
+
+ hasNonNullValue = true;
+ positionCount++;
+ return this;
+ }
}
[iotdb] 09/13: merge master
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cfbe60c9204517c725878a30ead4c344c076eaed
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 09:58:49 2023 +0800
merge master
---
.../mpp/execution/operator/source/FileAggregationScanOperator.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
index 083332ad9c..5250031f42 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
@@ -70,12 +70,12 @@ public class FileAggregationScanOperator extends AbstractSourceOperator
}
@Override
- public boolean hasNext() {
+ public boolean hasNext() throws Exception {
return aggregationScanUtil.hasNextFile();
}
@Override
- public TsBlock next() {
+ public TsBlock next() throws Exception {
tsBlockBuilder.reset();
try {
aggregationScanUtil.consume();
@@ -86,7 +86,7 @@ public class FileAggregationScanOperator extends AbstractSourceOperator
}
@Override
- public boolean isFinished() {
+ public boolean isFinished() throws Exception {
return !hasNextWithTimer();
}
[iotdb] 11/13: fix deserialize bug
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5787d0f85813ceb907d64fb8423b4cdb54228bfc
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 10:51:34 2023 +0800
fix deserialize bug
---
.../db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
index 9c1eeaa20c..5c997b9db7 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.mpp.plan.planner.plan.node.source;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
@@ -165,7 +166,7 @@ public class FileAggregationScanNode extends SeriesSourceNode {
}
public static PlanNode deserialize(ByteBuffer buffer) {
- PartialPath pathPattern = PartialPath.deserialize(buffer);
+ PartialPath pathPattern = (PartialPath) PathDeserializeUtil.deserialize(buffer);
AggregationDescriptor aggregationDescriptor = AggregationDescriptor.deserialize(buffer);
int levelsSize = ReadWriteIOUtils.readInt(buffer);
int[] levels = new int[levelsSize];
[iotdb] 02/13: rename
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fdc10a8ffbbc65cb9340c708fa1614efdb0f0a09
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Thu Mar 16 14:30:20 2023 +0800
rename
---
.../source/{FileScanUtil.java => FileAggregationScanUtil.java} | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
similarity index 96%
rename from server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java
rename to server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index bb0bd014d8..be136d531c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -32,7 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeSet;
-public class FileScanUtil {
+public class FileAggregationScanUtil {
private final Map<PartialPath, List<Aggregator>> pathToAggregatorsMap;
@@ -43,7 +43,7 @@ public class FileScanUtil {
Comparator.comparingLong(ChunkMetadata::getVersion)
.thenComparingLong(ChunkMetadata::getOffsetOfChunkHeader));
- public FileScanUtil(
+ public FileAggregationScanUtil(
Map<PartialPath, List<Aggregator>> pathToAggregatorsMap, QueryDataSource dataSource) {
this.pathToAggregatorsMap = pathToAggregatorsMap;
this.fileResourceMaterializer = new TsFileResourceMaterializer(dataSource);
[iotdb] 12/13: return tsblock only once
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9630dee026c111c13200ed74dbf48dbdd01e5565
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 14:55:12 2023 +0800
return tsblock only once
---
.../mpp/execution/operator/source/FileAggregationScanOperator.java | 7 +++++--
.../db/mpp/execution/operator/source/FileAggregationScanUtil.java | 2 --
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
index 5250031f42..0f61f0656b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
@@ -76,10 +76,13 @@ public class FileAggregationScanOperator extends AbstractSourceOperator
@Override
public TsBlock next() throws Exception {
- tsBlockBuilder.reset();
try {
aggregationScanUtil.consume();
- return aggregationScanUtil.getAggregationResult(tsBlockBuilder);
+ if (aggregationScanUtil.hasNextFile()) {
+ return null;
+ } else {
+ return aggregationScanUtil.getAggregationResult(tsBlockBuilder);
+ }
} catch (IOException e) {
throw new RuntimeException("Error happened while scanning the file", e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index e51fee215e..9267d68cb6 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -99,8 +99,6 @@ public class FileAggregationScanUtil {
}
public void consume() throws IOException {
- pathToAggregatorMap.clear();
-
TsFileResource nextFile = fileResourceMaterializer.next();
TsFileSequenceReader reader =
FileReaderManager.getInstance().get(nextFile.getTsFilePath(), nextFile.isClosed());
[iotdb] 13/13: use agg instead of last
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8cb7027ed056f37b09d1077a24e14c5732b203c0
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 16:45:02 2023 +0800
use agg instead of last
---
.../source/FileAggregationScanOperator.java | 5 +++--
.../operator/source/FileAggregationScanUtil.java | 11 +--------
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 16 ++++++++-----
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 15 ++++++-------
.../plan/planner/distribution/SourceRewriter.java | 19 ++++++++++++----
.../plan/node/source/FileAggregationScanNode.java | 26 +++++++++++++++++-----
6 files changed, 57 insertions(+), 35 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
index 0f61f0656b..23ed2cc12c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanOperator.java
@@ -22,14 +22,15 @@ package org.apache.iotdb.db.mpp.execution.operator.source;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
-import org.apache.iotdb.db.mpp.execution.operator.process.last.LastQueryUtil;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationDescriptor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import java.io.IOException;
+import java.util.Collections;
import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
@@ -51,7 +52,7 @@ public class FileAggregationScanOperator extends AbstractSourceOperator
this.aggregationScanUtil =
new FileAggregationScanUtil(
pathPattern, aggregationDescriptor, levels, new SeriesScanOptions.Builder().build());
- this.tsBlockBuilder = LastQueryUtil.createTsBlockBuilder();
+ this.tsBlockBuilder = new TsBlockBuilder(Collections.singletonList(TSDataType.INT64));
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index 9267d68cb6..4b91a9f6ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -41,7 +41,6 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.IPageReader;
-import org.apache.iotdb.tsfile.utils.Binary;
import java.io.IOException;
import java.util.ArrayList;
@@ -122,20 +121,12 @@ public class FileAggregationScanUtil {
public TsBlock getAggregationResult(TsBlockBuilder builder) {
for (Map.Entry<PartialPath, Aggregator> entry : pathToAggregatorMap.entrySet()) {
builder.getTimeColumnBuilder().writeLong(0L);
- builder.getValueColumnBuilders()[0].writeBinary(
- Binary.valueOf(
- String.format(
- "%s(%s)",
- aggregationDescriptor.getAggregationFuncName(), entry.getKey().toString())));
Aggregator aggregator = entry.getValue();
ColumnBuilder[] columnBuilders = new ColumnBuilder[1];
- columnBuilders[0] = builder.getValueColumnBuilders()[1];
+ columnBuilders[0] = builder.getValueColumnBuilders()[0];
aggregator.outputResult(columnBuilders);
- builder.getValueColumnBuilders()[2].writeBinary(
- Binary.valueOf(aggregator.getOutputType()[0].toString()));
-
builder.declarePosition();
}
return builder.build();
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index 5c03ff0763..35a6801a33 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -250,14 +250,20 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Set<Expression> aggregationExpressions = new HashSet<>();
FunctionExpression aggregationExpression =
new FunctionExpression(
- aggregationMeasurementExpression.getFunctionName(),
- aggregationMeasurementExpression.getFunctionAttributes(),
- Collections.singletonList(sourceExpression));
+ "count",
+ new LinkedHashMap<>(),
+ Collections.singletonList(
+ new TimeSeriesOperand(
+ new MeasurementPath("root.*.*.*.*.*.*", TSDataType.INT64))));
+ analyzeExpression(analysis, aggregationExpression);
aggregationExpressions.add(aggregationExpression);
analysis.setAggregationExpressions(aggregationExpressions);
analysis.setRespDatasetHeader(
- new DatasetHeader(ColumnHeaderConstant.lastQueryColumnHeaders, true));
+ new DatasetHeader(
+ Collections.singletonList(
+ new ColumnHeader(aggregationExpression.toString(), TSDataType.INT64)),
+ true));
Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>();
sgNameToQueryParamsMap.put("root.iov", Collections.emptyList());
@@ -381,7 +387,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
// fetch partition information
analyzeDataPartition(analysis, queryStatement, schemaTree);
- } catch (StatementAnalyzeException e) {
+ } catch (StatementAnalyzeException | IllegalPathException e) {
logger.warn("Meet error when analyzing the query statement: ", e);
throw new StatementAnalyzeException(
"Meet error when analyzing the query statement: " + e.getMessage());
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
index 70b470ee42..7a14f750fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java
@@ -1204,17 +1204,16 @@ public class LogicalPlanBuilder {
new AggregationDescriptor(
functionExpression.getFunctionName(),
AggregationStep.SINGLE,
- Collections.emptyList(),
+ Collections.singletonList(functionExpression.getExpressions().get(0)),
Collections.emptyMap());
+ updateTypeProvider(Collections.singletonList(functionExpression));
this.root =
new FileAggregationScanNode(
- context.getQueryId().genPlanNodeId(), pathPattern, aggregationDescriptor, levels);
-
- ColumnHeaderConstant.lastQueryColumnHeaders.forEach(
- columnHeader ->
- context
- .getTypeProvider()
- .setType(columnHeader.getColumnName(), columnHeader.getColumnType()));
+ context.getQueryId().genPlanNodeId(),
+ pathPattern,
+ aggregationDescriptor,
+ levels,
+ functionExpression);
return this;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
index c9cfeb1102..376284d2a7 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/SourceRewriter.java
@@ -454,17 +454,28 @@ public class SourceRewriter extends SimplePlanNodeRewriter<DistributionPlanConte
return Collections.singletonList(node);
}
- LastQueryCollectNode lastQueryCollectNode =
- new LastQueryCollectNode(context.queryContext.getQueryId().genPlanNodeId());
+ CrossSeriesAggregationDescriptor rootAggregationDescriptor =
+ new CrossSeriesAggregationDescriptor(
+ "count",
+ AggregationStep.FINAL,
+ Collections.singletonList(node.getOutputExpression().getExpressions().get(0)),
+ Collections.emptyMap(),
+ node.getOutputExpression().getExpressions().get(0));
+ AggregationNode aggregationNode =
+ new AggregationNode(
+ context.queryContext.getQueryId().genPlanNodeId(),
+ Collections.singletonList(rootAggregationDescriptor),
+ null,
+ Ordering.ASC);
for (TRegionReplicaSet dataRegion : dataDistribution) {
FileAggregationScanNode split = (FileAggregationScanNode) node.clone();
split.setAggregationDescriptor(node.getAggregationDescriptor());
split.setPlanNodeId(context.queryContext.getQueryId().genPlanNodeId());
split.setRegionReplicaSet(dataRegion);
- lastQueryCollectNode.addChild(split);
+ aggregationNode.addChild(split);
}
- return Collections.singletonList(lastQueryCollectNode);
+ return Collections.singletonList(aggregationNode);
}
private List<PlanNode> processSeriesAggregationSource(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
index 5c997b9db7..ae4b2f6b2c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/FileAggregationScanNode.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.mpp.plan.planner.plan.node.source;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
+import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeType;
@@ -35,10 +36,9 @@ import com.google.common.collect.ImmutableList;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.List;
-import static org.apache.iotdb.db.mpp.plan.planner.plan.node.source.LastQueryScanNode.LAST_QUERY_HEADER_COLUMNS;
-
public class FileAggregationScanNode extends SeriesSourceNode {
private final PartialPath pathPattern;
@@ -50,15 +50,19 @@ public class FileAggregationScanNode extends SeriesSourceNode {
// The id of DataRegion where the node will run
private TRegionReplicaSet regionReplicaSet;
+ private final Expression outputExpression;
+
public FileAggregationScanNode(
PlanNodeId id,
PartialPath pathPattern,
AggregationDescriptor aggregationDescriptor,
- int[] levels) {
+ int[] levels,
+ Expression outputExpression) {
super(id);
this.pathPattern = pathPattern;
this.aggregationDescriptor = aggregationDescriptor;
this.levels = levels;
+ this.outputExpression = outputExpression;
}
public FileAggregationScanNode(
@@ -66,8 +70,9 @@ public class FileAggregationScanNode extends SeriesSourceNode {
PartialPath pathPattern,
AggregationDescriptor aggregationDescriptor,
int[] levels,
+ Expression outputExpression,
TRegionReplicaSet regionReplicaSet) {
- this(id, pathPattern, aggregationDescriptor, levels);
+ this(id, pathPattern, aggregationDescriptor, levels, outputExpression);
this.regionReplicaSet = regionReplicaSet;
}
@@ -87,6 +92,10 @@ public class FileAggregationScanNode extends SeriesSourceNode {
return levels;
}
+ public Expression getOutputExpression() {
+ return outputExpression;
+ }
+
@Override
public PartialPath getPartitionPath() {
return null;
@@ -135,12 +144,13 @@ public class FileAggregationScanNode extends SeriesSourceNode {
getPathPattern(),
getAggregationDescriptor(),
getLevels(),
+ getOutputExpression(),
getRegionReplicaSet());
}
@Override
public List<String> getOutputColumnNames() {
- return LAST_QUERY_HEADER_COLUMNS;
+ return Collections.singletonList(outputExpression.toString());
}
@Override
@@ -152,6 +162,7 @@ public class FileAggregationScanNode extends SeriesSourceNode {
for (int level : levels) {
ReadWriteIOUtils.write(level, buffer);
}
+ Expression.serialize(outputExpression, buffer);
}
@Override
@@ -163,6 +174,7 @@ public class FileAggregationScanNode extends SeriesSourceNode {
for (int level : levels) {
ReadWriteIOUtils.write(level, stream);
}
+ Expression.serialize(outputExpression, stream);
}
public static PlanNode deserialize(ByteBuffer buffer) {
@@ -173,8 +185,10 @@ public class FileAggregationScanNode extends SeriesSourceNode {
for (int i = 0; i < levelsSize; i++) {
levels[i] = ReadWriteIOUtils.readInt(buffer);
}
+ Expression outputExpression = Expression.deserialize(buffer);
PlanNodeId planNodeId = PlanNodeId.deserialize(buffer);
- return new FileAggregationScanNode(planNodeId, pathPattern, aggregationDescriptor, levels);
+ return new FileAggregationScanNode(
+ planNodeId, pathPattern, aggregationDescriptor, levels, outputExpression);
}
@Override
[iotdb] 10/13: fix aligned bug
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/fileScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76eff499bf3ffe3caff0b6bf3f3d620a2c793f5b
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Mar 21 10:27:19 2023 +0800
fix aligned bug
---
.../db/mpp/execution/operator/source/FileAggregationScanUtil.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
index 12290b4ebc..e51fee215e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/FileAggregationScanUtil.java
@@ -167,6 +167,10 @@ public class FileAggregationScanUtil {
private void consumeTimeseriesMetadata(
PartialPath devicePath, TimeseriesMetadata timeseriesMetadata) {
+ if (timeseriesMetadata.getMeasurementId().isEmpty()) {
+ // skip time
+ return;
+ }
PartialPath fullPath = devicePath.concatNode(timeseriesMetadata.getMeasurementId());
if (!pathPattern.matchFullPath(fullPath)) {
return;