You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/05/15 06:28:56 UTC
[iotdb] branch beyyes/optimize_tsbs created (now 56b3a97afef)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 56b3a97afef change code for tsbs siatuation
This branch includes the following new commits:
new f3db0e129ce Merge branch 'master' of github.com:apache/iotdb
new fd7eb6b33ec Merge branch 'master' of github.com:apache/iotdb
new 4f8a48a0d60 Merge branch 'master' of github.com:apache/iotdb
new 56b3a97afef change code for tsbs siatuation
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 04/04: change code for tsbs siatuation
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 56b3a97afef931a4491163f23d88d916835428aa
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Mon May 15 14:28:33 2023 +0800
change code for tsbs siatuation
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 6 ++--
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../mpp/plan/scheduler/SimpleQueryTerminator.java | 1 +
.../iotdb/db/query/context/QueryContext.java | 42 +++++++++++-----------
.../tsfile/read/common/block/TsBlockBuilder.java | 2 +-
5 files changed, 28 insertions(+), 25 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index d93d73fd998..da970d2b44b 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -56,16 +56,16 @@ public class ConfigNodeConfig {
private int configRegionId = 0;
/** ConfigNodeGroup consensus protocol */
- private String configNodeConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+ private String configNodeConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Schema region consensus protocol */
- private String schemaRegionConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+ private String schemaRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Default number of SchemaRegion replicas */
private int schemaReplicationFactor = 1;
/** Data region consensus protocol */
- private String dataRegionConsensusProtocolClass = ConsensusFactory.IOT_CONSENSUS;
+ private String dataRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Default number of DataRegion replicas */
private int dataReplicationFactor = 1;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index 67b42a4319f..819625c53ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -90,7 +90,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
for (Aggregator aggregator : aggregators) {
dataTypes.addAll(Arrays.asList(aggregator.getOutputType()));
}
- this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
+ this.resultTsBlockBuilder = new TsBlockBuilder(60, dataTypes);
this.cachedRawDataSize =
(1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
index 841abf4deae..954df88d381 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
@@ -101,6 +101,7 @@ public class SimpleQueryTerminator implements IQueryTerminator {
if (unfinishedFIs.isEmpty()) {
continue;
}
+
try (SyncDataNodeInternalServiceClient client =
internalServiceClientManager.borrowClient(endPoint)) {
client.cancelQuery(new TCancelQueryReq(queryId.getId(), unfinishedFIs, false));
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 2a44762fddd..17366bccd2a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory;
import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory.ModsSerializer;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -95,26 +94,29 @@ public class QueryContext {
public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
// TODO change a way to do the existing check to avoid this IO call each time.
+ return Collections.emptyList();
+
// if the mods file does not exist, do not add it to the cache
- if (!modFile.exists()) {
- return Collections.emptyList();
- }
- Map<String, List<Modification>> fileModifications =
- filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new ConcurrentHashMap<>());
- return fileModifications.computeIfAbsent(
- path.getFullPath(),
- k -> {
- PatternTreeMap<Modification, ModsSerializer> allModifications =
- fileModCache.get(modFile.getFilePath());
- if (allModifications == null) {
- allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
- for (Modification modification : modFile.getModifications()) {
- allModifications.append(modification.getPath(), modification);
- }
- fileModCache.put(modFile.getFilePath(), allModifications);
- }
- return sortAndMerge(allModifications.getOverlapped(path));
- });
+ // if (!modFile.exists()) {
+ // return Collections.emptyList();
+ // }
+ // Map<String, List<Modification>> fileModifications =
+ // filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new
+ // ConcurrentHashMap<>());
+ // return fileModifications.computeIfAbsent(
+ // path.getFullPath(),
+ // k -> {
+ // PatternTreeMap<Modification, ModsSerializer> allModifications =
+ // fileModCache.get(modFile.getFilePath());
+ // if (allModifications == null) {
+ // allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
+ // for (Modification modification : modFile.getModifications()) {
+ // allModifications.append(modification.getPath(), modification);
+ // }
+ // fileModCache.put(modFile.getFilePath(), allModifications);
+ // }
+ // return sortAndMerge(allModifications.getOverlapped(path));
+ // });
}
private List<Modification> sortAndMerge(List<Modification> modifications) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
index c309835a09f..b13c6ded4d9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
@@ -44,7 +44,7 @@ public class TsBlockBuilder {
// so the underlying data is larger than the object overhead, and the size is power of 2.
//
// This could be any other small number.
- private static final int DEFAULT_INITIAL_EXPECTED_ENTRIES = 8;
+ private static final int DEFAULT_INITIAL_EXPECTED_ENTRIES = 60;
private static final int MAX_LINE_NUMBER =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
[iotdb] 02/04: Merge branch 'master' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fd7eb6b33ec7ba518036137ac2b04023f35fadd9
Merge: f3db0e129ce ea66fc5bee0
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu May 4 14:44:55 2023 +0800
Merge branch 'master' of github.com:apache/iotdb
client-py/SessionExample.py | 1 -
client-py/iotdb/Session.py | 323 ++++++++++-----------
client-py/iotdb/utils/IoTDBConnectionException.py | 29 ++
client-py/iotdb/utils/IoTDBRpcDataSet.py | 10 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 6 +-
.../statemachine/ConfigRegionStateMachine.java | 2 -
.../confignode/manager/ClusterSchemaManager.java | 7 +
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../iotdb/confignode/manager/load/LoadManager.java | 117 +++++---
.../manager/load/balancer/RouteBalancer.java | 268 ++++-------------
.../load/balancer/router/RegionRouteMap.java | 176 -----------
.../confignode/manager/load/cache/LoadCache.java | 156 +++++++++-
.../manager/load/cache/route/RegionRouteCache.java | 138 +++++++++
.../manager/load/service/HeartbeatService.java | 1 -
.../manager/load/service/StatisticsService.java | 70 +++--
.../manager/load/subscriber/RouteChangeEvent.java | 35 +--
.../manager/partition/PartitionManager.java | 6 +-
.../persistence/partition/PartitionInfo.java | 5 +
.../procedure/env/ConfigNodeProcedureEnv.java | 34 +--
.../procedure/env/DataNodeRemoveHandler.java | 4 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 2 +-
.../impl/schema/DataNodeRegionTaskExecutor.java | 4 +-
.../impl/schema/DeleteDatabaseProcedure.java | 2 -
.../statemachine/CreateRegionGroupsProcedure.java | 1 -
.../load/balancer/router/RegionRouteMapTest.java | 82 ------
docs/UserGuide/QuickStart/Files.md | 2 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 14 +-
docs/zh/UserGuide/QuickStart/Files.md | 2 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 21 +-
.../metrics/metricsets/jvm/JvmThreadMetrics.java | 31 +-
.../metricsets/net/LinuxNetMetricManager.java | 27 +-
.../iotdb/metrics/metricsets/net/NetMetrics.java | 2 +-
pom.xml | 4 +-
.../iotdb/db/engine/TsFileMetricManager.java | 63 ++--
.../performer/impl/FastCompactionPerformer.java | 16 +-
.../impl/ReadChunkCompactionPerformer.java | 11 +-
.../impl/ReadPointCompactionPerformer.java | 21 +-
.../execute/task/AbstractCompactionTask.java | 4 +
.../execute/task/CompactionTaskSummary.java | 18 ++
.../readchunk/AlignedSeriesCompactionExecutor.java | 6 -
.../iotdb/db/service/metrics/FileMetrics.java | 26 +-
41 files changed, 851 insertions(+), 900 deletions(-)
[iotdb] 03/04: Merge branch 'master' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4f8a48a0d6095e79dffbfa2706402f27f223593e
Merge: fd7eb6b33ec a99aeff5410
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu May 11 14:56:51 2023 +0800
Merge branch 'master' of github.com:apache/iotdb
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 22 +
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
client-py/tests/test_dataframe.py | 2 +
.../confignode/client/DataNodeRequestType.java | 7 +-
.../client/async/AsyncDataNodeClientPool.java | 15 +-
.../client/async/handlers/AsyncClientHandler.java | 1 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 27 +
.../confignode/manager/ClusterSchemaManager.java | 13 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 32 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 23 +-
.../pipe/task/AbstractOperatePipeProcedureV2.java | 50 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 34 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 16 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 28 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 28 +-
.../iotdb/confignode/service/ConfigNode.java | 6 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 1 +
consensus/pom.xml | 2 +-
.../iotdb/consensus/config/IoTConsensusConfig.java | 36 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 8 +-
.../logdispatcher/LogDispatcherThreadMetrics.java | 22 +
.../consensus/iot/logdispatcher/SyncStatus.java | 2 -
.../ratis/ApplicationStateMachineProxy.java | 10 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 3 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 17 +
docs/Community/Materials.md | 103 ++--
docs/UserGuide/API/Programming-Java-Native-API.md | 98 ---
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 8 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 29 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 4 +-
.../UserGuide/API/Programming-Java-Native-API.md | 93 ---
.../UserGuide/Ecosystem-Integration/Workbench.md | 79 +--
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 24 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 29 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 6 +-
.../Syntax-Conventions/Session-And-TsFile-API.md | 4 +-
.../iotdb/it/env/cluster/MppCommonConfig.java | 12 +
.../it/env/cluster/MppSharedCommonConfig.java | 14 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 10 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../iotdb/db/it/orderBy/IoTDBOrderBy2IT.java | 48 ++
.../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 6 +-
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 2 +
.../db/it/schema/IoTDBAutoCreateSchemaIT.java | 17 +-
.../it/schema/IoTDBCreateAlignedTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBCreateStorageGroupIT.java | 18 +-
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 17 +-
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 26 +-
.../it/schema/IoTDBDeleteAlignedTimeseriesIT.java | 18 +-
.../db/it/schema/IoTDBDeleteStorageGroupIT.java | 17 +-
.../db/it/schema/IoTDBDeleteTimeseriesIT.java | 18 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 49 +-
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 63 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 49 +-
.../db/it/schema/IoTDBSortedShowTimeseriesIT.java | 20 +-
.../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java | 17 +-
.../org/apache/iotdb/db/it/schema/IoTDBTagIT.java | 17 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 30 +
.../org/apache/iotdb/util/AbstractSchemaIT.java | 48 +-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 14 +-
.../metrics/metricsets/jvm/JvmMemoryMetrics.java | 6 +-
.../resources/conf/iotdb-common.properties | 27 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +-
.../plugin/builtin/collector/DefaultCollector.java | 60 ++
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 10 +-
.../commons/pipe/task/meta/PipeMetaKeeper.java | 4 +
.../commons/pipe/task/meta/PipeRuntimeMeta.java | 20 +-
.../commons/pipe/task/meta/PipeStaticMeta.java | 87 +--
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 51 +-
.../node/common/AbstractMeasurementMNode.java | 4 +
.../schema/node/role/IMeasurementMNode.java | 2 +
.../commons/schema/node/utils/IMNodeFactory.java | 4 +
.../commons/schema/tree/AbstractTreeVisitor.java | 122 +++-
.../iotdb/pipe/api/customizer/PipeParameters.java | 22 +
.../api/exception/PipeConnectionException.java | 18 +-
.../exception/PipeRuntimeCriticalException.java | 19 +-
.../pipe/api/exception/PipeRuntimeException.java | 19 +-
.../exception/PipeRuntimeNonCriticalException.java | 19 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 7 +
.../rocksdb/mnode/RMeasurementMNode.java | 5 +
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 7 +
.../resources/conf/iotdb-datanode.properties | 15 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 74 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 45 +-
.../statemachine/DataRegionStateMachine.java | 3 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 7 +-
.../CompactionMemoryNotEnoughException.java} | 9 +-
.../execute/task/CrossSpaceCompactionTask.java | 6 +-
.../estimator/AbstractCompactionEstimator.java | 3 +-
.../ReadPointCrossCompactionEstimator.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 35 +-
.../apache/iotdb/db/metadata/MetadataConstant.java | 6 +
.../db/metadata/cache/DataNodeSchemaCache.java | 14 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 17 +-
.../db/metadata/cache/TimeSeriesSchemaCache.java | 3 +-
.../dualkeycache/impl/DualKeyCacheBuilder.java | 18 +-
.../dualkeycache/impl/DualKeyCachePolicy.java | 3 +-
.../dualkeycache/impl/FIFOCacheEntryManager.java | 189 ++++++
.../mnode/config/factory/ConfigMNodeFactory.java | 7 +
.../mnode/mem/factory/MemMNodeFactory.java | 14 +
...MeasurementMNode.java => LogicalViewMNode.java} | 30 +-
.../metadata/mnode/mem/impl/LogicalViewSchema.java | 215 +++++++
.../metadata/mnode/mem/impl/MeasurementMNode.java | 5 +
.../metadata/mnode/mem/info/LogicalViewInfo.java | 161 +++++
.../schemafile/factory/CacheMNodeFactory.java | 7 +
.../schemafile/impl/CachedMeasurementMNode.java | 5 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 10 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 75 ++-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 48 +-
.../db/metadata/mtree/traverser/Traverser.java | 61 ++
.../traverser/TraverserWithLimitOffsetWrapper.java | 5 +
.../mtree/traverser/basic/DatabaseTraverser.java | 5 +
.../mtree/traverser/basic/EntityTraverser.java | 8 +
.../mtree/traverser/basic/MNodeTraverser.java | 5 +
.../traverser/basic/MeasurementTraverser.java | 5 +
.../plan/schemaregion/SchemaRegionPlanType.java | 2 +
.../plan/schemaregion/SchemaRegionPlanVisitor.java | 5 +
.../impl/SchemaRegionPlanDeserializer.java | 24 +
.../impl/SchemaRegionPlanSerializer.java | 27 +
.../impl/SchemaRegionPlanTxtSerializer.java | 21 +
.../impl/write/CreateLogicalViewPlanImpl.java | 75 +++
.../schemaregion/result/ShowTimeSeriesResult.java | 14 +-
.../schemaregion/write/ICreateLogicalViewPlan.java | 66 +++
.../metadata/query/info/ITimeSeriesSchemaInfo.java | 6 +-
.../metadata/rescon/MemSchemaRegionStatistics.java | 8 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 5 +
.../schemaregion/SchemaRegionMemoryImpl.java | 40 ++
.../schemaregion/SchemaRegionSchemaFileImpl.java | 7 +
.../metadata/template/ClusterTemplateManager.java | 36 ++
.../view/viewExpression/ViewExpression.java | 318 ++++++++++
.../view/viewExpression/ViewExpressionType.java | 76 +++
.../binary/BinaryViewExpression.java | 119 ++++
.../binary/arithmetic/AdditionViewExpression.java | 61 ++
.../arithmetic/ArithmeticBinaryViewExpression.java | 52 ++
.../binary/arithmetic/DivisionViewExpression.java | 61 ++
.../binary/arithmetic/ModuloViewExpression.java | 61 ++
.../arithmetic/MultiplicationViewExpression.java | 63 ++
.../arithmetic/SubtractionViewExpression.java | 61 ++
.../compare/CompareBinaryViewExpression.java | 52 ++
.../binary/compare/EqualToViewExpression.java | 57 ++
.../binary/compare/GreaterEqualViewExpression.java | 57 ++
.../binary/compare/GreaterThanViewExpression.java | 57 ++
.../binary/compare/LessEqualViewExpression.java | 57 ++
.../binary/compare/LessThanViewExpression.java | 57 ++
.../binary/compare/NonEqualViewExpression.java | 57 ++
.../binary/logic/LogicAndViewExpression.java | 57 ++
.../binary/logic/LogicBinaryViewExpression.java | 52 ++
.../binary/logic/LogicOrViewExpression.java | 57 ++
.../viewExpression/leaf/ConstantViewOperand.java | 98 +++
.../view/viewExpression/leaf/LeafViewOperand.java} | 20 +-
.../view/viewExpression/leaf/NullViewOperand.java | 61 ++
.../viewExpression/leaf/TimeSeriesViewOperand.java | 87 +++
.../viewExpression/leaf/TimestampViewOperand.java | 72 +++
.../multi/FunctionViewExpression.java | 198 +++++++
.../ternary/BetweenViewExpression.java | 111 ++++
.../ternary/TernaryViewExpression.java | 105 ++++
.../viewExpression/unary/InViewExpression.java | 106 ++++
.../viewExpression/unary/IsNullViewExpression.java | 89 +++
.../viewExpression/unary/LikeViewExpression.java | 163 +++++
.../unary/LogicNotViewExpression.java | 61 ++
.../unary/NegationViewExpression.java | 61 ++
.../unary/RegularViewExpression.java | 105 ++++
.../viewExpression/unary/UnaryViewExpression.java | 73 +++
.../visitor/GetSourcePathsVisitor.java | 95 +++
.../visitor/TransformToExpressionVisitor.java | 320 ++++++++++
.../visitor/ViewExpressionVisitor.java | 219 +++++++
.../metadata/visitor/SchemaExecutionVisitor.java | 22 +
.../db/mpp/common/header/ColumnHeaderConstant.java | 6 +-
.../visitor/SchemaTreeDeviceVisitor.java | 5 +
.../visitor/SchemaTreeMeasurementVisitor.java | 5 +
.../SchemaTreeVisitorWithLimitOffsetWrapper.java | 6 +
.../iotdb/db/mpp/execution/driver/Driver.java | 21 +
.../db/mpp/execution/driver/DriverContext.java | 9 +
.../mpp/execution/exchange/SharedTsBlockQueue.java | 12 +-
.../execution/executor/RegionWriteExecutor.java | 49 ++
.../fragment/FragmentInstanceContext.java | 10 +
.../fragment/FragmentInstanceExecution.java | 17 +
.../operator/process/DeviceViewIntoOperator.java | 6 +-
.../operator/process/MergeSortOperator.java | 9 +-
.../execution/operator/process/SortOperator.java | 258 ++++++--
.../process/join/merge/MergeSortComparator.java | 45 +-
...rtKeyComparator.java => SortKeyComparator.java} | 11 +-
.../schema/source/TimeSeriesSchemaSource.java | 14 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 50 +-
.../visitor/TransformToViewExpressionVisitor.java | 391 ++++++++++++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 100 +++-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 6 +-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 21 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 28 +-
.../plan/planner/distribution/SourceRewriter.java | 4 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../node/metedata/write/CreateLogicalViewNode.java | 250 ++++++++
.../plan/planner/plan/node/write/InsertNode.java | 8 +
.../db/mpp/plan/scheduler/AsyncPlanNodeSender.java | 74 ++-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 27 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 20 +-
.../iotdb/db/mpp/plan/statement/StatementType.java | 2 +
.../db/mpp/plan/statement/StatementVisitor.java | 7 +
.../plan/statement/component/OrderByComponent.java | 6 +-
.../component/{SortKey.java => OrderByKey.java} | 2 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 2 +-
.../metadata/CreateLogicalViewStatement.java | 246 ++++++++
.../plan/statement/sys/ShowQueriesStatement.java | 4 +-
.../org/apache/iotdb/db/pipe/agent/PipeAgent.java | 2 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 26 +-
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 281 ++++++++-
.../PipeCollectorConstant.java} | 20 +-
.../PipeConnectorConstant.java} | 12 +-
.../PipeProcessorConstant.java} | 12 +-
.../core/collector/IoTDBDataRegionCollector.java | 85 +++
.../PipeHistoricalDataRegionTsFileCollector.java | 103 ++++
.../realtime/PipeRealtimeDataRegionCollector.java | 21 +-
... => PipeRealtimeDataRegionHybridCollector.java} | 33 +-
.../connector/PipeConnectorSubtaskLifeCycle.java | 31 +-
.../connector/PipeConnectorSubtaskManager.java | 19 +-
.../core/event/impl/PipeTsFileInsertionEvent.java | 1 -
.../event/view/collector/PipeEventCollector.java | 59 +-
...anager.java => PipeSubtaskExecutorManager.java} | 12 +-
.../execution/scheduler/PipeTaskScheduler.java | 18 +-
.../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 66 +++
.../org/apache/iotdb/db/pipe/task/PipeTask.java | 9 +-
.../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 60 +-
.../apache/iotdb/db/pipe/task/PipeTaskManager.java | 96 +++
.../EventSupplier.java} | 24 +-
.../ListenableBlockingPendingQueue.java} | 18 +-
.../db/pipe/task/queue/ListenablePendingQueue.java | 159 +++++
.../ListenableUnblockingPendingQueue.java} | 18 +-
.../PendingQueueEmptyToNotEmptyListener.java} | 8 +-
.../queue/PendingQueueFullToNotFullListener.java} | 8 +-
.../PendingQueueNotEmptyToEmptyListener.java} | 8 +-
.../queue/PendingQueueNotFullToFullListener.java} | 8 +-
.../db/pipe/task/stage/PipeTaskCollectorStage.java | 83 ++-
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 100 +---
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 105 +++-
.../iotdb/db/pipe/task/stage/PipeTaskStage.java | 95 ++-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 87 ++-
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 20 +-
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 6 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 20 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 68 +--
.../org/apache/iotdb/db/tools/DiskSpiller.java | 166 ++++++
.../apache/iotdb/db/tools/FileSpillerReader.java | 143 +++++
.../MemoryReader.java} | 35 +-
.../apache/iotdb/db/tools/SortBufferManager.java | 73 +++
.../org/apache/iotdb/db/tools/SortReader.java} | 27 +-
.../db/utils/datastructure/MergeSortHeap.java | 4 +-
.../iotdb/db/utils/datastructure/MergeSortKey.java | 25 +-
.../{MergeSortKey.java => SortKey.java} | 14 +-
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 9 +-
.../cache/dualkeycache/DualKeyCacheTest.java | 17 +
.../schemaRegion/SchemaRegionBasicTest.java | 103 +++-
.../metadata/view/ViewExpressionToStringTest.java | 183 ++++++
.../execution/operator/MergeSortOperatorTest.java | 36 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 14 +-
.../mpp/execution/operator/SortOperatorTest.java | 273 +++++++++
.../db/mpp/plan/optimization/TestPlanBuilder.java | 6 +-
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 16 +-
.../plan/node/process/DeviceViewNodeSerdeTest.java | 6 +-
.../plan/plan/node/process/SortNodeSerdeTest.java | 4 +-
.../collector/CachedSchemaPatternMatcherTest.java | 48 +-
.../core/collector/PipeRealtimeCollectTest.java | 59 +-
.../executor/PipeConnectorSubtaskExecutorTest.java | 8 +-
.../executor/PipeProcessorSubtaskExecutorTest.java | 10 +-
.../DataNodeInternalRPCServiceImplTest.java | 31 +-
.../sync/pipedata/BufferedPipeDataQueueTest.java | 658 ---------------------
thrift/src/main/thrift/datanode.thrift | 35 +-
271 files changed, 10279 insertions(+), 2197 deletions(-)
[iotdb] 01/04: Merge branch 'master' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f3db0e129ce9c48dbd0b7f55f47a30cf4c6938c4
Merge: 2aa22287804 5fb6e2b76c6
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Apr 28 20:32:22 2023 +0800
Merge branch 'master' of github.com:apache/iotdb
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 1 +
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +
.../confignode/manager/ClusterSchemaManager.java | 13 +-
.../iotdb/confignode/manager/ConfigManager.java | 11 +-
.../consensus/iot/IoTConsensusServerImpl.java | 13 +-
.../db/it/last/IoTDBLastQueryLastCache2IT.java | 60 +
.../db/it/last/IoTDBLastQueryLastCacheIT.java | 145 +++
.../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 1224 ++++++++++++++++++++
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 49 +
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 61 +
.../iotdb/metrics/metricsets/net/NetMetrics.java | 10 +-
.../resources/conf/iotdb-common.properties | 14 +-
.../iotdb/commons/concurrent/ThreadName.java | 2 +-
.../org/apache/iotdb/commons/path/AlignedPath.java | 6 +
.../org/apache/iotdb/commons/path/PartialPath.java | 40 +
.../builtin/connector/DoNothingConnector.java | 6 +-
.../builtin/processor/DoNothingProcessor.java | 6 +-
.../commons/udf/service/UDFClassLoaderManager.java | 6 +-
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 12 +-
.../org/apache/iotdb/pipe/api/PipeCollector.java | 99 ++
.../org/apache/iotdb/pipe/api/PipeConnector.java | 6 +-
.../org/apache/iotdb/pipe/api/PipeProcessor.java | 6 +-
.../iotdb/pipe/api/collector/EventCollector.java | 6 +-
.../iotdb/pipe/api/collector/RowCollector.java | 2 +-
.../PipeCollectorRuntimeConfiguration.java | 13 +-
.../org/apache/iotdb/pipe/api/event/Event.java | 6 +-
.../org/apache/iotdb/pipe/api/event/EventType.java | 8 +-
.../event/{ => dml}/deletion/DeletionEvent.java | 8 +-
.../{ => dml}/insertion/TabletInsertionEvent.java | 8 +-
.../{ => dml}/insertion/TsFileInsertionEvent.java | 8 +-
pom.xml | 6 +
server/pom.xml | 6 +-
.../resources/conf/iotdb-datanode.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +-
.../directories/strategy/SequenceStrategy.java | 2 +-
.../performer/impl/FastCompactionPerformer.java | 3 +-
.../execute/task/AbstractCompactionTask.java | 8 +-
.../execute/task/CrossSpaceCompactionTask.java | 11 +-
.../execute/task/InnerSpaceCompactionTask.java | 7 +-
.../fast/AlignedSeriesCompactionExecutor.java | 16 +-
.../utils/writer/AbstractCompactionWriter.java | 3 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 2 +-
.../db/engine/storagegroup/TsFileProcessor.java | 17 +-
.../metadata/template/ClusterTemplateManager.java | 17 +-
.../db/metadata/template/ITemplateManager.java | 3 +-
.../apache/iotdb/db/metadata/utils/MetaUtils.java | 11 +-
.../iotdb/db/mpp/execution/driver/Driver.java | 2 +-
.../execution/exchange/sink/LocalSinkChannel.java | 12 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 15 +-
.../execution/operator/process/SortOperator.java | 28 +-
.../operator/process/TransformOperator.java | 13 +-
.../process/join/merge/MergeSortComparator.java | 178 +--
.../process/join/merge/MergeSortKeyComparator.java | 53 +
.../execution/schedule/AbstractDriverThread.java | 9 +
.../schedule/queue/IndexedBlockingQueue.java | 1 +
.../multilevelqueue/MultilevelPriorityQueue.java | 4 +-
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 68 ++
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 138 ++-
.../db/mpp/plan/analyze/ConcatPathRewriter.java | 22 +-
.../db/mpp/plan/execution/QueryExecution.java | 10 +-
.../visitor/ColumnTransformerVisitor.java | 40 +-
.../visitor/IntermediateLayerVisitor.java | 4 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 44 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 143 ++-
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 19 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 12 +-
.../planner/distribution/DistributionPlanner.java | 5 +-
.../planner/distribution/ExchangeNodeAdder.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 48 +
.../planner/plan/node/write/InsertRowNode.java | 4 +
.../planner/plan/node/write/InsertRowsNode.java | 5 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 7 +
.../planner/plan/node/write/InsertTabletNode.java | 8 +
.../component/{SortKey.java => NullOrdering.java} | 19 +-
.../plan/statement/component/OrderByComponent.java | 56 +-
.../db/mpp/plan/statement/component/SortItem.java | 61 +-
.../db/mpp/plan/statement/component/SortKey.java | 16 +-
.../plan/statement/crud/InsertBaseStatement.java | 4 -
.../crud/InsertMultiTabletsStatement.java | 18 -
.../plan/statement/crud/InsertRowStatement.java | 17 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 9 -
.../plan/statement/crud/InsertRowsStatement.java | 17 -
.../plan/statement/crud/InsertTabletStatement.java | 13 -
.../db/mpp/plan/statement/crud/QueryStatement.java | 80 ++
.../dag/builder/EvaluationDAGBuilder.java | 4 +-
.../dag/input/QueryDataSetInputLayer.java | 5 +-
.../intermediate/ConstantIntermediateLayer.java | 3 +-
.../dag/intermediate/IntermediateLayer.java | 4 +-
.../MultiInputColumnIntermediateLayer.java | 2 +-
...InputColumnMultiReferenceIntermediateLayer.java | 2 +-
...nputColumnSingleReferenceIntermediateLayer.java | 2 +-
.../db/mpp/transformation/dag/udf/UDTFContext.java | 2 +-
.../mpp/transformation/dag/udf/UDTFExecutor.java | 2 +-
.../datastructure/SerializableList.java | 6 +-
.../row/ElasticSerializableRowRecordList.java | 6 +-
.../row/SerializableRowRecordList.java | 2 +-
.../tv/ElasticSerializableBinaryTVList.java | 2 +-
.../tv/ElasticSerializableTVList.java | 8 +-
.../datastructure/tv/SerializableTVList.java | 2 +-
.../apache/iotdb/db/pipe/config/PipeConfig.java | 65 ++
.../collector/realtime/PipeRealtimeCollector.java | 22 -
.../realtime/PipeRealtimeDataRegionCollector.java | 81 ++
.../PipeRealtimeHybridDataRegionCollector.java | 171 +++
.../realtime/assigner/DisruptorQueue.java | 111 ++
.../realtime/assigner/PipeDataRegionAssigner.java | 74 ++
.../realtime/cache/PipeRealtimeEventCache.java | 22 -
.../realtime/listener/IoTLogListerner.java | 22 -
.../listener/PipeInsertionDataNodeListener.java | 115 ++
.../realtime/listener/RatisLogListener.java | 22 -
.../realtime/listener/SimpleLogListener.java | 22 -
.../listener/TsFileGenerationListener.java | 22 -
.../matcher/CachedSchemaPatternMatcher.java | 200 ++++
.../realtime/matcher/PipeDataRegionMatcher.java | 47 +
.../pipe/core/collector/realtime/matcher/Rule.java | 22 -
.../realtime/matcher/RulePrefixMatchTree.java | 22 -
.../collector/realtime/recorder/TsFileEpoch.java | 22 -
.../realtime/recorder/TsFileEpochRecorder.java | 22 -
.../event/{ => impl}/PipeTabletInsertionEvent.java | 22 +-
.../event/{ => impl}/PipeTsFileInsertionEvent.java | 23 +-
.../pipe/core/event/indexer/PipeEventIndexer.java | 22 -
.../core/event/indexer/PipeIoTEventIndexer.java | 22 -
.../core/event/indexer/PipeRatisEventIndexer.java | 22 -
.../core/event/indexer/PipeSimpleEventIndexer.java | 22 -
.../core/event/indexer/PipeTsFileEventIndexer.java | 22 -
.../event/realtime/PipeRealtimeCollectEvent.java | 66 ++
.../realtime/PipeRealtimeCollectEventFactory.java | 49 +
.../db/pipe/core/event/realtime/TsFileEpoch.java | 68 ++
.../core/event/realtime/TsFileEpochManager.java | 70 ++
.../realtime/TsFileEpochStateMigrator.java} | 7 +-
.../pipe/core/event/{ => view}/access/PipeRow.java | 2 +-
.../event/{ => view}/access/PipeRowIterator.java | 2 +-
.../{ => view}/collector/PipeEventCollector.java | 8 +-
.../{ => view}/collector/PipeRowCollector.java | 2 +-
.../PipeResourceManager.java} | 29 +-
.../db/pipe/task/subtask/PipeConnectorSubtask.java | 6 +-
.../db/pipe/task/subtask/PipeProcessorSubtask.java | 6 +-
.../rest/v1/handler/ExecuteStatementHandler.java | 4 +-
.../rest/v1/handler/QueryDataSetHandler.java | 5 +-
.../rest/v2/handler/ExecuteStatementHandler.java | 4 +-
.../rest/v2/handler/QueryDataSetHandler.java | 5 +-
.../db/query/control/QueryResourceManager.java | 7 +-
.../db/service/TemporaryQueryDataFileService.java | 14 +-
.../db/service/metrics/CompactionMetrics.java | 12 +-
.../metrics/recorder/CompactionMetricsManager.java | 6 +-
.../db/service/thrift/ProcessorWithMetrics.java | 2 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../java/org/apache/iotdb/db/wal/WALManager.java | 8 +-
.../db/wal/allocation/FirstCreateStrategy.java | 3 +
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 41 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 14 +-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 6 -
.../engine/compaction/AbstractCompactionTest.java | 3 +
.../compaction/CompactionTaskComparatorTest.java | 8 +-
.../compaction/FastAlignedCrossCompactionTest.java | 347 +++++-
.../FastInnerCompactionPerformerTest.java | 30 +-
.../FastNonAlignedCrossCompactionTest.java | 347 +++++-
.../ReadPointAlignedCrossCompactionTest.java | 347 +++++-
.../ReadPointNonAlignedCrossCompactionTest.java | 347 +++++-
.../CrossSpaceCompactionWithFastPerformerTest.java | 7 +-
...eCompactionWithFastPerformerValidationTest.java | 194 ++--
...actionWithReadPointPerformerValidationTest.java | 168 +--
.../utils/MultiTsFileDeviceIteratorTest.java | 12 +-
.../apache/iotdb/db/metric/MetricServiceTest.java | 3 -
.../execution/operator/MergeSortOperatorTest.java | 20 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 200 ++++
.../plan/node/write/WritePlanNodeSplitTest.java | 15 +-
.../collector/CachedSchemaPatternMatcherTest.java | 149 +++
.../core/collector/PipeRealtimeCollectTest.java | 236 ++++
.../udf/datastructure/SerializableListTest.java | 2 +-
server/src/test/resources/logback-test.xml | 1 +
.../tsfile/file/metadata/enums/TSDataType.java | 17 +
.../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 4 -
172 files changed, 6166 insertions(+), 1219 deletions(-)