You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/05/15 06:28:56 UTC

[iotdb] branch beyyes/optimize_tsbs created (now 56b3a97afef)

This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a change to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git


      at 56b3a97afef change code for tsbs siatuation

This branch includes the following new commits:

     new f3db0e129ce Merge branch 'master' of github.com:apache/iotdb
     new fd7eb6b33ec Merge branch 'master' of github.com:apache/iotdb
     new 4f8a48a0d60 Merge branch 'master' of github.com:apache/iotdb
     new 56b3a97afef change code for tsbs siatuation

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[iotdb] 04/04: change code for tsbs siatuation

Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 56b3a97afef931a4491163f23d88d916835428aa
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Mon May 15 14:28:33 2023 +0800

    change code for tsbs siatuation
---
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  6 ++--
 .../AbstractSeriesAggregationScanOperator.java     |  2 +-
 .../mpp/plan/scheduler/SimpleQueryTerminator.java  |  1 +
 .../iotdb/db/query/context/QueryContext.java       | 42 +++++++++++-----------
 .../tsfile/read/common/block/TsBlockBuilder.java   |  2 +-
 5 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index d93d73fd998..da970d2b44b 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -56,16 +56,16 @@ public class ConfigNodeConfig {
   private int configRegionId = 0;
 
   /** ConfigNodeGroup consensus protocol */
-  private String configNodeConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+  private String configNodeConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
 
   /** Schema region consensus protocol */
-  private String schemaRegionConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+  private String schemaRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
 
   /** Default number of SchemaRegion replicas */
   private int schemaReplicationFactor = 1;
 
   /** Data region consensus protocol */
-  private String dataRegionConsensusProtocolClass = ConsensusFactory.IOT_CONSENSUS;
+  private String dataRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
 
   /** Default number of DataRegion replicas */
   private int dataReplicationFactor = 1;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index 67b42a4319f..819625c53ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -90,7 +90,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
     for (Aggregator aggregator : aggregators) {
       dataTypes.addAll(Arrays.asList(aggregator.getOutputType()));
     }
-    this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
+    this.resultTsBlockBuilder = new TsBlockBuilder(60, dataTypes);
 
     this.cachedRawDataSize =
         (1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
index 841abf4deae..954df88d381 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/SimpleQueryTerminator.java
@@ -101,6 +101,7 @@ public class SimpleQueryTerminator implements IQueryTerminator {
       if (unfinishedFIs.isEmpty()) {
         continue;
       }
+      
       try (SyncDataNodeInternalServiceClient client =
           internalServiceClientManager.borrowClient(endPoint)) {
         client.cancelQuery(new TCancelQueryReq(queryId.getId(), unfinishedFIs, false));
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 2a44762fddd..17366bccd2a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.PatternTreeMap;
 import org.apache.iotdb.db.engine.modification.Deletion;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory;
 import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory.ModsSerializer;
 import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
 
@@ -95,26 +94,29 @@ public class QueryContext {
   public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
     // TODO change a way to do the existing check to avoid this IO call each time.
 
+    return Collections.emptyList();
+
     // if the mods file does not exist, do not add it to the cache
-    if (!modFile.exists()) {
-      return Collections.emptyList();
-    }
-    Map<String, List<Modification>> fileModifications =
-        filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new ConcurrentHashMap<>());
-    return fileModifications.computeIfAbsent(
-        path.getFullPath(),
-        k -> {
-          PatternTreeMap<Modification, ModsSerializer> allModifications =
-              fileModCache.get(modFile.getFilePath());
-          if (allModifications == null) {
-            allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
-            for (Modification modification : modFile.getModifications()) {
-              allModifications.append(modification.getPath(), modification);
-            }
-            fileModCache.put(modFile.getFilePath(), allModifications);
-          }
-          return sortAndMerge(allModifications.getOverlapped(path));
-        });
+    //    if (!modFile.exists()) {
+    //      return Collections.emptyList();
+    //    }
+    //    Map<String, List<Modification>> fileModifications =
+    //        filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new
+    // ConcurrentHashMap<>());
+    //    return fileModifications.computeIfAbsent(
+    //        path.getFullPath(),
+    //        k -> {
+    //          PatternTreeMap<Modification, ModsSerializer> allModifications =
+    //              fileModCache.get(modFile.getFilePath());
+    //          if (allModifications == null) {
+    //            allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
+    //            for (Modification modification : modFile.getModifications()) {
+    //              allModifications.append(modification.getPath(), modification);
+    //            }
+    //            fileModCache.put(modFile.getFilePath(), allModifications);
+    //          }
+    //          return sortAndMerge(allModifications.getOverlapped(path));
+    //        });
   }
 
   private List<Modification> sortAndMerge(List<Modification> modifications) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
index c309835a09f..b13c6ded4d9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java
@@ -44,7 +44,7 @@ public class TsBlockBuilder {
   // so the underlying data is larger than the object overhead, and the size is power of 2.
   //
   // This could be any other small number.
-  private static final int DEFAULT_INITIAL_EXPECTED_ENTRIES = 8;
+  private static final int DEFAULT_INITIAL_EXPECTED_ENTRIES = 60;
 
   private static final int MAX_LINE_NUMBER =
       TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();


[iotdb] 02/04: Merge branch 'master' of github.com:apache/iotdb

Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit fd7eb6b33ec7ba518036137ac2b04023f35fadd9
Merge: f3db0e129ce ea66fc5bee0
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu May 4 14:44:55 2023 +0800

    Merge branch 'master' of github.com:apache/iotdb

 client-py/SessionExample.py                        |   1 -
 client-py/iotdb/Session.py                         | 323 ++++++++++-----------
 client-py/iotdb/utils/IoTDBConnectionException.py  |  29 ++
 client-py/iotdb/utils/IoTDBRpcDataSet.py           |  10 +-
 .../heartbeat/DataNodeHeartbeatHandler.java        |   6 +-
 .../statemachine/ConfigRegionStateMachine.java     |   2 -
 .../confignode/manager/ClusterSchemaManager.java   |   7 +
 .../iotdb/confignode/manager/ConfigManager.java    |   4 +-
 .../iotdb/confignode/manager/load/LoadManager.java | 117 +++++---
 .../manager/load/balancer/RouteBalancer.java       | 268 ++++-------------
 .../load/balancer/router/RegionRouteMap.java       | 176 -----------
 .../confignode/manager/load/cache/LoadCache.java   | 156 +++++++++-
 .../manager/load/cache/route/RegionRouteCache.java | 138 +++++++++
 .../manager/load/service/HeartbeatService.java     |   1 -
 .../manager/load/service/StatisticsService.java    |  70 +++--
 .../manager/load/subscriber/RouteChangeEvent.java  |  35 +--
 .../manager/partition/PartitionManager.java        |   6 +-
 .../persistence/partition/PartitionInfo.java       |   5 +
 .../procedure/env/ConfigNodeProcedureEnv.java      |  34 +--
 .../procedure/env/DataNodeRemoveHandler.java       |   4 +-
 .../impl/pipe/task/CreatePipeProcedureV2.java      |   2 +-
 .../impl/schema/DataNodeRegionTaskExecutor.java    |   4 +-
 .../impl/schema/DeleteDatabaseProcedure.java       |   2 -
 .../statemachine/CreateRegionGroupsProcedure.java  |   1 -
 .../load/balancer/router/RegionRouteMapTest.java   |  82 ------
 docs/UserGuide/QuickStart/Files.md                 |   2 +-
 docs/UserGuide/Reference/DataNode-Config-Manual.md |  14 +-
 docs/zh/UserGuide/QuickStart/Files.md              |   2 +-
 .../UserGuide/Reference/DataNode-Config-Manual.md  |  21 +-
 .../metrics/metricsets/jvm/JvmThreadMetrics.java   |  31 +-
 .../metricsets/net/LinuxNetMetricManager.java      |  27 +-
 .../iotdb/metrics/metricsets/net/NetMetrics.java   |   2 +-
 pom.xml                                            |   4 +-
 .../iotdb/db/engine/TsFileMetricManager.java       |  63 ++--
 .../performer/impl/FastCompactionPerformer.java    |  16 +-
 .../impl/ReadChunkCompactionPerformer.java         |  11 +-
 .../impl/ReadPointCompactionPerformer.java         |  21 +-
 .../execute/task/AbstractCompactionTask.java       |   4 +
 .../execute/task/CompactionTaskSummary.java        |  18 ++
 .../readchunk/AlignedSeriesCompactionExecutor.java |   6 -
 .../iotdb/db/service/metrics/FileMetrics.java      |  26 +-
 41 files changed, 851 insertions(+), 900 deletions(-)


[iotdb] 03/04: Merge branch 'master' of github.com:apache/iotdb

Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 4f8a48a0d6095e79dffbfa2706402f27f223593e
Merge: fd7eb6b33ec a99aeff5410
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu May 11 14:56:51 2023 +0800

    Merge branch 'master' of github.com:apache/iotdb

 .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4   |  22 +
 .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4  |   4 +
 client-py/tests/test_dataframe.py                  |   2 +
 .../confignode/client/DataNodeRequestType.java     |   7 +-
 .../client/async/AsyncDataNodeClientPool.java      |  15 +-
 .../client/async/handlers/AsyncClientHandler.java  |   1 +
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  27 +
 .../confignode/manager/ClusterSchemaManager.java   |  13 +-
 .../persistence/executor/ConfigPlanExecutor.java   |   7 +-
 .../confignode/persistence/pipe/PipeTaskInfo.java  |  32 +-
 .../procedure/env/ConfigNodeProcedureEnv.java      |  23 +-
 .../pipe/task/AbstractOperatePipeProcedureV2.java  |  50 +-
 .../impl/pipe/task/CreatePipeProcedureV2.java      |  34 +-
 .../impl/pipe/task/DropPipeProcedureV2.java        |  16 +-
 .../impl/pipe/task/StartPipeProcedureV2.java       |  28 +-
 .../impl/pipe/task/StopPipeProcedureV2.java        |  28 +-
 .../iotdb/confignode/service/ConfigNode.java       |   6 +-
 .../iotdb/confignode/persistence/PipeInfoTest.java |   1 +
 consensus/pom.xml                                  |   2 +-
 .../iotdb/consensus/config/IoTConsensusConfig.java |  36 +-
 .../consensus/iot/logdispatcher/LogDispatcher.java |   8 +-
 .../logdispatcher/LogDispatcherThreadMetrics.java  |  22 +
 .../consensus/iot/logdispatcher/SyncStatus.java    |   2 -
 .../ratis/ApplicationStateMachineProxy.java        |  10 +-
 .../iotdb/consensus/ratis/RatisConsensus.java      |   3 +-
 .../apache/iotdb/consensus/ratis/utils/Utils.java  |  17 +
 docs/Community/Materials.md                        | 103 ++--
 docs/UserGuide/API/Programming-Java-Native-API.md  |  98 ---
 docs/UserGuide/Monitor-Alert/Metric-Tool.md        |   8 +-
 docs/UserGuide/QuickStart/WayToGetIoTDB.md         |   2 +-
 docs/UserGuide/Reference/Common-Config-Manual.md   |  29 +-
 docs/UserGuide/Reference/DataNode-Config-Manual.md |   6 +-
 .../Syntax-Conventions/Session-And-TsFile-API.md   |   4 +-
 .../UserGuide/API/Programming-Java-Native-API.md   |  93 ---
 .../UserGuide/Ecosystem-Integration/Workbench.md   |  79 +--
 docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md     |  24 +-
 .../zh/UserGuide/Reference/Common-Config-Manual.md |  29 +-
 .../UserGuide/Reference/DataNode-Config-Manual.md  |   6 +-
 .../Syntax-Conventions/Session-And-TsFile-API.md   |   4 +-
 .../iotdb/it/env/cluster/MppCommonConfig.java      |  12 +
 .../it/env/cluster/MppSharedCommonConfig.java      |  14 +
 .../iotdb/it/env/remote/RemoteCommonConfig.java    |  10 +
 .../org/apache/iotdb/itbase/env/CommonConfig.java  |   4 +
 .../iotdb/db/it/orderBy/IoTDBOrderBy2IT.java       |  48 ++
 .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java |   6 +-
 .../apache/iotdb/db/it/query/IoTDBResultSetIT.java |   2 +
 .../db/it/schema/IoTDBAutoCreateSchemaIT.java      |  17 +-
 .../it/schema/IoTDBCreateAlignedTimeseriesIT.java  |  17 +-
 .../db/it/schema/IoTDBCreateStorageGroupIT.java    |  18 +-
 .../db/it/schema/IoTDBCreateTimeseriesIT.java      |  17 +-
 .../db/it/schema/IoTDBDeactivateTemplateIT.java    |  26 +-
 .../it/schema/IoTDBDeleteAlignedTimeseriesIT.java  |  18 +-
 .../db/it/schema/IoTDBDeleteStorageGroupIT.java    |  17 +-
 .../db/it/schema/IoTDBDeleteTimeseriesIT.java      |  18 +-
 .../iotdb/db/it/schema/IoTDBExtendTemplateIT.java  |  49 +-
 .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java   |  63 +-
 .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java  |  49 +-
 .../db/it/schema/IoTDBSortedShowTimeseriesIT.java  |  20 +-
 .../apache/iotdb/db/it/schema/IoTDBTagAlterIT.java |  17 +-
 .../org/apache/iotdb/db/it/schema/IoTDBTagIT.java  |  17 +-
 .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java  |  30 +
 .../org/apache/iotdb/util/AbstractSchemaIT.java    |  48 +-
 .../iotdb/zeppelin/it/IoTDBInterpreterIT.java      |  14 +-
 .../metrics/metricsets/jvm/JvmMemoryMetrics.java   |   6 +-
 .../resources/conf/iotdb-common.properties         |  27 +-
 .../org/apache/iotdb/commons/path/PartialPath.java |   4 +-
 .../pipe/plugin/builtin/BuiltinPipePlugin.java     |   8 +-
 .../plugin/builtin/collector/DefaultCollector.java |  60 ++
 .../iotdb/commons/pipe/task/meta/PipeMeta.java     |  10 +-
 .../commons/pipe/task/meta/PipeMetaKeeper.java     |   4 +
 .../commons/pipe/task/meta/PipeRuntimeMeta.java    |  20 +-
 .../commons/pipe/task/meta/PipeStaticMeta.java     |  87 +--
 .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java |  51 +-
 .../node/common/AbstractMeasurementMNode.java      |   4 +
 .../schema/node/role/IMeasurementMNode.java        |   2 +
 .../commons/schema/node/utils/IMNodeFactory.java   |   4 +
 .../commons/schema/tree/AbstractTreeVisitor.java   | 122 +++-
 .../iotdb/pipe/api/customizer/PipeParameters.java  |  22 +
 .../api/exception/PipeConnectionException.java     |  18 +-
 .../exception/PipeRuntimeCriticalException.java    |  19 +-
 .../pipe/api/exception/PipeRuntimeException.java   |  19 +-
 .../exception/PipeRuntimeNonCriticalException.java |  19 +-
 .../schemaregion/rocksdb/RSchemaRegion.java        |   7 +
 .../rocksdb/mnode/RMeasurementMNode.java           |   5 +
 .../metadata/tagSchemaRegion/TagSchemaRegion.java  |   7 +
 .../resources/conf/iotdb-datanode.properties       |  15 +-
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |  74 ++-
 .../org/apache/iotdb/db/conf/IoTDBDescriptor.java  |  45 +-
 .../statemachine/DataRegionStateMachine.java       |   3 +-
 .../db/engine/cache/TimeSeriesMetadataCache.java   |   7 +-
 .../CompactionMemoryNotEnoughException.java}       |   9 +-
 .../execute/task/CrossSpaceCompactionTask.java     |   6 +-
 .../estimator/AbstractCompactionEstimator.java     |   3 +-
 .../ReadPointCrossCompactionEstimator.java         |   6 +-
 .../iotdb/db/engine/storagegroup/DataRegion.java   |  35 +-
 .../apache/iotdb/db/metadata/MetadataConstant.java |   6 +
 .../db/metadata/cache/DataNodeSchemaCache.java     |  14 +-
 .../metadata/cache/DataNodeSchemaCacheMetrics.java |  17 +-
 .../db/metadata/cache/TimeSeriesSchemaCache.java   |   3 +-
 .../dualkeycache/impl/DualKeyCacheBuilder.java     |  18 +-
 .../dualkeycache/impl/DualKeyCachePolicy.java      |   3 +-
 .../dualkeycache/impl/FIFOCacheEntryManager.java   | 189 ++++++
 .../mnode/config/factory/ConfigMNodeFactory.java   |   7 +
 .../mnode/mem/factory/MemMNodeFactory.java         |  14 +
 ...MeasurementMNode.java => LogicalViewMNode.java} |  30 +-
 .../metadata/mnode/mem/impl/LogicalViewSchema.java | 215 +++++++
 .../metadata/mnode/mem/impl/MeasurementMNode.java  |   5 +
 .../metadata/mnode/mem/info/LogicalViewInfo.java   | 161 +++++
 .../schemafile/factory/CacheMNodeFactory.java      |   7 +
 .../schemafile/impl/CachedMeasurementMNode.java    |   5 +
 .../db/metadata/mtree/MTreeBelowSGCachedImpl.java  |  10 +-
 .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java  |  75 ++-
 .../mtree/snapshot/MemMTreeSnapshotUtil.java       |  48 +-
 .../db/metadata/mtree/traverser/Traverser.java     |  61 ++
 .../traverser/TraverserWithLimitOffsetWrapper.java |   5 +
 .../mtree/traverser/basic/DatabaseTraverser.java   |   5 +
 .../mtree/traverser/basic/EntityTraverser.java     |   8 +
 .../mtree/traverser/basic/MNodeTraverser.java      |   5 +
 .../traverser/basic/MeasurementTraverser.java      |   5 +
 .../plan/schemaregion/SchemaRegionPlanType.java    |   2 +
 .../plan/schemaregion/SchemaRegionPlanVisitor.java |   5 +
 .../impl/SchemaRegionPlanDeserializer.java         |  24 +
 .../impl/SchemaRegionPlanSerializer.java           |  27 +
 .../impl/SchemaRegionPlanTxtSerializer.java        |  21 +
 .../impl/write/CreateLogicalViewPlanImpl.java      |  75 +++
 .../schemaregion/result/ShowTimeSeriesResult.java  |  14 +-
 .../schemaregion/write/ICreateLogicalViewPlan.java |  66 +++
 .../metadata/query/info/ITimeSeriesSchemaInfo.java |   6 +-
 .../metadata/rescon/MemSchemaRegionStatistics.java |   8 +-
 .../db/metadata/schemaregion/ISchemaRegion.java    |   5 +
 .../schemaregion/SchemaRegionMemoryImpl.java       |  40 ++
 .../schemaregion/SchemaRegionSchemaFileImpl.java   |   7 +
 .../metadata/template/ClusterTemplateManager.java  |  36 ++
 .../view/viewExpression/ViewExpression.java        | 318 ++++++++++
 .../view/viewExpression/ViewExpressionType.java    |  76 +++
 .../binary/BinaryViewExpression.java               | 119 ++++
 .../binary/arithmetic/AdditionViewExpression.java  |  61 ++
 .../arithmetic/ArithmeticBinaryViewExpression.java |  52 ++
 .../binary/arithmetic/DivisionViewExpression.java  |  61 ++
 .../binary/arithmetic/ModuloViewExpression.java    |  61 ++
 .../arithmetic/MultiplicationViewExpression.java   |  63 ++
 .../arithmetic/SubtractionViewExpression.java      |  61 ++
 .../compare/CompareBinaryViewExpression.java       |  52 ++
 .../binary/compare/EqualToViewExpression.java      |  57 ++
 .../binary/compare/GreaterEqualViewExpression.java |  57 ++
 .../binary/compare/GreaterThanViewExpression.java  |  57 ++
 .../binary/compare/LessEqualViewExpression.java    |  57 ++
 .../binary/compare/LessThanViewExpression.java     |  57 ++
 .../binary/compare/NonEqualViewExpression.java     |  57 ++
 .../binary/logic/LogicAndViewExpression.java       |  57 ++
 .../binary/logic/LogicBinaryViewExpression.java    |  52 ++
 .../binary/logic/LogicOrViewExpression.java        |  57 ++
 .../viewExpression/leaf/ConstantViewOperand.java   |  98 +++
 .../view/viewExpression/leaf/LeafViewOperand.java} |  20 +-
 .../view/viewExpression/leaf/NullViewOperand.java  |  61 ++
 .../viewExpression/leaf/TimeSeriesViewOperand.java |  87 +++
 .../viewExpression/leaf/TimestampViewOperand.java  |  72 +++
 .../multi/FunctionViewExpression.java              | 198 +++++++
 .../ternary/BetweenViewExpression.java             | 111 ++++
 .../ternary/TernaryViewExpression.java             | 105 ++++
 .../viewExpression/unary/InViewExpression.java     | 106 ++++
 .../viewExpression/unary/IsNullViewExpression.java |  89 +++
 .../viewExpression/unary/LikeViewExpression.java   | 163 +++++
 .../unary/LogicNotViewExpression.java              |  61 ++
 .../unary/NegationViewExpression.java              |  61 ++
 .../unary/RegularViewExpression.java               | 105 ++++
 .../viewExpression/unary/UnaryViewExpression.java  |  73 +++
 .../visitor/GetSourcePathsVisitor.java             |  95 +++
 .../visitor/TransformToExpressionVisitor.java      | 320 ++++++++++
 .../visitor/ViewExpressionVisitor.java             | 219 +++++++
 .../metadata/visitor/SchemaExecutionVisitor.java   |  22 +
 .../db/mpp/common/header/ColumnHeaderConstant.java |   6 +-
 .../visitor/SchemaTreeDeviceVisitor.java           |   5 +
 .../visitor/SchemaTreeMeasurementVisitor.java      |   5 +
 .../SchemaTreeVisitorWithLimitOffsetWrapper.java   |   6 +
 .../iotdb/db/mpp/execution/driver/Driver.java      |  21 +
 .../db/mpp/execution/driver/DriverContext.java     |   9 +
 .../mpp/execution/exchange/SharedTsBlockQueue.java |  12 +-
 .../execution/executor/RegionWriteExecutor.java    |  49 ++
 .../fragment/FragmentInstanceContext.java          |  10 +
 .../fragment/FragmentInstanceExecution.java        |  17 +
 .../operator/process/DeviceViewIntoOperator.java   |   6 +-
 .../operator/process/MergeSortOperator.java        |   9 +-
 .../execution/operator/process/SortOperator.java   | 258 ++++++--
 .../process/join/merge/MergeSortComparator.java    |  45 +-
 ...rtKeyComparator.java => SortKeyComparator.java} |  11 +-
 .../schema/source/TimeSeriesSchemaSource.java      |  14 +-
 .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java  |  50 +-
 .../visitor/TransformToViewExpressionVisitor.java  | 391 ++++++++++++
 .../iotdb/db/mpp/plan/parser/ASTVisitor.java       | 100 +++-
 .../db/mpp/plan/planner/LogicalPlanBuilder.java    |   6 +-
 .../db/mpp/plan/planner/LogicalPlanVisitor.java    |  21 +
 .../db/mpp/plan/planner/OperatorTreeGenerator.java |  28 +-
 .../plan/planner/distribution/SourceRewriter.java  |   4 +-
 .../mpp/plan/planner/plan/node/PlanNodeType.java   |   6 +-
 .../db/mpp/plan/planner/plan/node/PlanVisitor.java |   5 +
 .../node/metedata/write/CreateLogicalViewNode.java | 250 ++++++++
 .../plan/planner/plan/node/write/InsertNode.java   |   8 +
 .../db/mpp/plan/scheduler/AsyncPlanNodeSender.java |  74 ++-
 .../plan/scheduler/AsyncSendPlanNodeHandler.java   |  27 +-
 .../scheduler/FragmentInstanceDispatcherImpl.java  |  20 +-
 .../iotdb/db/mpp/plan/statement/StatementType.java |   2 +
 .../db/mpp/plan/statement/StatementVisitor.java    |   7 +
 .../plan/statement/component/OrderByComponent.java |   6 +-
 .../component/{SortKey.java => OrderByKey.java}    |   2 +-
 .../db/mpp/plan/statement/crud/QueryStatement.java |   2 +-
 .../metadata/CreateLogicalViewStatement.java       | 246 ++++++++
 .../plan/statement/sys/ShowQueriesStatement.java   |   4 +-
 .../org/apache/iotdb/db/pipe/agent/PipeAgent.java  |   2 +-
 .../db/pipe/agent/plugin/PipePluginAgent.java      |  26 +-
 .../iotdb/db/pipe/agent/task/PipeTaskAgent.java    | 281 ++++++++-
 .../PipeCollectorConstant.java}                    |  20 +-
 .../PipeConnectorConstant.java}                    |  12 +-
 .../PipeProcessorConstant.java}                    |  12 +-
 .../core/collector/IoTDBDataRegionCollector.java   |  85 +++
 .../PipeHistoricalDataRegionTsFileCollector.java   | 103 ++++
 .../realtime/PipeRealtimeDataRegionCollector.java  |  21 +-
 ... => PipeRealtimeDataRegionHybridCollector.java} |  33 +-
 .../connector/PipeConnectorSubtaskLifeCycle.java   |  31 +-
 .../connector/PipeConnectorSubtaskManager.java     |  19 +-
 .../core/event/impl/PipeTsFileInsertionEvent.java  |   1 -
 .../event/view/collector/PipeEventCollector.java   |  59 +-
 ...anager.java => PipeSubtaskExecutorManager.java} |  12 +-
 .../execution/scheduler/PipeTaskScheduler.java     |  18 +-
 .../org/apache/iotdb/db/pipe/task/PipeBuilder.java |  66 +++
 .../org/apache/iotdb/db/pipe/task/PipeTask.java    |   9 +-
 .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java |  60 +-
 .../apache/iotdb/db/pipe/task/PipeTaskManager.java |  96 +++
 .../EventSupplier.java}                            |  24 +-
 .../ListenableBlockingPendingQueue.java}           |  18 +-
 .../db/pipe/task/queue/ListenablePendingQueue.java | 159 +++++
 .../ListenableUnblockingPendingQueue.java}         |  18 +-
 .../PendingQueueEmptyToNotEmptyListener.java}      |   8 +-
 .../queue/PendingQueueFullToNotFullListener.java}  |   8 +-
 .../PendingQueueNotEmptyToEmptyListener.java}      |   8 +-
 .../queue/PendingQueueNotFullToFullListener.java}  |   8 +-
 .../db/pipe/task/stage/PipeTaskCollectorStage.java |  83 ++-
 .../db/pipe/task/stage/PipeTaskConnectorStage.java | 100 +---
 .../db/pipe/task/stage/PipeTaskProcessorStage.java | 105 +++-
 .../iotdb/db/pipe/task/stage/PipeTaskStage.java    |  95 ++-
 .../db/pipe/task/subtask/PipeConnectorSubtask.java |  87 ++-
 .../db/pipe/task/subtask/PipeProcessorSubtask.java |  20 +-
 .../iotdb/db/pipe/task/subtask/PipeSubtask.java    |   6 +-
 .../org/apache/iotdb/db/rescon/SystemInfo.java     |  20 +-
 .../impl/DataNodeInternalRPCServiceImpl.java       |  68 +--
 .../org/apache/iotdb/db/tools/DiskSpiller.java     | 166 ++++++
 .../apache/iotdb/db/tools/FileSpillerReader.java   | 143 +++++
 .../MemoryReader.java}                             |  35 +-
 .../apache/iotdb/db/tools/SortBufferManager.java   |  73 +++
 .../org/apache/iotdb/db/tools/SortReader.java}     |  27 +-
 .../db/utils/datastructure/MergeSortHeap.java      |   4 +-
 .../iotdb/db/utils/datastructure/MergeSortKey.java |  25 +-
 .../{MergeSortKey.java => SortKey.java}            |  14 +-
 .../org/apache/iotdb/db/wal/buffer/WALBuffer.java  |   9 +-
 .../cache/dualkeycache/DualKeyCacheTest.java       |  17 +
 .../schemaRegion/SchemaRegionBasicTest.java        | 103 +++-
 .../metadata/view/ViewExpressionToStringTest.java  | 183 ++++++
 .../execution/operator/MergeSortOperatorTest.java  |  36 +-
 .../mpp/execution/operator/OperatorMemoryTest.java |  14 +-
 .../mpp/execution/operator/SortOperatorTest.java   | 273 +++++++++
 .../db/mpp/plan/optimization/TestPlanBuilder.java  |   6 +-
 .../db/mpp/plan/plan/QueryLogicalPlanUtil.java     |  16 +-
 .../plan/node/process/DeviceViewNodeSerdeTest.java |   6 +-
 .../plan/plan/node/process/SortNodeSerdeTest.java  |   4 +-
 .../collector/CachedSchemaPatternMatcherTest.java  |  48 +-
 .../core/collector/PipeRealtimeCollectTest.java    |  59 +-
 .../executor/PipeConnectorSubtaskExecutorTest.java |   8 +-
 .../executor/PipeProcessorSubtaskExecutorTest.java |  10 +-
 .../DataNodeInternalRPCServiceImplTest.java        |  31 +-
 .../sync/pipedata/BufferedPipeDataQueueTest.java   | 658 ---------------------
 thrift/src/main/thrift/datanode.thrift             |  35 +-
 271 files changed, 10279 insertions(+), 2197 deletions(-)


[iotdb] 01/04: Merge branch 'master' of github.com:apache/iotdb

Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

caogaofei pushed a commit to branch beyyes/optimize_tsbs
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit f3db0e129ce9c48dbd0b7f55f47a30cf4c6938c4
Merge: 2aa22287804 5fb6e2b76c6
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Apr 28 20:32:22 2023 +0800

    Merge branch 'master' of github.com:apache/iotdb

 .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4   |    1 +
 .../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4  |    8 +
 .../confignode/manager/ClusterSchemaManager.java   |   13 +-
 .../iotdb/confignode/manager/ConfigManager.java    |   11 +-
 .../consensus/iot/IoTConsensusServerImpl.java      |   13 +-
 .../db/it/last/IoTDBLastQueryLastCache2IT.java     |   60 +
 .../db/it/last/IoTDBLastQueryLastCacheIT.java      |  145 +++
 .../apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java | 1224 ++++++++++++++++++++
 .../iotdb/db/it/schema/IoTDBMetadataFetchIT.java   |   49 +
 .../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java  |   61 +
 .../iotdb/metrics/metricsets/net/NetMetrics.java   |   10 +-
 .../resources/conf/iotdb-common.properties         |   14 +-
 .../iotdb/commons/concurrent/ThreadName.java       |    2 +-
 .../org/apache/iotdb/commons/path/AlignedPath.java |    6 +
 .../org/apache/iotdb/commons/path/PartialPath.java |   40 +
 .../builtin/connector/DoNothingConnector.java      |    6 +-
 .../builtin/processor/DoNothingProcessor.java      |    6 +-
 .../commons/udf/service/UDFClassLoaderManager.java |    6 +-
 .../apache/iotdb/commons/utils/JVMCommonUtils.java |   12 +-
 .../org/apache/iotdb/pipe/api/PipeCollector.java   |   99 ++
 .../org/apache/iotdb/pipe/api/PipeConnector.java   |    6 +-
 .../org/apache/iotdb/pipe/api/PipeProcessor.java   |    6 +-
 .../iotdb/pipe/api/collector/EventCollector.java   |    6 +-
 .../iotdb/pipe/api/collector/RowCollector.java     |    2 +-
 .../PipeCollectorRuntimeConfiguration.java         |   13 +-
 .../org/apache/iotdb/pipe/api/event/Event.java     |    6 +-
 .../org/apache/iotdb/pipe/api/event/EventType.java |    8 +-
 .../event/{ => dml}/deletion/DeletionEvent.java    |    8 +-
 .../{ => dml}/insertion/TabletInsertionEvent.java  |    8 +-
 .../{ => dml}/insertion/TsFileInsertionEvent.java  |    8 +-
 pom.xml                                            |    6 +
 server/pom.xml                                     |    6 +-
 .../resources/conf/iotdb-datanode.properties       |    2 +-
 .../java/org/apache/iotdb/db/conf/IoTDBConfig.java |   10 +-
 .../directories/strategy/SequenceStrategy.java     |    2 +-
 .../performer/impl/FastCompactionPerformer.java    |    3 +-
 .../execute/task/AbstractCompactionTask.java       |    8 +-
 .../execute/task/CrossSpaceCompactionTask.java     |   11 +-
 .../execute/task/InnerSpaceCompactionTask.java     |    7 +-
 .../fast/AlignedSeriesCompactionExecutor.java      |   16 +-
 .../utils/writer/AbstractCompactionWriter.java     |    3 +-
 .../iotdb/db/engine/flush/MemTableFlushTask.java   |    2 +-
 .../db/engine/storagegroup/TsFileProcessor.java    |   17 +-
 .../metadata/template/ClusterTemplateManager.java  |   17 +-
 .../db/metadata/template/ITemplateManager.java     |    3 +-
 .../apache/iotdb/db/metadata/utils/MetaUtils.java  |   11 +-
 .../iotdb/db/mpp/execution/driver/Driver.java      |    2 +-
 .../execution/exchange/sink/LocalSinkChannel.java  |   12 +-
 .../mpp/execution/exchange/sink/SinkChannel.java   |   15 +-
 .../execution/operator/process/SortOperator.java   |   28 +-
 .../operator/process/TransformOperator.java        |   13 +-
 .../process/join/merge/MergeSortComparator.java    |  178 +--
 .../process/join/merge/MergeSortKeyComparator.java |   53 +
 .../execution/schedule/AbstractDriverThread.java   |    9 +
 .../schedule/queue/IndexedBlockingQueue.java       |    1 +
 .../multilevelqueue/MultilevelPriorityQueue.java   |    4 +-
 .../apache/iotdb/db/mpp/plan/analyze/Analysis.java |   68 ++
 .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java  |  138 ++-
 .../db/mpp/plan/analyze/ConcatPathRewriter.java    |   22 +-
 .../db/mpp/plan/execution/QueryExecution.java      |   10 +-
 .../visitor/ColumnTransformerVisitor.java          |   40 +-
 .../visitor/IntermediateLayerVisitor.java          |    4 +-
 .../iotdb/db/mpp/plan/parser/ASTVisitor.java       |   44 +-
 .../db/mpp/plan/planner/LogicalPlanBuilder.java    |  143 ++-
 .../db/mpp/plan/planner/LogicalPlanVisitor.java    |   19 +-
 .../db/mpp/plan/planner/OperatorTreeGenerator.java |   12 +-
 .../planner/distribution/DistributionPlanner.java  |    5 +-
 .../planner/distribution/ExchangeNodeAdder.java    |    6 +
 .../plan/planner/distribution/SourceRewriter.java  |   48 +
 .../planner/plan/node/write/InsertRowNode.java     |    4 +
 .../planner/plan/node/write/InsertRowsNode.java    |    5 +
 .../plan/node/write/InsertRowsOfOneDeviceNode.java |    7 +
 .../planner/plan/node/write/InsertTabletNode.java  |    8 +
 .../component/{SortKey.java => NullOrdering.java}  |   19 +-
 .../plan/statement/component/OrderByComponent.java |   56 +-
 .../db/mpp/plan/statement/component/SortItem.java  |   61 +-
 .../db/mpp/plan/statement/component/SortKey.java   |   16 +-
 .../plan/statement/crud/InsertBaseStatement.java   |    4 -
 .../crud/InsertMultiTabletsStatement.java          |   18 -
 .../plan/statement/crud/InsertRowStatement.java    |   17 +-
 .../crud/InsertRowsOfOneDeviceStatement.java       |    9 -
 .../plan/statement/crud/InsertRowsStatement.java   |   17 -
 .../plan/statement/crud/InsertTabletStatement.java |   13 -
 .../db/mpp/plan/statement/crud/QueryStatement.java |   80 ++
 .../dag/builder/EvaluationDAGBuilder.java          |    4 +-
 .../dag/input/QueryDataSetInputLayer.java          |    5 +-
 .../intermediate/ConstantIntermediateLayer.java    |    3 +-
 .../dag/intermediate/IntermediateLayer.java        |    4 +-
 .../MultiInputColumnIntermediateLayer.java         |    2 +-
 ...InputColumnMultiReferenceIntermediateLayer.java |    2 +-
 ...nputColumnSingleReferenceIntermediateLayer.java |    2 +-
 .../db/mpp/transformation/dag/udf/UDTFContext.java |    2 +-
 .../mpp/transformation/dag/udf/UDTFExecutor.java   |    2 +-
 .../datastructure/SerializableList.java            |    6 +-
 .../row/ElasticSerializableRowRecordList.java      |    6 +-
 .../row/SerializableRowRecordList.java             |    2 +-
 .../tv/ElasticSerializableBinaryTVList.java        |    2 +-
 .../tv/ElasticSerializableTVList.java              |    8 +-
 .../datastructure/tv/SerializableTVList.java       |    2 +-
 .../apache/iotdb/db/pipe/config/PipeConfig.java    |   65 ++
 .../collector/realtime/PipeRealtimeCollector.java  |   22 -
 .../realtime/PipeRealtimeDataRegionCollector.java  |   81 ++
 .../PipeRealtimeHybridDataRegionCollector.java     |  171 +++
 .../realtime/assigner/DisruptorQueue.java          |  111 ++
 .../realtime/assigner/PipeDataRegionAssigner.java  |   74 ++
 .../realtime/cache/PipeRealtimeEventCache.java     |   22 -
 .../realtime/listener/IoTLogListerner.java         |   22 -
 .../listener/PipeInsertionDataNodeListener.java    |  115 ++
 .../realtime/listener/RatisLogListener.java        |   22 -
 .../realtime/listener/SimpleLogListener.java       |   22 -
 .../listener/TsFileGenerationListener.java         |   22 -
 .../matcher/CachedSchemaPatternMatcher.java        |  200 ++++
 .../realtime/matcher/PipeDataRegionMatcher.java    |   47 +
 .../pipe/core/collector/realtime/matcher/Rule.java |   22 -
 .../realtime/matcher/RulePrefixMatchTree.java      |   22 -
 .../collector/realtime/recorder/TsFileEpoch.java   |   22 -
 .../realtime/recorder/TsFileEpochRecorder.java     |   22 -
 .../event/{ => impl}/PipeTabletInsertionEvent.java |   22 +-
 .../event/{ => impl}/PipeTsFileInsertionEvent.java |   23 +-
 .../pipe/core/event/indexer/PipeEventIndexer.java  |   22 -
 .../core/event/indexer/PipeIoTEventIndexer.java    |   22 -
 .../core/event/indexer/PipeRatisEventIndexer.java  |   22 -
 .../core/event/indexer/PipeSimpleEventIndexer.java |   22 -
 .../core/event/indexer/PipeTsFileEventIndexer.java |   22 -
 .../event/realtime/PipeRealtimeCollectEvent.java   |   66 ++
 .../realtime/PipeRealtimeCollectEventFactory.java  |   49 +
 .../db/pipe/core/event/realtime/TsFileEpoch.java   |   68 ++
 .../core/event/realtime/TsFileEpochManager.java    |   70 ++
 .../realtime/TsFileEpochStateMigrator.java}        |    7 +-
 .../pipe/core/event/{ => view}/access/PipeRow.java |    2 +-
 .../event/{ => view}/access/PipeRowIterator.java   |    2 +-
 .../{ => view}/collector/PipeEventCollector.java   |    8 +-
 .../{ => view}/collector/PipeRowCollector.java     |    2 +-
 .../PipeResourceManager.java}                      |   29 +-
 .../db/pipe/task/subtask/PipeConnectorSubtask.java |    6 +-
 .../db/pipe/task/subtask/PipeProcessorSubtask.java |    6 +-
 .../rest/v1/handler/ExecuteStatementHandler.java   |    4 +-
 .../rest/v1/handler/QueryDataSetHandler.java       |    5 +-
 .../rest/v2/handler/ExecuteStatementHandler.java   |    4 +-
 .../rest/v2/handler/QueryDataSetHandler.java       |    5 +-
 .../db/query/control/QueryResourceManager.java     |    7 +-
 .../db/service/TemporaryQueryDataFileService.java  |   14 +-
 .../db/service/metrics/CompactionMetrics.java      |   12 +-
 .../metrics/recorder/CompactionMetricsManager.java |    6 +-
 .../db/service/thrift/ProcessorWithMetrics.java    |    2 +-
 .../service/thrift/impl/ClientRPCServiceImpl.java  |    2 +-
 .../java/org/apache/iotdb/db/wal/WALManager.java   |    8 +-
 .../db/wal/allocation/FirstCreateStrategy.java     |    3 +
 .../iotdb/db/wal/buffer/AbstractWALBuffer.java     |   41 +-
 .../java/org/apache/iotdb/db/wal/node/WALNode.java |   14 +-
 .../iotdb/db/wal/recover/WALNodeRecoverTask.java   |    6 -
 .../engine/compaction/AbstractCompactionTest.java  |    3 +
 .../compaction/CompactionTaskComparatorTest.java   |    8 +-
 .../compaction/FastAlignedCrossCompactionTest.java |  347 +++++-
 .../FastInnerCompactionPerformerTest.java          |   30 +-
 .../FastNonAlignedCrossCompactionTest.java         |  347 +++++-
 .../ReadPointAlignedCrossCompactionTest.java       |  347 +++++-
 .../ReadPointNonAlignedCrossCompactionTest.java    |  347 +++++-
 .../CrossSpaceCompactionWithFastPerformerTest.java |    7 +-
 ...eCompactionWithFastPerformerValidationTest.java |  194 ++--
 ...actionWithReadPointPerformerValidationTest.java |  168 +--
 .../utils/MultiTsFileDeviceIteratorTest.java       |   12 +-
 .../apache/iotdb/db/metric/MetricServiceTest.java  |    3 -
 .../execution/operator/MergeSortOperatorTest.java  |   20 +-
 .../iotdb/db/mpp/plan/analyze/AnalyzeTest.java     |  200 ++++
 .../plan/node/write/WritePlanNodeSplitTest.java    |   15 +-
 .../collector/CachedSchemaPatternMatcherTest.java  |  149 +++
 .../core/collector/PipeRealtimeCollectTest.java    |  236 ++++
 .../udf/datastructure/SerializableListTest.java    |    2 +-
 server/src/test/resources/logback-test.xml         |    1 +
 .../tsfile/file/metadata/enums/TSDataType.java     |   17 +
 .../iotdb/tsfile/write/chunk/ValueChunkWriter.java |    4 -
 172 files changed, 6166 insertions(+), 1219 deletions(-)