You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/04/24 01:51:40 UTC
[iotdb] 01/01: Merge branch 'ISSUE_5792' into native_raft
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch native_raft
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 34ff352fe51b6ccab5699d7b3c26b89a11b04471
Merge: e6c647b43d 64fd6d49a2
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Apr 24 09:54:05 2023 +0800
Merge branch 'ISSUE_5792' into native_raft
# Conflicts:
# client-go
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 15 +-
.../org/apache/iotdb/db/qp/sql/PathParser.g4 | 14 +-
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 8 +
.../client/async/handlers/AsyncClientHandler.java | 10 +
.../rpc/CheckTimeSeriesExistenceRPCHandler.java | 87 ++++
.../consensus/request/ConfigPhysicalPlan.java | 12 +
.../consensus/request/ConfigPhysicalPlanType.java | 3 +
.../template/CommitSetSchemaTemplatePlan.java | 80 +++
.../write/template/ExtendSchemaTemplatePlan.java | 58 +++
.../write/template/PreSetSchemaTemplatePlan.java | 80 +++
.../confignode/manager/ClusterQuotaManager.java | 18 +-
.../confignode/manager/ClusterSchemaManager.java | 155 +++---
.../iotdb/confignode/manager/ConfigManager.java | 26 +-
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ProcedureManager.java | 44 ++
.../persistence/executor/ConfigPlanExecutor.java | 10 +
.../confignode/persistence/quota/QuotaInfo.java | 36 +-
.../persistence/schema/ClusterSchemaInfo.java | 108 ++++-
.../persistence/schema/TemplatePreSetTable.java | 195 ++++++++
.../persistence/schema/TemplateTable.java | 48 +-
.../impl/schema/SetTemplateProcedure.java | 534 ++++++++++++++++++++
.../procedure/state/schema/SetTemplateState.java | 43 +-
.../procedure/store/ProcedureFactory.java | 6 +
.../confignode/procedure/store/ProcedureType.java | 1 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../{ => schema}/ClusterSchemaInfoTest.java | 3 +-
.../schema/TemplatePreSetTableTest.java | 114 +++++
.../{ => schema}/TemplateTableTest.java | 3 +-
docs/UserGuide/Operate-Metadata/Template.md | 15 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 49 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 11 +-
docs/zh/UserGuide/Operate-Metadata/Template.md | 14 +
.../UserGuide/QuickStart/Command-Line-Interface.md | 48 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +
grafana-plugin/README.md | 63 +--
grafana-plugin/src/plugin.json | 2 +-
.../iotdb/it/env/cluster/MppCommonConfig.java | 6 +
.../it/env/cluster/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 40 ++
.../iotdb/db/it/query/IoTDBPaginationIT.java | 59 ++-
.../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java | 535 +++++++++++++++++++++
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 3 +-
.../iotdb/db/it/schema/IoTDBExtendTemplateIT.java | 222 +++++++++
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 164 ++++++-
.../it/udf/IoTDBUDTFBuiltinScalarFunctionIT.java | 499 +++++++++++++++++--
.../iotdb/libudf/it/dprofile/DProfileIT.java | 17 +
.../session/it/IoTDBSessionSchemaTemplateIT.java | 103 ++++
.../resources/conf/iotdb-common.properties | 24 +
.../commons/concurrent/dynamic/DynamicThread.java | 115 +++++
.../concurrent/dynamic/DynamicThreadGroup.java | 110 +++++
.../iotdb/commons/concurrent/pipeline/Task.java | 46 +-
.../commons/concurrent/pipeline/TaskRunner.java | 73 +++
.../threadpool/WrappedThreadPoolExecutor.java | 2 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 32 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../BuiltinTimeSeriesGeneratingFunction.java | 10 +
.../apache/iotdb/commons/path/PartialPathTest.java | 2 +-
pom.xml | 2 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 2 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 8 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 22 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 84 +++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 37 ++
.../estimator/AbstractCompactionEstimator.java | 6 +
.../ReadPointCrossCompactionEstimator.java | 108 +++--
.../impl/RewriteCrossSpaceCompactionSelector.java | 13 +-
.../utils/CrossCompactionTaskResource.java | 12 +
.../iotdb/db/engine/flush/MemTableFlushTask.java | 338 ++++++-------
.../iotdb/db/engine/flush/tasks/DeviceIOTask.java | 93 ++++
.../db/engine/flush/tasks/EncodeSeriesTask.java | 96 ++++
.../iotdb/db/engine/flush/tasks/FinalTask.java | 43 +-
.../iotdb/db/engine/flush/tasks/FlushContext.java | 90 ++++
.../db/engine/flush/tasks/FlushDeviceContext.java | 74 +++
.../db/engine/flush/tasks/SortSeriesTask.java | 76 +++
.../template/TemplateImcompatibeException.java | 19 +-
.../db/metadata/cache/DataNodeDevicePathCache.java | 69 +++
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../cache/DataNodeTemplateSchemaCache.java | 196 ++++++++
.../iotdb/db/metadata/mtree/ConfigMTree.java | 4 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 8 -
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 8 -
.../mtree/store/disk/cache/CacheMemoryManager.java | 4 +-
.../store/disk/schemafile/SchemaFileConfig.java | 2 +-
.../mtree/store/disk/schemafile/SegmentedPage.java | 1 +
.../store/disk/schemafile/pagemgr/PageManager.java | 90 +++-
.../metadata/rescon/MemSchemaEngineStatistics.java | 26 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 2 +-
.../db/metadata/schemaregion/SchemaEngine.java | 8 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 2 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 17 +-
.../metadata/template/ClusterTemplateManager.java | 253 +++++++++-
.../db/metadata/template/ITemplateManager.java | 7 +-
.../iotdb/db/metadata/template/Template.java | 16 +-
...teType.java => TemplateAlterOperationType.java} | 24 +-
.../template/TemplateInternalRPCUpdateType.java | 20 +-
.../metadata/template/TemplateInternalRPCUtil.java | 50 +-
.../metadata/template/alter/TemplateAlterInfo.java | 49 +-
.../template/alter/TemplateAlterOperationUtil.java | 51 ++
.../template/alter/TemplateExtendInfo.java | 161 +++++++
.../mpp/execution/exchange/sink/SinkChannel.java | 29 +-
.../operator/schema/SchemaCountOperator.java | 23 +-
.../operator/schema/SchemaQueryScanOperator.java | 2 +-
.../operator/schema/source/DeviceSchemaSource.java | 12 +
.../operator/schema/source/ISchemaSource.java | 4 +
.../operator/schema/source/NodeSchemaSource.java | 10 +
.../schema/source/PathsUsingTemplateSource.java | 10 +
.../schema/source/SchemaSourceFactory.java | 5 +
.../schema/source/TimeSeriesSchemaSource.java | 12 +
.../execution/operator/source/SeriesScanUtil.java | 24 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 116 +++--
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 69 +--
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 40 +-
.../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 3 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 254 ++++++++--
.../plan/analyze/schema/ClusterSchemaFetcher.java | 237 ++++-----
.../db/mpp/plan/analyze/schema/ISchemaFetcher.java | 5 +-
.../plan/analyze/schema/NormalSchemaFetcher.java | 202 ++++++++
.../plan/analyze/schema/TemplateSchemaFetcher.java | 246 ++++++++++
.../plan/execution/config/ConfigTaskVisitor.java | 10 +-
.../config/executor/ClusterConfigTaskExecutor.java | 58 ++-
.../config/executor/IConfigTaskExecutor.java | 6 +-
...plateTask.java => AlterSchemaTemplateTask.java} | 15 +-
.../metadata/template/SetSchemaTemplateTask.java | 7 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 7 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 34 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 115 +++--
.../db/mpp/plan/parser/StatementGenerator.java | 25 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 8 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 12 +-
.../source/AlignedSeriesAggregationScanNode.java | 50 +-
.../plan/node/source/AlignedSeriesScanNode.java | 4 +
.../node/source/SeriesAggregationScanNode.java | 50 +-
.../node/source/SeriesAggregationSourceNode.java | 49 +-
.../planner/plan/node/source/SeriesScanNode.java | 4 +
.../planner/plan/parameter/SeriesScanOptions.java | 5 +-
.../iotdb/db/mpp/plan/statement/StatementType.java | 1 +
.../db/mpp/plan/statement/StatementVisitor.java | 6 +
.../template/AlterSchemaTemplateStatement.java | 85 ++++
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 5 +-
.../v1/handler/StatementConstructionHandler.java | 5 +-
.../v2/handler/StatementConstructionHandler.java | 5 +-
.../iotdb/db/query/reader/chunk/MemPageReader.java | 12 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 15 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 129 ++++-
.../cross/RewriteCompactionFileSelectorTest.java | 12 +
.../cache/DataNodeDevicePathCacheTest.java | 62 +++
.../metadata/mtree/schemafile/SchemaFileTest.java | 137 ++++--
.../schemaRegion/SchemaRegionManagementTest.java | 4 +-
.../schema/SchemaQueryScanOperatorTest.java | 4 +-
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 8 +-
.../mpp/plan/analyze/QueryTimePartitionTest.java | 14 +-
.../iotdb/db/mpp/plan/plan/distribution/Util.java | 8 +-
.../node/process/AggregationNodeSerdeTest.java | 7 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 8 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 7 +-
site/src/main/.vuepress/styles/index.scss | 6 +-
.../src/main/thrift/confignode.thrift | 12 +-
thrift/src/main/thrift/datanode.thrift | 12 +
.../iotdb/tsfile/read/filter/GroupByFilter.java | 13 +-
.../read/filter/PredicateRemoveNotRewriter.java | 50 ++
.../iotdb/tsfile/read/filter/TimeFilter.java | 158 +++---
.../iotdb/tsfile/read/filter/ValueFilter.java | 179 ++++---
.../tsfile/read/filter/basic/BinaryFilter.java | 5 -
.../iotdb/tsfile/read/filter/basic/Filter.java | 11 +-
.../tsfile/read/filter/factory/FilterFactory.java | 10 +-
.../tsfile/read/filter/operator/AndFilter.java | 11 +
.../iotdb/tsfile/read/filter/operator/Between.java | 127 +++--
.../iotdb/tsfile/read/filter/operator/Eq.java | 25 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 23 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 23 +-
.../iotdb/tsfile/read/filter/operator/In.java | 34 +-
.../iotdb/tsfile/read/filter/operator/Like.java | 55 ++-
.../iotdb/tsfile/read/filter/operator/Lt.java | 23 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 23 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 25 +-
.../tsfile/read/filter/operator/NotFilter.java | 26 +-
.../tsfile/read/filter/operator/OrFilter.java | 11 +
.../iotdb/tsfile/read/filter/operator/Regexp.java | 57 ++-
.../iotdb/tsfile/read/reader/page/PageReader.java | 12 +-
.../tsfile/read/filter/FilterSerializeTest.java | 21 +-
.../read/filter/MinTimeMaxTimeFilterTest.java | 42 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 4 +-
.../filter/PredicateRemoveNotRewriterTest.java | 121 +++++
.../tsfile/read/filter/StatisticsFilterTest.java | 113 ++++-
189 files changed, 8064 insertions(+), 1676 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index 99f00997ce,f29b8016e5..1136f30fe5
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@@ -286,11 -292,11 +292,12 @@@ public class StatementGenerator
final long startTime = System.nanoTime();
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
- insertStatement.setDevicePath(new PartialPath(insertTabletReq.getPrefixPath()));
+ insertStatement.setDevicePath(
+ DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath()));
insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new String[0]));
insertStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size));
+ QueryDataSetUtils.readTimesFromBuffer(
+ insertTabletReq.timestamps, insertTabletReq.size, insertTabletReq.compression));
insertStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
insertTabletReq.values,
@@@ -320,11 -325,10 +327,11 @@@
List<InsertTabletStatement> insertTabletStatementList = new ArrayList<>();
for (int i = 0; i < req.prefixPaths.size(); i++) {
InsertTabletStatement insertTabletStatement = new InsertTabletStatement();
- insertTabletStatement.setDevicePath(new PartialPath(req.prefixPaths.get(i)));
+ insertTabletStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPaths.get(i)));
insertTabletStatement.setMeasurements(req.measurementsList.get(i).toArray(new String[0]));
insertTabletStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i)));
+ QueryDataSetUtils.readTimesFromBuffer(
+ req.timestampsList.get(i), req.sizeList.get(i), req.compression));
insertTabletStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
req.valuesList.get(i),