You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/04/21 02:15:23 UTC
[iotdb] branch ISSUE_5792 updated (a53832e3a5 -> c22dbf7058)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch ISSUE_5792
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from a53832e3a5 refinement
add e12afecd6e [IOTDB-5735] Fix result of some UDF with align by device is incorrect (#9642)
add b02502fa14 [IOTDB-4837] Refactor Set Schema Template (#9637)
add f13699611f [IOTDB-5798] Fix concurrent problem when sinkChannel acknowledgeTsBlock() and close()
add 98adb10391 [IOTDB-5773] fix set space quota problem (#9596)
add 348763f208 Add a cache to avoid initialize duplicated device id object in write process (#9644)
add 9870f332d6 Resolve quadratic complexity issue when flushing numerous Internal/Entity nodes in SchemaFile (#9659)
add 3405c49177 Accelerate count all schema via schema statistic (#9660)
add bc9ee590ed [IOTDB-5720] Fix release processor fail to release memory due to writer-preferred starvation (#9661)
add c19bcdc5f1 Double-check protect memory-control boundary to avoid concurrent bug (#9662)
add 4a822f2da1 TemplateSchemaCache in schema fetcher (#9664)
add e15a3c770b [IOTDB-5774] Fix the syntax that path nodes start or end with a wildcard to fuzzy match is not supported (#9600)
new c22dbf7058 Merge branch 'master' into ISSUE_5792
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 10 +-
.../org/apache/iotdb/db/qp/sql/PathParser.g4 | 14 +-
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 8 +
.../client/async/handlers/AsyncClientHandler.java | 10 +
...ava => CheckTimeSeriesExistenceRPCHandler.java} | 22 +-
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
...ePlan.java => CommitSetSchemaTemplatePlan.java} | 25 +-
...latePlan.java => PreSetSchemaTemplatePlan.java} | 25 +-
.../confignode/manager/ClusterQuotaManager.java | 18 +-
.../confignode/manager/ClusterSchemaManager.java | 84 +---
.../iotdb/confignode/manager/ConfigManager.java | 6 +-
.../iotdb/confignode/manager/ProcedureManager.java | 44 ++
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../confignode/persistence/quota/QuotaInfo.java | 36 +-
.../persistence/schema/ClusterSchemaInfo.java | 96 +++-
.../persistence/schema/TemplatePreSetTable.java | 195 ++++++++
.../persistence/schema/TemplateTable.java | 1 -
.../impl/schema/SetTemplateProcedure.java | 534 ++++++++++++++++++++
...ateTemplateState.java => SetTemplateState.java} | 12 +-
.../procedure/store/ProcedureFactory.java | 6 +
.../confignode/procedure/store/ProcedureType.java | 1 +
.../{ => schema}/ClusterSchemaInfoTest.java | 3 +-
.../schema/TemplatePreSetTableTest.java | 114 +++++
.../{ => schema}/TemplateTableTest.java | 3 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 11 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../iotdb/it/env/cluster/MppCommonConfig.java | 6 +
.../it/env/cluster/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 40 ++
.../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java | 535 +++++++++++++++++++++
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 3 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 160 +++++-
.../it/udf/IoTDBUDTFBuiltinScalarFunctionIT.java | 499 +++++++++++++++++--
.../iotdb/libudf/it/dprofile/DProfileIT.java | 17 +
.../session/it/IoTDBSessionSchemaTemplateIT.java | 41 ++
.../resources/conf/iotdb-common.properties | 3 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 32 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../BuiltinTimeSeriesGeneratingFunction.java | 10 +
.../apache/iotdb/commons/path/PartialPathTest.java | 2 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 2 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 8 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +
.../iotdb/db/engine/storagegroup/DataRegion.java | 3 +
.../template/TemplateImcompatibeException.java | 19 +-
.../db/metadata/cache/DataNodeDevicePathCache.java | 69 +++
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../cache/DataNodeTemplateSchemaCache.java | 95 ++++
.../iotdb/db/metadata/mtree/ConfigMTree.java | 4 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 8 -
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 8 -
.../mtree/store/disk/cache/CacheMemoryManager.java | 4 +-
.../store/disk/schemafile/SchemaFileConfig.java | 2 +-
.../mtree/store/disk/schemafile/SegmentedPage.java | 1 +
.../store/disk/schemafile/pagemgr/PageManager.java | 90 +++-
.../metadata/rescon/MemSchemaEngineStatistics.java | 26 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 2 +-
.../db/metadata/schemaregion/SchemaEngine.java | 8 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 2 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 17 +-
.../metadata/template/ClusterTemplateManager.java | 242 +++++++++-
.../db/metadata/template/ITemplateManager.java | 7 +-
.../template/TemplateInternalRPCUpdateType.java | 17 +-
.../metadata/template/TemplateInternalRPCUtil.java | 40 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 23 +-
.../operator/schema/SchemaCountOperator.java | 23 +-
.../operator/schema/SchemaQueryScanOperator.java | 2 +-
.../operator/schema/source/DeviceSchemaSource.java | 12 +
.../operator/schema/source/ISchemaSource.java | 4 +
.../operator/schema/source/NodeSchemaSource.java | 10 +
.../schema/source/PathsUsingTemplateSource.java | 10 +
.../schema/source/SchemaSourceFactory.java | 5 +
.../schema/source/TimeSeriesSchemaSource.java | 12 +
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 65 +--
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 29 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 157 +++++-
.../db/mpp/plan/analyze/schema/ISchemaFetcher.java | 5 +-
.../plan/execution/config/ConfigTaskVisitor.java | 2 +-
.../config/executor/ClusterConfigTaskExecutor.java | 12 +-
.../config/executor/IConfigTaskExecutor.java | 2 +-
.../metadata/template/SetSchemaTemplateTask.java | 7 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 7 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 34 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 88 ++--
.../db/mpp/plan/parser/StatementGenerator.java | 25 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 5 +-
.../v1/handler/StatementConstructionHandler.java | 5 +-
.../v2/handler/StatementConstructionHandler.java | 5 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 115 ++++-
.../cache/DataNodeDevicePathCacheTest.java} | 44 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 137 ++++--
.../schemaRegion/SchemaRegionManagementTest.java | 4 +-
.../schema/SchemaQueryScanOperatorTest.java | 4 +-
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 8 +-
.../iotdb/db/mpp/plan/plan/distribution/Util.java | 8 +-
.../src/main/thrift/confignode.thrift | 5 +-
thrift/src/main/thrift/datanode.thrift | 12 +
103 files changed, 3711 insertions(+), 542 deletions(-)
copy confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/{CountPathsUsingTemplateRPCHandler.java => CheckTimeSeriesExistenceRPCHandler.java} (79%)
copy confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/{SetSchemaTemplatePlan.java => CommitSetSchemaTemplatePlan.java} (71%)
copy confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/template/{SetSchemaTemplatePlan.java => PreSetSchemaTemplatePlan.java} (71%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTable.java
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java
copy confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/{DeactivateTemplateState.java => SetTemplateState.java} (85%)
rename confignode/src/test/java/org/apache/iotdb/confignode/persistence/{ => schema}/ClusterSchemaInfoTest.java (98%)
create mode 100644 confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/TemplatePreSetTableTest.java
rename confignode/src/test/java/org/apache/iotdb/confignode/persistence/{ => schema}/TemplateTableTest.java (97%)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeDevicePathCache.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeTemplateSchemaCache.java
copy server/src/test/java/org/apache/iotdb/db/{wal/DisableWALTest.java => metadata/cache/DataNodeDevicePathCacheTest.java} (50%)
[iotdb] 01/01: Merge branch 'master' into ISSUE_5792
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch ISSUE_5792
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c22dbf70588a444d8f477df41e09cb00b3396261
Merge: a53832e3a5 e15a3c770b
Author: Tian Jiang <jt...@163.com>
AuthorDate: Fri Apr 21 10:17:51 2023 +0800
Merge branch 'master' into ISSUE_5792
# Conflicts:
# node-commons/src/assembly/resources/conf/iotdb-common.properties
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 10 +-
.../org/apache/iotdb/db/qp/sql/PathParser.g4 | 14 +-
.../confignode/client/DataNodeRequestType.java | 1 +
.../client/async/AsyncDataNodeClientPool.java | 8 +
.../client/async/handlers/AsyncClientHandler.java | 10 +
.../rpc/CheckTimeSeriesExistenceRPCHandler.java | 87 ++++
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../template/CommitSetSchemaTemplatePlan.java | 80 +++
.../write/template/PreSetSchemaTemplatePlan.java | 80 +++
.../confignode/manager/ClusterQuotaManager.java | 18 +-
.../confignode/manager/ClusterSchemaManager.java | 84 +---
.../iotdb/confignode/manager/ConfigManager.java | 6 +-
.../iotdb/confignode/manager/ProcedureManager.java | 44 ++
.../persistence/executor/ConfigPlanExecutor.java | 7 +
.../confignode/persistence/quota/QuotaInfo.java | 36 +-
.../persistence/schema/ClusterSchemaInfo.java | 96 +++-
.../persistence/schema/TemplatePreSetTable.java | 195 ++++++++
.../persistence/schema/TemplateTable.java | 1 -
.../impl/schema/SetTemplateProcedure.java | 534 ++++++++++++++++++++
.../procedure/state/schema/SetTemplateState.java | 43 +-
.../procedure/store/ProcedureFactory.java | 6 +
.../confignode/procedure/store/ProcedureType.java | 1 +
.../{ => schema}/ClusterSchemaInfoTest.java | 3 +-
.../schema/TemplatePreSetTableTest.java | 114 +++++
.../{ => schema}/TemplateTableTest.java | 3 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 11 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +
.../iotdb/it/env/cluster/MppCommonConfig.java | 6 +
.../it/env/cluster/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 40 ++
.../iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java | 535 +++++++++++++++++++++
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 3 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 160 +++++-
.../it/udf/IoTDBUDTFBuiltinScalarFunctionIT.java | 499 +++++++++++++++++--
.../iotdb/libudf/it/dprofile/DProfileIT.java | 17 +
.../session/it/IoTDBSessionSchemaTemplateIT.java | 41 ++
.../resources/conf/iotdb-common.properties | 3 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 32 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 4 +-
.../BuiltinTimeSeriesGeneratingFunction.java | 10 +
.../apache/iotdb/commons/path/PartialPathTest.java | 2 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 2 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 8 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +
.../iotdb/db/engine/storagegroup/DataRegion.java | 3 +
.../template/TemplateImcompatibeException.java | 19 +-
.../db/metadata/cache/DataNodeDevicePathCache.java | 69 +++
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../cache/DataNodeTemplateSchemaCache.java | 95 ++++
.../iotdb/db/metadata/mtree/ConfigMTree.java | 4 +
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 8 -
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 8 -
.../mtree/store/disk/cache/CacheMemoryManager.java | 4 +-
.../store/disk/schemafile/SchemaFileConfig.java | 2 +-
.../mtree/store/disk/schemafile/SegmentedPage.java | 1 +
.../store/disk/schemafile/pagemgr/PageManager.java | 90 +++-
.../metadata/rescon/MemSchemaEngineStatistics.java | 26 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 2 +-
.../db/metadata/schemaregion/SchemaEngine.java | 8 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 2 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 17 +-
.../metadata/template/ClusterTemplateManager.java | 242 +++++++++-
.../db/metadata/template/ITemplateManager.java | 7 +-
.../template/TemplateInternalRPCUpdateType.java | 17 +-
.../metadata/template/TemplateInternalRPCUtil.java | 40 +-
.../mpp/execution/exchange/sink/SinkChannel.java | 23 +-
.../operator/schema/SchemaCountOperator.java | 23 +-
.../operator/schema/SchemaQueryScanOperator.java | 2 +-
.../operator/schema/source/DeviceSchemaSource.java | 12 +
.../operator/schema/source/ISchemaSource.java | 4 +
.../operator/schema/source/NodeSchemaSource.java | 10 +
.../schema/source/PathsUsingTemplateSource.java | 10 +
.../schema/source/SchemaSourceFactory.java | 5 +
.../schema/source/TimeSeriesSchemaSource.java | 12 +
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 65 +--
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 29 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 157 +++++-
.../db/mpp/plan/analyze/schema/ISchemaFetcher.java | 5 +-
.../plan/execution/config/ConfigTaskVisitor.java | 2 +-
.../config/executor/ClusterConfigTaskExecutor.java | 12 +-
.../config/executor/IConfigTaskExecutor.java | 2 +-
.../metadata/template/SetSchemaTemplateTask.java | 7 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 7 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 34 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 88 ++--
.../db/mpp/plan/parser/StatementGenerator.java | 25 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 5 +-
.../v1/handler/StatementConstructionHandler.java | 5 +-
.../v2/handler/StatementConstructionHandler.java | 5 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 115 ++++-
.../cache/DataNodeDevicePathCacheTest.java | 62 +++
.../metadata/mtree/schemafile/SchemaFileTest.java | 137 ++++--
.../schemaRegion/SchemaRegionManagementTest.java | 4 +-
.../schema/SchemaQueryScanOperatorTest.java | 4 +-
.../db/mpp/plan/analyze/FakeSchemaFetcherImpl.java | 8 +-
.../iotdb/db/mpp/plan/plan/distribution/Util.java | 8 +-
.../src/main/thrift/confignode.thrift | 5 +-
thrift/src/main/thrift/datanode.thrift | 12 +
103 files changed, 3946 insertions(+), 531 deletions(-)
diff --cc node-commons/src/assembly/resources/conf/iotdb-common.properties
index 2e5b787964,61b0bbc4a7..32dccb40b9
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@@ -528,13 -528,10 +528,16 @@@ cluster_name=defaultCluste
# Datatype: int
# upgrade_thread_count=1
+ # The max size of the device path cache. This cache is for avoiding initialize duplicated device id object in write process.
+ # Datatype: int
+ # device_path_cache_size=500000
+# When flushing a MemTable, the range of thread number that will be available for each pipeline state.
+# Set to 1 when less than or equal to 0.
+# Datatype: int
+# flush_min_sub_thread_num = 1
+# flush_max_sub_thread_num = 16
+
####################
### Compaction Configurations
####################