You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/02/23 08:46:47 UTC
[iotdb] 01/04: Merge branch 'master' into ConfigNode-metric
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch ConfigNode-metric
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f0efcba8f65a55396f9c02fa552bcd0a7ea9b8fb
Merge: 72c1208cf9 16c1c9efab
Author: YongzaoDan <53...@qq.com>
AuthorDate: Wed Feb 22 15:05:15 2023 +0800
Merge branch 'master' into ConfigNode-metric
README.md | 2 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 2 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +
client-py/iotdb/sqlalchemy/tests/test_dialect.py | 2 +-
client-py/tests/test_dataframe.py | 4 +-
client-py/tests/test_todf.py | 6 +-
.../confignode/client/ConfigNodeRequestType.java | 3 +-
.../client/sync/SyncConfigNodeClientPool.java | 2 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 6 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 1 +
.../iotdb/confignode/manager/ConfigManager.java | 33 +
.../apache/iotdb/confignode/manager/IManager.java | 17 +
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../region/CopySetRegionGroupAllocator.java | 2 +-
.../region/GreedyRegionGroupAllocator.java | 8 +-
.../balancer/region/IRegionGroupAllocator.java | 2 +-
.../manager/node/ClusterNodeStartUtils.java | 21 +-
.../iotdb/confignode/manager/node/NodeManager.java | 4 +-
.../node/heartbeat/ConfigNodeHeartbeatCache.java | 11 +-
.../node/heartbeat/DataNodeHeartbeatCache.java | 4 +-
.../node/heartbeat/NodeHeartbeatSample.java | 20 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 10 +-
.../iotdb/confignode/service/ConfigNode.java | 31 +-
.../confignode/service/ConfigNodeShutdownHook.java | 96 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 25 +-
.../region/GreedyRegionGroupAllocatorTest.java | 14 +-
.../consensus/iot/IoTConsensusServerImpl.java | 9 +-
.../iot/logdispatcher/IndexController.java | 13 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 2 +-
.../consensus/iot/logdispatcher/SyncStatus.java | 9 +-
.../iot/logdispatcher/IndexControllerTest.java | 25 +-
.../iot/logdispatcher/SyncStatusTest.java | 10 +-
docs/Download/README.md | 22 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 2 +-
.../Data-Concept/Data-Model-and-Terminology.md | 2 +-
docs/UserGuide/Data-Concept/Encoding.md | 4 +-
.../Apache-IoTDB-ConfigNode-Dashboard.json | 3127 ++++++---
.../Apache-IoTDB-DataNode-Dashboard.json | 7344 +++++++++++---------
docs/UserGuide/Query-Data/Align-By.md | 2 +-
docs/UserGuide/Query-Data/Group-By.md | 267 +-
docs/UserGuide/Query-Data/Overview.md | 5 +-
docs/zh/Download/README.md | 21 +-
.../UserGuide/API/Programming-Java-Native-API.md | 2 +-
.../Data-Concept/Data-Model-and-Terminology.md | 2 +-
docs/zh/UserGuide/Data-Concept/Encoding.md | 6 +-
docs/zh/UserGuide/Operate-Metadata/Template.md | 2 +-
docs/zh/UserGuide/Query-Data/Group-By.md | 258 +
docs/zh/UserGuide/Query-Data/Overview.md | 7 +-
.../iotdb/it/env/cluster/DataNodeWrapper.java | 1 +
.../iotdb/it/env/cluster/MppCommonConfig.java | 6 +
.../it/env/cluster/MppSharedCommonConfig.java | 7 +
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 57 +-
.../it/cluster/IoTDBClusterNodeGetterIT.java | 1 +
.../it/cluster/IoTDBClusterNodeShutdownHookIT.java | 103 +
.../it/cluster/IoTDBClusterRestartIT.java | 1 +
.../it/aggregation/IoTDBAggregationByLevel2IT.java | 22 +-
.../it/aggregation/IoTDBAggregationByLevelIT.java | 4 +-
.../iotdb/db/it/aggregation/IoTDBCountIfIT.java | 18 +
.../db/it/alignbydevice/IoTDBAlignByDevice2IT.java | 22 +-
.../db/it/alignbydevice/IoTDBAlignByDeviceIT.java | 2 +-
.../IoTDBOrderByWithAlignByDevice2IT.java | 22 +-
.../IoTDBOrderByWithAlignByDeviceIT.java | 2 +-
.../db/it/aligned/IoTDBAlignedSeriesQuery4IT.java | 54 +
.../iotdb/db/it/groupby/IoTDBGroupBySeriesIT.java | 404 ++
.../iotdb/db/it/groupby/IoTDBGroupBySessionIT.java | 509 ++
.../db/it/groupby/IoTDBGroupByVariationIT.java | 16 +-
.../dropwizard/DropwizardMetricManager.java | 4 +-
.../dropwizard/type/DropwizardAutoGauge.java | 16 +-
.../iotdb/metrics/AbstractMetricManager.java | 27 +-
.../iotdb/metrics/AbstractMetricService.java | 18 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 2 +-
.../metrics/config/MetricConfigDescriptor.java | 7 +-
.../apache/iotdb/metrics/config/ReloadLevel.java | 2 -
.../iotdb/metrics/impl/DoNothingAutoGauge.java | 4 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 28 +-
.../iotdb/metrics/metricsets/disk/DiskMetrics.java | 277 +
.../metricsets/disk/IDiskMetricsManager.java | 123 +
.../metricsets/disk/LinuxDiskMetricsManager.java | 462 ++
.../disk/MacDiskMetricsManager.java} | 12 +-
.../disk/WindowsDiskMetricsManager.java} | 12 +-
.../org/apache/iotdb/metrics/type/AutoGauge.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 4 -
.../micrometer/MicrometerMetricManager.java | 6 +-
.../micrometer/type/MicrometerAutoGauge.java | 17 +-
.../resources/conf/iotdb-common.properties | 8 +-
.../iotdb/commons/concurrent/ThreadName.java | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 9 +-
.../commons/service/metric/MetricService.java | 10 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +
.../schemaregion/rocksdb/RSchemaRegion.java | 6 +
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 6 +
.../resources/conf/iotdb-datanode.properties | 7 +-
.../codegen/templates/evEventWindowManager.ftl | 6 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 23 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 20 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 17 +-
.../impl/ReadPointCompactionPerformer.java | 11 +-
.../fast/AlignedSeriesCompactionExecutor.java | 11 +-
.../fast/NonAlignedSeriesCompactionExecutor.java | 17 +-
.../executor/fast/SeriesCompactionExecutor.java | 345 +-
.../fast/element/ChunkMetadataElement.java | 5 -
.../utils/executor/fast/element/PointElement.java | 12 +-
.../execute/utils/reader/PointPriorityReader.java | 60 +-
.../utils/reader/SeriesDataBlockReader.java | 44 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 13 +-
.../db/engine/querycontext/QueryDataSource.java | 25 -
.../iotdb/db/engine/storagegroup/DataRegion.java | 4 +-
.../db/metadata/cache/DataNodeSchemaCache.java | 187 +-
.../iotdb/db/metadata/cache/SchemaCacheEntry.java | 18 +-
.../metadata/cache/dualkeycache/IDualKeyCache.java | 57 +
.../dualkeycache/IDualKeyCacheComputation.java} | 28 +-
.../cache/dualkeycache/IDualKeyCacheStats.java} | 28 +-
.../dualkeycache/impl/CacheEntryGroupImpl.java | 75 +
.../dualkeycache/impl/CacheSizeComputerImpl.java | 55 +
.../cache/dualkeycache/impl/CacheStats.java | 110 +
.../dualkeycache/impl/DualKeyCacheBuilder.java | 85 +
.../cache/dualkeycache/impl/DualKeyCacheImpl.java | 281 +
.../dualkeycache/impl/DualKeyCachePolicy.java | 13 +-
.../cache/dualkeycache/impl/ICacheEntry.java} | 25 +-
.../cache/dualkeycache/impl/ICacheEntryGroup.java | 39 +-
.../dualkeycache/impl/ICacheEntryManager.java} | 44 +-
.../dualkeycache/impl/ICacheSizeComputer.java | 18 +-
.../dualkeycache/impl/LRUCacheEntryManager.java | 216 +
.../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 4 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 64 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 55 +-
.../mtree/snapshot/MemMTreeSnapshotUtil.java | 45 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 76 +-
.../iotdb/db/metadata/mtree/store/IMTreeStore.java | 8 +
.../db/metadata/mtree/store/MemMTreeStore.java | 66 +-
.../store/ReentrantReadOnlyCachedMTreeStore.java | 6 +
.../mtree/store/disk/cache/CacheManager.java | 11 +-
.../mtree/store/disk/cache/CacheMemoryManager.java | 180 +-
.../mtree/store/disk/cache/LRUCacheManager.java | 6 +-
.../mtree/store/disk/cache/PlainCacheManager.java | 5 +
.../disk/memcontrol/IReleaseFlushStrategy.java | 16 +-
.../mtree/store/disk/memcontrol/MemManager.java | 86 +
.../MemManagerNodeEstimatedSizeBasedImpl.java | 133 -
.../memcontrol/MemManagerNodeNumBasedImpl.java | 105 -
....java => ReleaseFlushStrategyNumBasedImpl.java} | 29 +-
.../ReleaseFlushStrategySizeBasedImpl.java | 51 +
.../rescon/CachedSchemaEngineStatistics.java | 84 +
.../rescon/CachedSchemaRegionStatistics.java | 94 +
.../metadata/rescon/ISchemaEngineStatistics.java | 23 +-
.../metadata/rescon/ISchemaRegionStatistics.java | 22 +-
...tistics.java => MemSchemaEngineStatistics.java} | 68 +-
.../metadata/rescon/MemSchemaRegionStatistics.java | 93 +
.../SchemaEngineStatisticsHolder.java} | 24 +-
.../db/metadata/rescon/SchemaResourceManager.java | 10 +-
.../rescon/SchemaResourceManagerMetrics.java | 28 +-
.../metadata/rescon/SchemaStatisticsManager.java | 56 -
.../db/metadata/schemaregion/ISchemaRegion.java | 5 +
.../db/metadata/schemaregion/SchemaEngine.java | 6 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 52 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 58 +-
.../db/mpp/aggregation/AccumulatorFactory.java | 8 +-
.../db/mpp/aggregation/CountIfAccumulator.java | 7 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 12 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 100 +-
.../db/mpp/execution/driver/DataDriverContext.java | 3 +-
.../fragment/FragmentInstanceContext.java | 86 +-
.../fragment/FragmentInstanceExecution.java | 1 +
.../process/RawDataAggregationOperator.java | 8 +-
.../AbstractSeriesAggregationScanOperator.java | 6 +-
.../AlignedSeriesAggregationScanOperator.java | 18 +-
.../operator/source/AlignedSeriesScanOperator.java | 18 +-
.../operator/source/AlignedSeriesScanUtil.java | 80 +-
.../source/SeriesAggregationScanOperator.java | 20 +-
.../operator/source/SeriesScanOperator.java | 33 +-
.../execution/operator/source/SeriesScanUtil.java | 228 +-
.../operator/window/EventWindowManager.java | 18 +-
.../execution/operator/window/IWindowManager.java | 26 +-
.../execution/operator/window/SeriesWindow.java | 113 +
.../operator/window/SeriesWindowManager.java | 194 +
...{WindowType.java => SeriesWindowParameter.java} | 33 +-
.../execution/operator/window/SessionWindow.java | 157 +
...indowManager.java => SessionWindowManager.java} | 95 +-
...WindowType.java => SessionWindowParameter.java} | 17 +-
.../operator/window/TimeWindowManager.java | 2 +-
.../operator/window/WindowManagerFactory.java | 19 +-
.../mpp/execution/operator/window/WindowType.java | 5 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 114 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 11 +
.../plan/analyze/schema/ClusterSchemaFetcher.java | 2 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 59 +-
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 3 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 301 +-
.../plan/node/process/HorizontallyConcatNode.java | 1 -
.../planner/plan/parameter/GroupByParameter.java | 21 +-
...nParameter.java => GroupBySeriesParameter.java} | 40 +-
...Parameter.java => GroupBySessionParameter.java} | 35 +-
.../plan/parameter/GroupByVariationParameter.java | 22 +-
.../planner/plan/parameter/SeriesScanOptions.java | 151 +
.../component/GroupBySeriesComponent.java} | 26 +-
.../component/GroupBySessionComponent.java} | 18 +-
.../db/mpp/plan/statement/component/Ordering.java | 4 +
.../db/mpp/plan/statement/crud/QueryStatement.java | 15 +-
.../query/reader/chunk/MemAlignedPageReader.java | 61 +-
.../iotdb/db/query/reader/chunk/MemPageReader.java | 211 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 2 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 41 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 3 +
.../iotdb/db/service/metrics/SystemMetrics.java | 2 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 18 +
.../impl/DataNodeInternalRPCServiceImpl.java | 28 +-
.../iotdb/db/utils/concurrent/FiniteSemaphore.java | 56 +
.../engine/compaction/AbstractCompactionTest.java | 13 +-
.../engine/compaction/CompactionSchedulerTest.java | 1 +
.../CompactionSchedulerWithFastPerformerTest.java | 1 +
.../compaction/FastAlignedCrossCompactionTest.java | 398 ++
.../FastCrossCompactionPerformerTest.java | 34 -
.../FastInnerCompactionPerformerTest.java | 30 -
.../FastNonAlignedCrossCompactionTest.java | 420 +-
.../ReadPointCompactionPerformerTest.java | 73 +-
.../CrossSpaceCompactionWithFastPerformerTest.java | 2 +
...eCompactionWithFastPerformerValidationTest.java | 1 +
...sSpaceCompactionWithReadPointPerformerTest.java | 2 +
...actionWithReadPointPerformerValidationTest.java | 1 +
.../engine/compaction/cross/MergeUpgradeTest.java | 2 +
.../cross/RewriteCompactionFileSelectorTest.java | 33 +-
...eCrossSpaceCompactionWithFastPerformerTest.java | 16 +-
...sSpaceCompactionWithReadPointPerformerTest.java | 16 +-
.../inner/InnerCompactionMoreDataTest.java | 1 -
.../SizeTieredCompactionRecoverTest.java | 17 -
.../compaction/utils/CompactionCheckerUtils.java | 1 -
.../compaction/utils/CompactionConfigRestorer.java | 4 +
.../compaction/utils/TsFileGeneratorUtils.java | 10 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 1 -
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 8 -
.../cache/dualkeycache/DualKeyCacheTest.java | 110 +
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 11 +-
.../schemaRegion/SchemaStatisticsTest.java | 249 +
.../apache/iotdb/db/metric/MetricServiceTest.java | 24 +-
.../iotdb/db/mpp/execution/DataDriverTest.java | 18 +-
.../operator/AggregationOperatorTest.java | 19 +-
.../AlignedSeriesAggregationScanOperatorTest.java | 15 +-
.../operator/AlignedSeriesScanOperatorTest.java | 130 +-
.../operator/DeviceMergeOperatorTest.java | 61 +-
.../execution/operator/DeviceViewOperatorTest.java | 16 +-
.../operator/HorizontallyConcatOperatorTest.java | 15 +-
.../execution/operator/LastQueryOperatorTest.java | 28 +-
.../operator/LastQuerySortOperatorTest.java | 26 +-
.../mpp/execution/operator/LimitOperatorTest.java | 18 +-
.../execution/operator/MergeSortOperatorTest.java | 151 +-
.../mpp/execution/operator/OffsetOperatorTest.java | 52 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 24 +-
.../operator/RawDataAggregationOperatorTest.java | 78 +-
.../SeriesAggregationScanOperatorTest.java | 10 +-
.../execution/operator/SeriesScanOperatorTest.java | 11 +-
.../operator/SingleDeviceViewOperatorTest.java | 17 +-
.../SlidingWindowAggregationOperatorTest.java | 9 +-
.../execution/operator/TimeJoinOperatorTest.java | 66 +-
.../operator/UpdateLastCacheOperatorTest.java | 10 +-
.../db/mpp/plan/plan/PipelineBuilderTest.java | 9 +-
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 682 ++
.../series/SeriesScanLimitOffsetPushDownTest.java | 368 +
.../src/main/thrift/confignode.thrift | 16 +
thrift/src/main/thrift/datanode.thrift | 4 +-
.../tsfile/file/metadata/AlignedChunkMetadata.java | 9 +
.../file/metadata/AlignedTimeSeriesMetadata.java | 8 +
.../file/metadata/statistics/Statistics.java | 4 +
.../iotdb/tsfile/read/reader/IPageReader.java | 3 +
.../tsfile/read/reader/page/AlignedPageReader.java | 73 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 96 +-
.../tsfile/read/reader/page/ValuePageReader.java | 6 +-
.../read/reader/series/PaginationController.java | 66 +
269 files changed, 16118 insertions(+), 7313 deletions(-)
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 183b4d538e,44a3d26d38..84cb1b7550
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@@ -86,9 -84,8 +86,10 @@@ import org.apache.iotdb.confignode.mana
import org.apache.iotdb.confignode.manager.load.LoadManager;
import org.apache.iotdb.confignode.manager.node.ClusterNodeStartUtils;
import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.manager.node.NodeMetrics;
+ import org.apache.iotdb.confignode.manager.node.heartbeat.NodeHeartbeatSample;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.manager.partition.PartitionMetrics;
import org.apache.iotdb.confignode.persistence.AuthorInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
import org.apache.iotdb.confignode.persistence.TriggerInfo;
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
index 1ecd98db8c,616743b9fe..4ff4739bfd
--- a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
@@@ -219,22 -233,6 +221,18 @@@ public class ConfigNode implements Conf
LOGGER.info("Successfully setup internal services.");
}
+ private void initConfigManager() {
+ try {
+ configManager = new ConfigManager();
+ } catch (IOException e) {
+ LOGGER.error("Can't start ConfigNode consensus group!", e);
- try {
- stop();
- } catch (IOException e2) {
- LOGGER.error("Meet error when stop ConfigNode!", e);
- }
++ stop();
+ }
+ // Add some Metrics for configManager
+ configManager.addMetrics();
+ LOGGER.info("Successfully initialize ConfigManager.");
+ }
-
++
/** Register Non-seed ConfigNode when first startup */
private void sendRegisterConfigNodeRequest() throws StartupException, IOException {
TConfigNodeRegisterReq req =