You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/07/28 03:01:27 UTC
[iotdb] branch rel/0.13 updated: [To rel/0.13][IOTDB-3969]Fix timeseries statistic bug (#6808)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new 010e65d5f1 [To rel/0.13][IOTDB-3969]Fix timeseries statistic bug (#6808)
010e65d5f1 is described below
commit 010e65d5f15583201d6eabb2ae63792d63891016
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Thu Jul 28 11:01:21 2022 +0800
[To rel/0.13][IOTDB-3969]Fix timeseries statistic bug (#6808)
---
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 2 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 2 +-
.../MeasurementInsideTemplateException.java | 34 ++++++++
.../org/apache/iotdb/db/metadata/MManager.java | 74 ++++++++++++------
.../org/apache/iotdb/db/metadata/mtree/MTree.java | 10 ++-
.../db/metadata/mtree/traverser/Traverser.java | 4 +
.../iotdb/db/metadata/MManagerBasicTest.java | 90 +++++++++++++++++++++-
7 files changed, 183 insertions(+), 33 deletions(-)
diff --git a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
index d7fa3ef222..689e91aaa9 100644
--- a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -111,7 +111,7 @@ Next, we will choose Prometheus format data as samples to describe each kind of
| Metric | Tag | level | Description | Sample |
| -------- | ------------------------------------- | --------- | ------------------------------------------------------------- | -------------------------------- |
-| quantity | name="timeSeries/storageGroup/device" | important | The current count of timeSeries/storageGroup/devices in IoTDB | quantity{name="timeSeries",} 1.0 |
+| quantity | name="storageGroup/device";<br/>name="timeSeries", type="normal/template" | important | The current count of timeSeries/storageGroup/devices in IoTDB | quantity{name="timeSeries",type="normal"} 1.0 |
#### 4.3.6. Cluster
diff --git a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
index 44be673b9c..05b68d2296 100644
--- a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -110,7 +110,7 @@ IoTDB对外提供JMX和Prometheus格式的监控指标，对于JMX，可以通
| Metric | Tag | level | 说明 | 示例 |
| -------- | ------------------------------------- | --------- | -------------------------------------------- | -------------------------------- |
-| quantity | name="timeSeries/storageGroup/device" | important | 当前时间timeSeries/storageGroup/device的数量 | quantity{name="timeSeries",} 1.0 |
+| quantity | name="timeSeries/storageGroup/device";<br/>name="timeSeries", type="normal/template" | important | 当前时间timeSeries/storageGroup/device的数量 | quantity{name="timeSeries",type="normal"} 1.0 |
#### 4.3.6. 集群
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/MeasurementInsideTemplateException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/MeasurementInsideTemplateException.java
new file mode 100644
index 0000000000..ba900102de
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/MeasurementInsideTemplateException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.exception.metadata;
+
+public class MeasurementInsideTemplateException extends MetadataException {
+
+ private String path;
+
+ public MeasurementInsideTemplateException(String path) {
+ super(String.format("Cannot delete a timeseries inside a template: %s", path));
+ this.path = path;
+ }
+
+ public String getPath() {
+ return path;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 2bb80ce64c..6996430acc 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
import org.apache.iotdb.db.exception.metadata.DeleteFailedException;
import org.apache.iotdb.db.exception.metadata.DifferentTemplateException;
import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
+import org.apache.iotdb.db.exception.metadata.MeasurementInsideTemplateException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.NoTemplateOnMNodeException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
@@ -185,7 +186,8 @@ public class MManager {
private boolean initialized;
private boolean allowToCreateNewSeries = true;
- private AtomicLong totalSeriesNumber = new AtomicLong();
+ private AtomicLong totalNormalSeriesNumber = new AtomicLong();
+ private AtomicLong totalTemplateSeriesNumber = new AtomicLong();
private final int mtreeSnapshotInterval;
private final long mtreeSnapshotThresholdTime;
@@ -319,10 +321,24 @@ public class MManager {
.getOrCreateAutoGauge(
Metric.QUANTITY.toString(),
MetricLevel.IMPORTANT,
- totalSeriesNumber,
+ totalNormalSeriesNumber,
AtomicLong::get,
Tag.NAME.toString(),
- "timeSeries");
+ "timeSeries",
+ Tag.TYPE.toString(),
+ "normal");
+
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateAutoGauge(
+ Metric.QUANTITY.toString(),
+ MetricLevel.IMPORTANT,
+ totalTemplateSeriesNumber,
+ AtomicLong::get,
+ Tag.NAME.toString(),
+ "timeSeries",
+ Tag.TYPE.toString(),
+ "template");
MetricsService.getInstance()
.getMetricManager()
@@ -445,7 +461,8 @@ public class MManager {
if (this.mNodeCache != null) {
this.mNodeCache.invalidateAll();
}
- this.totalSeriesNumber.set(0);
+ this.totalNormalSeriesNumber.set(0);
+ this.totalTemplateSeriesNumber.set(0);
this.templateManager.clear();
if (logWriter != null) {
logWriter.close();
@@ -596,9 +613,9 @@ public class MManager {
}
// update statistics and schemaDataTypeNumMap
- totalSeriesNumber.addAndGet(1);
- if (totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
- logger.warn("Current series number {} is too large...", totalSeriesNumber);
+ totalNormalSeriesNumber.addAndGet(1);
+ if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
+ logger.warn("Current series number {} is too large...", totalNormalSeriesNumber);
allowToCreateNewSeries = false;
}
@@ -700,9 +717,9 @@ public class MManager {
mNodeCache.invalidate(prefixPath);
// update statistics and schemaDataTypeNumMap
- totalSeriesNumber.addAndGet(measurements.size());
- if (totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
- logger.warn("Current series number {} is too large...", totalSeriesNumber);
+ totalNormalSeriesNumber.addAndGet(measurements.size());
+ if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
+ logger.warn("Current series number {} is too large...", totalNormalSeriesNumber);
allowToCreateNewSeries = false;
}
// write log
@@ -793,6 +810,8 @@ public class MManager {
deleteTimeSeriesPlan.setDeletePathList(Collections.singletonList(p));
logWriter.deleteTimeseries(deleteTimeSeriesPlan);
}
+ } catch (MeasurementInsideTemplateException e) {
+ failedNames.add(e.getMessage());
} catch (DeleteFailedException e) {
failedNames.add(e.getName());
}
@@ -825,10 +844,10 @@ public class MManager {
mNodeCache.invalidate(node.getPartialPath());
node = node.getParent();
}
- totalSeriesNumber.addAndGet(-1);
+ totalNormalSeriesNumber.addAndGet(-1);
if (!allowToCreateNewSeries
- && totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
- logger.info("Current series number {} come back to normal level", totalSeriesNumber);
+ && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
+ logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber);
allowToCreateNewSeries = true;
}
return storageGroupPath;
@@ -865,12 +884,14 @@ public class MManager {
public void deleteStorageGroups(List<PartialPath> storageGroups) throws MetadataException {
try {
for (PartialPath storageGroup : storageGroups) {
- totalSeriesNumber.addAndGet(
- -mtree.getAllTimeseriesCount(storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD)));
+ totalNormalSeriesNumber.addAndGet(
+ -mtree.getAllTimeseriesCount(
+ storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false));
// clear cached MNode
if (!allowToCreateNewSeries
- && totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
- logger.info("Current series number {} come back to normal level", totalSeriesNumber);
+ && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
+ logger.info(
+ "Current series number {} come back to normal level", totalNormalSeriesNumber);
allowToCreateNewSeries = true;
}
mNodeCache.invalidateAll();
@@ -998,8 +1019,12 @@ public class MManager {
// region Interfaces for metadata count
- public long getTotalSeriesNumber() {
- return totalSeriesNumber.get();
+ public long getTotalNormalSeriesNumber() {
+ return totalNormalSeriesNumber.get();
+ }
+
+ public long getTotalTemplateSeriesNumber() {
+ return totalTemplateSeriesNumber.get();
}
/**
@@ -1009,7 +1034,7 @@ public class MManager {
*/
public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
throws MetadataException {
- return mtree.getAllTimeseriesCount(pathPattern, isPrefixMatch);
+ return mtree.getAllTimeseriesCount(pathPattern, isPrefixMatch, true);
}
/**
@@ -2452,6 +2477,8 @@ public class MManager {
}
node.setUseTemplate(false);
+ totalTemplateSeriesNumber.addAndGet(-node.getUpperTemplate().getMeasurementsCount());
+
// clear caches within MManger
mNodeCache.invalidate(node);
if (node.isEntity()) {
@@ -2470,7 +2497,8 @@ public class MManager {
IMNode setUsingSchemaTemplate(IMNode node) throws MetadataException {
// check whether any template has been set on designated path
- if (node.getUpperTemplate() == null) {
+ Template template = node.getUpperTemplate();
+ if (template == null) {
throw new MetadataException(
String.format("Path [%s] has not been set any template.", node.getFullPath()));
}
@@ -2480,8 +2508,7 @@ public class MManager {
// check alignment of template and mounted node
// if direct measurement exists, node will be replaced
- IMNode mountedMNode =
- mtree.checkTemplateAlignmentWithMountedNode(node, node.getUpperTemplate());
+ IMNode mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template);
// if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or
// template
@@ -2494,6 +2521,7 @@ public class MManager {
: node.getUpperTemplate().isDirectAligned());
}
mountedMNode.setUseTemplate(true);
+ totalTemplateSeriesNumber.addAndGet(template.getMeasurementsCount());
if (node != mountedMNode) {
mNodeCache.invalidate(mountedMNode.getPartialPath());
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
index eb38dd6cc6..6cdbac93b4 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
+import org.apache.iotdb.db.exception.metadata.MeasurementInsideTemplateException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
@@ -540,8 +541,7 @@ public class MTree implements Serializable {
}
if (isPathExistsWithinTemplate(path)) {
- throw new MetadataException(
- "Cannot delete a timeseries inside a template: " + path.toString());
+ throw new MeasurementInsideTemplateException(path.getFullPath());
}
IMeasurementMNode deletedNode = getMeasurementMNode(path);
@@ -1272,9 +1272,11 @@ public class MTree implements Serializable {
*
* @param pathPattern a path pattern or a full path, may contain wildcard
*/
- public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
+ public int getAllTimeseriesCount(
+ PartialPath pathPattern, boolean isPrefixMatch, boolean traverseTemplate)
throws MetadataException {
CounterTraverser counter = new MeasurementCounter(root, pathPattern);
+ counter.setShouldTraverseTemplate(traverseTemplate);
counter.setPrefixMatch(isPrefixMatch);
counter.traverse();
return counter.getCount();
@@ -1286,7 +1288,7 @@ public class MTree implements Serializable {
* @param pathPattern a path pattern or a full path, may contain wildcard
*/
public int getAllTimeseriesCount(PartialPath pathPattern) throws MetadataException {
- return getAllTimeseriesCount(pathPattern, false);
+ return getAllTimeseriesCount(pathPattern, false, true);
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
index ef3a0c1203..85b31a4403 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
@@ -279,6 +279,10 @@ public abstract class Traverser {
this.isPrefixMatch = isPrefixMatch;
}
+ public void setShouldTraverseTemplate(boolean shouldTraverseTemplate) {
+ this.shouldTraverseTemplate = shouldTraverseTemplate;
+ }
+
/**
* @param currentNode the node need to get the full path of
* @return full path from traverse start node to the current node
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index fb6c0505d2..21b1ca63c6 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.DeactivateTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
@@ -1918,13 +1919,13 @@ public class MManagerBasicTest {
CompressionType.GZIP,
null);
- assertEquals(6, manager.getTotalSeriesNumber());
+ assertEquals(6, manager.getTotalNormalSeriesNumber());
EnvironmentUtils.restartDaemon();
- assertEquals(6, manager.getTotalSeriesNumber());
+ assertEquals(6, manager.getTotalNormalSeriesNumber());
manager.deleteTimeseries(new PartialPath("root.laptop.d2.s1"));
- assertEquals(5, manager.getTotalSeriesNumber());
+ assertEquals(5, manager.getTotalNormalSeriesNumber());
manager.deleteStorageGroups(Collections.singletonList(new PartialPath("root.laptop")));
- assertEquals(0, manager.getTotalSeriesNumber());
+ assertEquals(0, manager.getTotalNormalSeriesNumber());
} catch (MetadataException e) {
e.printStackTrace();
fail(e.getMessage());
@@ -2582,4 +2583,85 @@ public class MManagerBasicTest {
Assert.assertEquals(1, result.size());
Assert.assertTrue(result.contains("d1"));
}
+
+ @Test
+ public void testTimeseriesNumberStatistic() throws Exception {
+ MManager manager = IoTDB.metaManager;
+
+ try {
+ manager.setStorageGroup(new PartialPath("root.laptop"));
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d0"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d1.s1"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d1.s2.t1"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d1.s3"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d2.s1"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d2.s2"),
+ TSDataType.INT32,
+ TSEncoding.PLAIN,
+ CompressionType.GZIP,
+ null);
+
+ assertEquals(6, manager.getTotalNormalSeriesNumber());
+
+ manager.setStorageGroup(new PartialPath("root.sg"));
+
+ CreateTemplatePlan plan = getCreateTemplatePlan("s0");
+ manager.createSchemaTemplate(plan);
+ SetTemplatePlan setPlan = new SetTemplatePlan("template1", "root.sg.d1");
+ manager.setSchemaTemplate(setPlan);
+
+ manager.setUsingSchemaTemplate(manager.getDeviceNode(new PartialPath("root.sg.d1")));
+
+ assertEquals(6, manager.getTotalNormalSeriesNumber());
+ assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+ manager.deleteTimeseries(new PartialPath("root.laptop.d2.s1"));
+ assertEquals(5, manager.getTotalNormalSeriesNumber());
+ assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+ manager.deleteStorageGroups(Collections.singletonList(new PartialPath("root.laptop")));
+ assertEquals(0, manager.getTotalNormalSeriesNumber());
+ assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+ String failedName = manager.deleteTimeseries(new PartialPath("root.sg.d1.s0"));
+ assertTrue(failedName.contains("root.sg.d1.s0"));
+ assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+ DeactivateTemplatePlan deactivateTemplatePlan =
+ new DeactivateTemplatePlan("template1", "root.sg.d1");
+ deactivateTemplatePlan.setPaths(Collections.singletonList(new PartialPath("root.sg.d1")));
+ manager.deactivateSchemaTemplate(deactivateTemplatePlan);
+ assertEquals(0, manager.getTotalTemplateSeriesNumber());
+
+ } catch (MetadataException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}