You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2023/03/28 00:33:39 UTC
[iotdb] branch rel/1.1 updated: [To rel/1.1][IOTDB-5732] Add Device Schema Statistics (#9465)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new 96b0d796d7 [To rel/1.1][IOTDB-5732] Add Device Schema Statistics (#9465)
96b0d796d7 is described below
commit 96b0d796d704904246b97d8dc090ba0641868ae9
Author: Chen YZ <43...@users.noreply.github.com>
AuthorDate: Tue Mar 28 08:33:30 2023 +0800
[To rel/1.1][IOTDB-5732] Add Device Schema Statistics (#9465)
---
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 18 +++---
.../db/metadata/mtree/store/CachedMTreeStore.java | 7 +--
.../db/metadata/mtree/store/MemMTreeStore.java | 2 +
.../metadata/rescon/ISchemaRegionStatistics.java | 2 +
.../metadata/rescon/MemSchemaRegionStatistics.java | 15 +++++
.../schemaregion/SchemaRegionMemoryImpl.java | 37 +++++++------
.../schemaregion/SchemaRegionSchemaFileImpl.java | 64 +++++++++++-----------
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 3 +
.../schemaRegion/SchemaStatisticsTest.java | 30 ++++++++++
9 files changed, 116 insertions(+), 62 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGCachedImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGCachedImpl.java
index d70d235512..d340268ca2 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGCachedImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGCachedImpl.java
@@ -121,6 +121,7 @@ public class MTreeBelowSGCachedImpl implements IMTreeBelowSG {
Function<IMeasurementMNode, Map<String, String>> tagGetter,
Runnable flushCallback,
Consumer<IMeasurementMNode> measurementProcess,
+ Consumer<IEntityMNode> deviceProcess,
int schemaRegionId,
CachedSchemaRegionStatistics regionStatistics)
throws MetadataException, IOException {
@@ -132,14 +133,17 @@ public class MTreeBelowSGCachedImpl implements IMTreeBelowSG {
this.rootNode = store.generatePrefix(storageGroupPath);
levelOfSG = storageGroupPath.getNodeLength() - 1;
- // recover measurement
- try (MeasurementCollector<?> collector =
- new MeasurementCollector<Void>(
+ // recover MNode
+ try (MNodeCollector<?> collector =
+ new MNodeCollector<Void>(
this.rootNode, new PartialPath(storageGroupMNode.getFullPath()), this.store, true) {
@Override
- protected Void collectMeasurement(IMeasurementMNode node) {
- measurementProcess.accept(node);
- regionStatistics.addTimeseries(1L);
+ protected Void collectMNode(IMNode node) {
+ if (node.isMeasurement()) {
+ measurementProcess.accept(node.getAsMeasurementMNode());
+ } else if (node.isEntity()) {
+ deviceProcess.accept(node.getAsEntityMNode());
+ }
return null;
}
}) {
@@ -163,7 +167,7 @@ public class MTreeBelowSGCachedImpl implements IMTreeBelowSG {
levelOfSG = storageGroupMNode.getPartialPath().getNodeLength() - 1;
this.tagGetter = tagGetter;
- // recover measurement
+ // recover MNode
try (MNodeCollector<?> collector =
new MNodeCollector<Void>(
this.rootNode, new PartialPath(storageGroupMNode.getFullPath()), this.store, true) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java
index 769f01162c..93c9238433 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/CachedMTreeStore.java
@@ -57,15 +57,10 @@ public class CachedMTreeStore implements IMTreeStore {
private static final Logger logger = LoggerFactory.getLogger(CachedMTreeStore.class);
private final MemManager memManager;
-
private final ICacheManager cacheManager;
-
private ISchemaFile file;
-
private IMNode root;
-
private final Runnable flushCallback;
-
private final CachedSchemaRegionStatistics regionStatistics;
private final StampedWriterPreferredLock lock = new StampedWriterPreferredLock();
@@ -333,6 +328,7 @@ public class CachedMTreeStore implements IMTreeStore {
public IEntityMNode setToEntity(IMNode node) {
IEntityMNode result = MNodeUtils.setToEntity(node);
if (result != node) {
+ regionStatistics.addDevice();
memManager.updatePinnedSize(IMNodeSizeEstimator.getEntityNodeBaseSize());
}
updateMNode(result);
@@ -343,6 +339,7 @@ public class CachedMTreeStore implements IMTreeStore {
public IMNode setToInternal(IEntityMNode entityMNode) {
IMNode result = MNodeUtils.setToInternal(entityMNode);
if (result != entityMNode) {
+ regionStatistics.deleteDevice();
memManager.updatePinnedSize(-IMNodeSizeEstimator.getEntityNodeBaseSize());
}
updateMNode(result);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java
index 791eb17e57..3ca77d321d 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/MemMTreeStore.java
@@ -157,6 +157,7 @@ public class MemMTreeStore implements IMTreeStore {
public IEntityMNode setToEntity(IMNode node) {
IEntityMNode result = MNodeUtils.setToEntity(node);
if (result != node) {
+ regionStatistics.addDevice();
requestMemory(IMNodeSizeEstimator.getEntityNodeBaseSize());
}
@@ -170,6 +171,7 @@ public class MemMTreeStore implements IMTreeStore {
public IMNode setToInternal(IEntityMNode entityMNode) {
IMNode result = MNodeUtils.setToInternal(entityMNode);
if (result != entityMNode) {
+ regionStatistics.deleteDevice();
releaseMemory(IMNodeSizeEstimator.getEntityNodeBaseSize());
}
if (result.isStorageGroup()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
index 6dade84995..066ccbed5d 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
@@ -28,6 +28,8 @@ public interface ISchemaRegionStatistics {
long getSeriesNumber();
+ long getDevicesNumber();
+
int getTemplateActivatedNumber();
long getTemplateSeriesNumber();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
index c8747d3b07..35e6dfa964 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
@@ -31,6 +31,7 @@ public class MemSchemaRegionStatistics implements ISchemaRegionStatistics {
private final int schemaRegionId;
private final AtomicLong memoryUsage = new AtomicLong(0);
private final AtomicLong seriesNumber = new AtomicLong(0);
+ private final AtomicLong devicesNumber = new AtomicLong(0);
private final Map<Integer, Integer> templateUsage = new ConcurrentHashMap<>();
private long mLogLength = 0;
@@ -70,6 +71,19 @@ public class MemSchemaRegionStatistics implements ISchemaRegionStatistics {
schemaEngineStatistics.deleteTimeseries(deletedNum);
}
+ @Override
+ public long getDevicesNumber() {
+ return devicesNumber.get();
+ }
+
+ public void addDevice() {
+ devicesNumber.incrementAndGet();
+ }
+
+ public void deleteDevice() {
+ devicesNumber.decrementAndGet();
+ }
+
@Override
public int getTemplateActivatedNumber() {
return templateUsage.size();
@@ -130,6 +144,7 @@ public class MemSchemaRegionStatistics implements ISchemaRegionStatistics {
schemaEngineStatistics.deleteTimeseries(seriesNumber.get());
memoryUsage.getAndSet(0);
seriesNumber.getAndSet(0);
+ devicesNumber.getAndSet(0);
templateUsage.forEach(
(templateId, cnt) -> schemaEngineStatistics.deactivateTemplate(templateId, cnt));
templateUsage.clear();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
index fb9d0e8dc5..f27b857395 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
@@ -129,10 +129,10 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
private boolean isRecovering = true;
private volatile boolean initialized = false;
- private String storageGroupDirPath;
- private String schemaRegionDirPath;
- private String storageGroupFullPath;
- private SchemaRegionId schemaRegionId;
+ private final String storageGroupDirPath;
+ private final String schemaRegionDirPath;
+ private final String storageGroupFullPath;
+ private final SchemaRegionId schemaRegionId;
// the log file writer
private boolean usingMLog = true;
@@ -144,7 +144,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
private TagManager tagManager;
// seriesNumberMonitor may be null
- private final ISeriesNumerMonitor seriesNumerMonitor;
+ private final ISeriesNumerMonitor seriesNumberMonitor;
// region Interfaces and Implementation of initialization、snapshot、recover and clear
public SchemaRegionMemoryImpl(ISchemaRegionParams schemaRegionParams) throws MetadataException {
@@ -167,7 +167,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
}
}
- this.seriesNumerMonitor = schemaRegionParams.getSeriesNumberMonitor();
+ this.seriesNumberMonitor = schemaRegionParams.getSeriesNumberMonitor();
this.regionStatistics =
new MemSchemaRegionStatistics(
schemaRegionId.getId(), schemaRegionParams.getSchemaEngineStatistics());
@@ -389,8 +389,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
public synchronized void deleteSchemaRegion() throws MetadataException {
// collect all the LeafMNode in this schema region
long seriesCount = regionStatistics.getSeriesNumber();
- if (seriesNumerMonitor != null) {
- seriesNumerMonitor.deleteTimeSeries((int) seriesCount);
+ if (seriesNumberMonitor != null) {
+ seriesNumberMonitor.deleteTimeSeries((int) seriesCount);
}
// clear all the components and release all the file handlers
@@ -477,6 +477,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
}
},
deviceMNode -> {
+ regionStatistics.addDevice();
if (deviceMNode.getSchemaTemplateIdWithState() >= 0) {
regionStatistics.activateTemplate(deviceMNode.getSchemaTemplateId());
}
@@ -530,7 +531,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
throw new SeriesOverflowException();
}
- if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) {
+ if (seriesNumberMonitor != null && !seriesNumberMonitor.addTimeSeries(1)) {
throw new SeriesNumberOverflowException();
}
@@ -553,8 +554,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
plan.getProps(),
plan.getAlias());
} catch (Throwable t) {
- if (seriesNumerMonitor != null) {
- seriesNumerMonitor.deleteTimeSeries(1);
+ if (seriesNumberMonitor != null) {
+ seriesNumberMonitor.deleteTimeSeries(1);
}
throw t;
}
@@ -610,7 +611,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
throw new SeriesOverflowException();
}
- if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) {
+ if (seriesNumberMonitor != null && !seriesNumberMonitor.addTimeSeries(seriesCount)) {
throw new SeriesNumberOverflowException();
}
@@ -639,8 +640,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
plan.getCompressors(),
plan.getAliasList());
} catch (Throwable t) {
- if (seriesNumerMonitor != null) {
- seriesNumerMonitor.deleteTimeSeries(seriesCount);
+ if (seriesNumberMonitor != null) {
+ seriesNumberMonitor.deleteTimeSeries(seriesCount);
}
throw t;
}
@@ -782,8 +783,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
removeFromTagInvertedIndex(measurementMNode);
regionStatistics.deleteTimeseries(1L);
- if (seriesNumerMonitor != null) {
- seriesNumerMonitor.deleteTimeSeries(1);
+ if (seriesNumberMonitor != null) {
+ seriesNumberMonitor.deleteTimeSeries(1);
}
}
@@ -799,8 +800,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
removeFromTagInvertedIndex(measurementMNode);
regionStatistics.deleteTimeseries(1L);
- if (seriesNumerMonitor != null) {
- seriesNumerMonitor.deleteTimeSeries(1);
+ if (seriesNumberMonitor != null) {
+ seriesNumberMonitor.deleteTimeSeries(1);
}
}
// endregion
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
index 2c91340ad3..cbca5643b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
@@ -43,6 +43,7 @@ import org.apache.iotdb.db.metadata.logfile.SchemaLogReader;
import org.apache.iotdb.db.metadata.logfile.SchemaLogWriter;
import org.apache.iotdb.db.metadata.metric.ISchemaRegionMetric;
import org.apache.iotdb.db.metadata.metric.SchemaRegionCachedMetric;
+import org.apache.iotdb.db.metadata.mnode.IEntityMNode;
import org.apache.iotdb.db.metadata.mnode.IMNode;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.mtree.MTreeBelowSGCachedImpl;
@@ -93,6 +94,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Consumer;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
@@ -189,19 +191,8 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
new PartialPath(storageGroupFullPath),
tagManager::readTags,
this::flushCallback,
- measurementMNode -> {
- if (measurementMNode.getOffset() == -1) {
- return;
- }
- try {
- tagManager.recoverIndex(measurementMNode.getOffset(), measurementMNode);
- } catch (IOException e) {
- logger.error(
- "Failed to recover tagIndex for {} in schemaRegion {}.",
- storageGroupFullPath + PATH_SEPARATOR + measurementMNode.getFullPath(),
- schemaRegionId);
- }
- },
+ measurementInitProcess(),
+ deviceInitProcess(),
schemaRegionId.getId(),
regionStatistics);
@@ -225,6 +216,32 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
initialized = true;
}
+ private Consumer<IMeasurementMNode> measurementInitProcess() {
+ return measurementMNode -> {
+ regionStatistics.addTimeseries(1L);
+ if (measurementMNode.getOffset() == -1) {
+ return;
+ }
+ try {
+ tagManager.recoverIndex(measurementMNode.getOffset(), measurementMNode);
+ } catch (IOException e) {
+ logger.error(
+ "Failed to recover tagIndex for {} in schemaRegion {}.",
+ storageGroupFullPath + PATH_SEPARATOR + measurementMNode.getFullPath(),
+ schemaRegionId);
+ }
+ };
+ }
+
+ private Consumer<IEntityMNode> deviceInitProcess() {
+ return deviceMNode -> {
+ regionStatistics.addDevice();
+ if (deviceMNode.getSchemaTemplateIdWithState() >= 0) {
+ regionStatistics.activateTemplate(deviceMNode.getSchemaTemplateId());
+ }
+ };
+ }
+
private void flushCallback() {
if (usingMLog && !isRecovering) {
try {
@@ -504,25 +521,8 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
storageGroupFullPath,
schemaRegionId.getId(),
regionStatistics,
- measurementMNode -> {
- regionStatistics.addTimeseries(1L);
- if (measurementMNode.getOffset() == -1) {
- return;
- }
- try {
- tagManager.recoverIndex(measurementMNode.getOffset(), measurementMNode);
- } catch (IOException e) {
- logger.error(
- "Failed to recover tagIndex for {} in schemaRegion {}.",
- storageGroupFullPath + PATH_SEPARATOR + measurementMNode.getFullPath(),
- schemaRegionId);
- }
- },
- deviceMNode -> {
- if (deviceMNode.getSchemaTemplateIdWithState() >= 0) {
- regionStatistics.activateTemplate(deviceMNode.getSchemaTemplateId());
- }
- },
+ measurementInitProcess(),
+ deviceInitProcess(),
tagManager::readTags,
this::flushCallback);
logger.info(
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGTest.java
index 7d2fb985e6..c75e37631a 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGTest.java
@@ -99,6 +99,9 @@ public abstract class MTreeBelowSGTest {
node -> {
// do nothing
},
+ node -> {
+ // do nothing
+ },
0,
new CachedSchemaRegionStatistics(0, new CachedSchemaEngineStatistics()));
} else {
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java
index c494b55b47..b501c01ea2 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java
@@ -199,6 +199,36 @@ public class SchemaStatisticsTest extends AbstractSchemaRegionTest {
Assert.assertEquals(4, engineStatistics.getTotalSeriesNumber());
}
+ @Test
+ public void testDeviceNumStatistics() throws Exception {
+ ISchemaRegion schemaRegion1 = getSchemaRegion("root.sg1", 0);
+ ISchemaRegion schemaRegion2 = getSchemaRegion("root.sg2", 1);
+ ISchemaEngineStatistics engineStatistics =
+ SchemaEngine.getInstance().getSchemaEngineStatistics();
+
+ SchemaRegionTestUtil.createSimpleTimeseriesByList(
+ schemaRegion1, Arrays.asList("root.sg1.d0", "root.sg1.d1.s1", "root.sg1.d1.s2.t1"));
+ SchemaRegionTestUtil.createSimpleTimeseriesByList(
+ schemaRegion2, Arrays.asList("root.sg2.d1.s3", "root.sg2.d2.s1", "root.sg2.d2.s2"));
+ SchemaRegionTestUtil.createSimpleTimeseriesByList(
+ schemaRegion2, Collections.singletonList("root.sg2.s1"));
+ // check series number
+ Assert.assertEquals(3, schemaRegion1.getSchemaRegionStatistics().getDevicesNumber());
+ Assert.assertEquals(3, schemaRegion2.getSchemaRegionStatistics().getDevicesNumber());
+
+ PathPatternTree patternTree = new PathPatternTree();
+ patternTree.appendPathPattern(new PartialPath("root.**.s1"));
+ patternTree.constructTree();
+ Assert.assertTrue(schemaRegion1.constructSchemaBlackList(patternTree) >= 1);
+ Assert.assertTrue(schemaRegion2.constructSchemaBlackList(patternTree) >= 1);
+ schemaRegion1.deleteTimeseriesInBlackList(patternTree);
+ schemaRegion2.deleteTimeseriesInBlackList(patternTree);
+
+ // check series number
+ Assert.assertEquals(2, schemaRegion1.getSchemaRegionStatistics().getDevicesNumber());
+ Assert.assertEquals(2, schemaRegion2.getSchemaRegionStatistics().getDevicesNumber());
+ }
+
@Test
public void testSchemaFileNodeStatistics() throws Exception {
if (testParams.getSchemaEngineMode().equals("Schema_File")) {