You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2023/04/14 04:15:08 UTC
[iotdb] branch rel/1.1 updated: [To rel/1.1] Accelerate count all schema via schema statistic (#9609)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new ae0111a4af [To rel/1.1] Accelerate count all schema via schema statistic (#9609)
ae0111a4af is described below
commit ae0111a4af6f6e463995aa4d2790be51c4c290c7
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Fri Apr 14 12:15:01 2023 +0800
[To rel/1.1] Accelerate count all schema via schema statistic (#9609)
---
.../mtree/store/disk/cache/CacheMemoryManager.java | 2 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 15 +++++---------
.../operator/schema/SchemaCountOperator.java | 23 +++++++++++++---------
.../operator/schema/source/DeviceSchemaSource.java | 12 +++++++++++
.../operator/schema/source/ISchemaSource.java | 4 ++++
.../operator/schema/source/NodeSchemaSource.java | 10 ++++++++++
.../schema/source/PathsUsingTemplateSource.java | 10 ++++++++++
.../schema/source/TimeSeriesSchemaSource.java | 14 +++++++++++++
8 files changed, 70 insertions(+), 20 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/cache/CacheMemoryManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/cache/CacheMemoryManager.java
index b10ea98500..3e1339dceb 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/cache/CacheMemoryManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/cache/CacheMemoryManager.java
@@ -70,7 +70,7 @@ public class CacheMemoryManager {
private IReleaseFlushStrategy releaseFlushStrategy;
- private static final int MAX_WAITING_TIME_WHEN_RELEASING = 10_000;
+ private static final int MAX_WAITING_TIME_WHEN_RELEASING = 3_000;
private final Object blockObject = new Object();
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
index 804447ab87..79bf93e2d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionSchemaFileImpl.java
@@ -581,12 +581,8 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
@Override
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public void createTimeseries(ICreateTimeSeriesPlan plan, long offset) throws MetadataException {
- if (!regionStatistics.isAllowToCreateNewSeries()) {
+ while (!regionStatistics.isAllowToCreateNewSeries()) {
CacheMemoryManager.getInstance().waitIfReleasing();
- if (!regionStatistics.isAllowToCreateNewSeries()) {
- logger.warn("Series overflow when creating: [{}]", plan.getPath().getFullPath());
- throw new SeriesOverflowException();
- }
}
schemaQuotaManager.checkMeasurementLevel(1);
@@ -693,11 +689,8 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
@Override
public void createAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan) throws MetadataException {
int seriesCount = plan.getMeasurements().size();
- if (!regionStatistics.isAllowToCreateNewSeries()) {
+ while (!regionStatistics.isAllowToCreateNewSeries()) {
CacheMemoryManager.getInstance().waitIfReleasing();
- if (!regionStatistics.isAllowToCreateNewSeries()) {
- throw new SeriesOverflowException();
- }
}
schemaQuotaManager.checkMeasurementLevel(seriesCount);
@@ -1181,7 +1174,9 @@ public class SchemaRegionSchemaFileImpl implements ISchemaRegion {
@Override
public void activateSchemaTemplate(IActivateTemplateInClusterPlan plan, Template template)
throws MetadataException {
-
+ while (!regionStatistics.isAllowToCreateNewSeries()) {
+ CacheMemoryManager.getInstance().waitIfReleasing();
+ }
try {
IMNode deviceNode = getDeviceNodeWithAutoCreate(plan.getActivatePath());
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/SchemaCountOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/SchemaCountOperator.java
index d91caee311..74cb161505 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/SchemaCountOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/SchemaCountOperator.java
@@ -76,15 +76,20 @@ public class SchemaCountOperator<T extends ISchemaInfo> implements SourceOperato
isFinished = true;
TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(OUTPUT_DATA_TYPES);
long count = 0;
- if (schemaReader == null) {
- schemaReader = createSchemaReader();
- }
- while (schemaReader.hasNext()) {
- schemaReader.next();
- count++;
- }
- if (!schemaReader.isSuccess()) {
- throw new RuntimeException(schemaReader.getFailure());
+ ISchemaRegion schemaRegion = getSchemaRegion();
+ if (schemaSource.hasSchemaStatistic(schemaRegion)) {
+ count = schemaSource.getSchemaStatistic(schemaRegion);
+ } else {
+ if (schemaReader == null) {
+ schemaReader = createSchemaReader();
+ }
+ while (schemaReader.hasNext()) {
+ schemaReader.next();
+ count++;
+ }
+ if (!schemaReader.isSuccess()) {
+ throw new RuntimeException(schemaReader.getFailure());
+ }
}
tsBlockBuilder.getTimeColumnBuilder().writeLong(0L);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/DeviceSchemaSource.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/DeviceSchemaSource.java
index 2abebdfff9..802f9d54d2 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/DeviceSchemaSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/DeviceSchemaSource.java
@@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.utils.Binary;
import java.util.List;
+import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN;
+
public class DeviceSchemaSource implements ISchemaSource<IDeviceSchemaInfo> {
private final PartialPath pathPattern;
@@ -84,4 +86,14 @@ public class DeviceSchemaSource implements ISchemaSource<IDeviceSchemaInfo> {
}
builder.declarePosition();
}
+
+ @Override
+ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) {
+ return pathPattern.equals(ALL_MATCH_PATTERN);
+ }
+
+ @Override
+ public long getSchemaStatistic(ISchemaRegion schemaRegion) {
+ return schemaRegion.getSchemaRegionStatistics().getDevicesNumber();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/ISchemaSource.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/ISchemaSource.java
index 853cbc6e84..d7a8cd7325 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/ISchemaSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/ISchemaSource.java
@@ -47,4 +47,8 @@ public interface ISchemaSource<T extends ISchemaInfo> {
* @param database the belonged databased of given SchemaInfo
*/
void transformToTsBlockColumns(T schemaInfo, TsBlockBuilder tsBlockBuilder, String database);
+
+ boolean hasSchemaStatistic(ISchemaRegion schemaRegion);
+
+ long getSchemaStatistic(ISchemaRegion schemaRegion);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/NodeSchemaSource.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/NodeSchemaSource.java
index 2a3dbffebc..78b4d8d18f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/NodeSchemaSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/NodeSchemaSource.java
@@ -78,4 +78,14 @@ public class NodeSchemaSource implements ISchemaSource<INodeSchemaInfo> {
.writeBinary(new Binary(String.valueOf(nodeSchemaInfo.getNodeType().getNodeType())));
tsBlockBuilder.declarePosition();
}
+
+ @Override
+ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) {
+ return false;
+ }
+
+ @Override
+ public long getSchemaStatistic(ISchemaRegion schemaRegion) {
+ return 0;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/PathsUsingTemplateSource.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/PathsUsingTemplateSource.java
index ba9b450244..70e612d757 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/PathsUsingTemplateSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/PathsUsingTemplateSource.java
@@ -61,6 +61,16 @@ public class PathsUsingTemplateSource implements ISchemaSource<IDeviceSchemaInfo
builder.declarePosition();
}
+ @Override
+ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) {
+ return false;
+ }
+
+ @Override
+ public long getSchemaStatistic(ISchemaRegion schemaRegion) {
+ return schemaRegion.getSchemaRegionStatistics().getTemplateActivatedNumber();
+ }
+
private class DevicesUsingTemplateReader implements ISchemaReader<IDeviceSchemaInfo> {
final Iterator<PartialPath> pathPatternIterator;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/TimeSeriesSchemaSource.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/TimeSeriesSchemaSource.java
index 1ad90c5f80..6ebe5a1744 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/TimeSeriesSchemaSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/schema/source/TimeSeriesSchemaSource.java
@@ -36,6 +36,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN;
+
public class TimeSeriesSchemaSource implements ISchemaSource<ITimeSeriesSchemaInfo> {
private final PartialPath pathPattern;
@@ -106,6 +108,18 @@ public class TimeSeriesSchemaSource implements ISchemaSource<ITimeSeriesSchemaIn
builder.declarePosition();
}
+ @Override
+ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) {
+ return pathPattern.equals(ALL_MATCH_PATTERN)
+ && key == null
+ && schemaRegion.getSchemaRegionStatistics().getTemplateActivatedNumber() == 0;
+ }
+
+ @Override
+ public long getSchemaStatistic(ISchemaRegion schemaRegion) {
+ return schemaRegion.getSchemaRegionStatistics().getSeriesNumber();
+ }
+
private String mapToString(Map<String, String> map) {
if (map == null || map.isEmpty()) {
return null;