You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/06/29 03:54:35 UTC
[incubator-iotdb] 01/02: add the statistic for the number of series
in each storage group;
and set the ChunkBufferPool capacity as (core = the maximal number of series
of storage groups, max = 2 * core)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 5e41f43df6b64b6b9d9858a0d2706e9ac02e5d62
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Sat Jun 29 11:52:09 2019 +0800
add the statistic for the number of series in each storage group; and set the ChunkBufferPool capacity as (core = the maximal number of series of storage groups, max = 2 * core)
---
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 40 +++++++++++++++-------
.../java/org/apache/iotdb/db/metadata/MGraph.java | 16 +++++++++
.../org/apache/iotdb/db/metadata/MManager.java | 38 ++++++++++++++++++++
.../iotdb/db/metadata/MManagerBasicTest.java | 24 +++++++++++++
4 files changed, 106 insertions(+), 12 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
index 4b371e4..e40a7cf 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
@@ -17,6 +17,7 @@ package org.apache.iotdb.db.engine.memtable;
import java.util.ArrayDeque;
import java.util.Deque;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
@@ -37,11 +38,18 @@ public class ChunkBufferPool {
private static final Deque<ChunkBuffer> availableChunkBuffer = new ArrayDeque<>();
+// /**
+// * the number of required FlushTasks is no more than {@linkplain MemTablePool}.
+// */
+// private static final int capacity = IoTDBDescriptor.getInstance().getConfig()
+// .getMemtableNumber();
+
/**
- * the number of required FlushTasks is no more than {@linkplain MemTablePool}.
+ * The number of chunkBuffer in the pool is less than the memtable number by default.
+ * Once the maximal number of time series is greater than the capacity, the capacity will be updated
+ * to the maximal number.
*/
- private static final int capacity = IoTDBDescriptor.getInstance().getConfig()
- .getMemtableNumber();
+ //private volatile int capacity = IoTDBDescriptor.getInstance().getConfig().getMemtableNumber();
private int size = 0;
@@ -52,6 +60,10 @@ public class ChunkBufferPool {
public ChunkBuffer getEmptyChunkBuffer(Object applier, MeasurementSchema schema) {
synchronized (availableChunkBuffer) {
+ //we use the memtable number * maximal series number in one StroageGroup * 2 as the capacity
+ int capacity =
+ 2 * MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor
+ .getInstance().getConfig().getMemtableNumber();
if (availableChunkBuffer.isEmpty() && size < capacity) {
size++;
// LOGGER.info("For fask, generated a new ChunkBuffer for {}, system ChunkBuffer size: {}, stack size: {}",
@@ -61,7 +73,7 @@ public class ChunkBufferPool {
// LOGGER
// .info("ReusableChunkBuffer size: {}, stack size: {}, then get a ChunkBuffer from stack for {}",
// size, availableChunkBuffer.size(), applier);
- ChunkBuffer chunkBuffer = availableChunkBuffer.pop();
+ ChunkBuffer chunkBuffer = availableChunkBuffer.pop();
chunkBuffer.reInit(schema);
return chunkBuffer;
}
@@ -80,7 +92,8 @@ public class ChunkBufferPool {
} catch (InterruptedException e) {
LOGGER.error("{} fails to wait fot ReusableChunkBuffer {}, continue to wait", applier, e);
}
- LOGGER.info("{} has waited for a ReusableChunkBuffer for {}ms", applier, waitCount++ * WAIT_TIME);
+ LOGGER.info("{} has waited for a ReusableChunkBuffer for {}ms", applier,
+ waitCount++ * WAIT_TIME);
}
}
}
@@ -88,19 +101,22 @@ public class ChunkBufferPool {
public void putBack(ChunkBuffer chunkBuffer) {
synchronized (availableChunkBuffer) {
chunkBuffer.reset();
- availableChunkBuffer.push(chunkBuffer);
+ //we use the memtable number * maximal series number in one StroageGroup as the capacity
+ int capacity =
+ MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor
+ .getInstance().getConfig().getMemtableNumber();
+ if (size > capacity) {
+ size --;
+ } else {
+ availableChunkBuffer.push(chunkBuffer);
+ }
availableChunkBuffer.notify();
// LOGGER.info("a chunk buffer returned, stack size {}", availableChunkBuffer.size());
}
}
public void putBack(ChunkBuffer chunkBuffer, String storageGroup) {
- synchronized (availableChunkBuffer) {
- chunkBuffer.reset();
- availableChunkBuffer.push(chunkBuffer);
- availableChunkBuffer.notify();
-// LOGGER.info("{} return a chunk buffer, stack size {}", storageGroup, availableChunkBuffer.size());
- }
+ putBack(chunkBuffer);
}
public static ChunkBufferPool getInstance() {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index 70e9b17..b0aea4f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -378,4 +379,19 @@ public class MGraph implements Serializable {
public static String combineMetadataInStrings(String[] metadatas) {
return MTree.combineMetadataInStrings(metadatas);
}
+
+ /**
+ *
+ * @return storage group name -> the series number
+ */
+ public Map<String, Integer> countSeriesNumberInEachStorageGroup() throws PathErrorException {
+ Map<String, Integer> res = new HashMap<>();
+ Set<String> storageGroups = this.getAllStorageGroup();
+ for (String sg : storageGroups) {
+ this.getNumSchemaMapForOneFileNode(sg);
+ MNode node = mtree.getNodeByPath(sg);
+ res.put(sg, node.getLeafCount());
+ }
+ return res;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 6d6c224..0680025 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -76,6 +76,9 @@ public class MManager {
private RandomDeleteCache<String, PathCheckRet> checkAndGetDataTypeCache;
private RandomDeleteCache<String, MNode> mNodeCache;
+ private Map<String, Integer> seriesNumberInStorageGroups = new HashMap<>();
+ private int maxSeriesNumberAmongStorageGroup;
+
private MManager() {
metadataDirPath = IoTDBDescriptor.getInstance().getConfig().getMetadataDir();
if (metadataDirPath.length() > 0
@@ -133,14 +136,23 @@ public class MManager {
lock.writeLock().lock();
File dataFile = new File(datafilePath);
File logFile = new File(logFilePath);
+
try {
if (dataFile.exists()) {
initFromDataFile(dataFile);
} else {
initFromLog(logFile);
}
+ seriesNumberInStorageGroups = mgraph.countSeriesNumberInEachStorageGroup();
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
logWriter = new BufferedWriter(new FileWriter(logFile, true));
writeToLog = true;
+
} catch (PathErrorException | MetadataArgsErrorException
| ClassNotFoundException | IOException e) {
mgraph = new MGraph(ROOT_NAME);
@@ -266,6 +278,13 @@ public class MManager {
lock.writeLock().lock();
try {
mgraph.addPathToMTree(path, dataType, encoding, compressor, props);
+ String storageName = mgraph.getFileNameByPath(path);
+ int size = seriesNumberInStorageGroups.get(mgraph.getFileNameByPath(path));
+ seriesNumberInStorageGroups
+ .put(storageName, size + 1);
+ if (size + 1 > maxSeriesNumberAmongStorageGroup) {
+ maxSeriesNumberAmongStorageGroup = size + 1;
+ }
if (writeToLog) {
initLogStream();
logWriter.write(String.format("%s,%s,%s,%s,%s", MetadataOperationType.ADD_PATH_TO_MTREE,
@@ -316,6 +335,20 @@ public class MManager {
logWriter.newLine();
logWriter.flush();
}
+ String storageGroup = getFileNameByPath(path);
+ int size = seriesNumberInStorageGroups.get(storageGroup);
+ seriesNumberInStorageGroups.put(storageGroup, size - 1);
+ if (size == maxSeriesNumberAmongStorageGroup) {
+ //recalculate
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
+ } else {
+ maxSeriesNumberAmongStorageGroup--;
+ }
return dataFileName;
} finally {
lock.writeLock().unlock();
@@ -332,6 +365,7 @@ public class MManager {
checkAndGetDataTypeCache.clear();
mNodeCache.clear();
mgraph.setStorageLevel(path);
+ seriesNumberInStorageGroups.put(path, 0);
if (writeToLog) {
initLogStream();
logWriter.write(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE + "," + path);
@@ -1025,6 +1059,10 @@ public class MManager {
}
}
+ public int getMaximalSeriesNumberAmongStorageGroups() {
+ return maxSeriesNumberAmongStorageGroup;
+ }
+
private static class MManagerHolder {
private MManagerHolder(){
//allowed to do nothing
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 4fbf8bf..61c08d6 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -344,4 +344,28 @@ public class MManagerBasicTest {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testMaximalSeriesNumberAmongStorageGroup() throws IOException, PathErrorException {
+ MManager manager = MManager.getInstance();
+ assertEquals(0, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.setStorageLevelToMTree("root.laptop");
+ assertEquals(0, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.addPathToMTree("root.laptop.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.addPathToMTree("root.laptop.d1.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ assertEquals(2, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.setStorageLevelToMTree("root.vehicle");
+ manager.addPathToMTree("root.vehicle.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ assertEquals(2, manager.getMaximalSeriesNumberAmongStorageGroups());
+
+ manager.deletePathFromMTree("root.laptop.d1.s1");
+ assertEquals(1, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.deletePathFromMTree("root.laptop.d1.s2");
+ assertEquals(1, manager.getMaximalSeriesNumberAmongStorageGroups());
+
+
+ }
}