You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2020/04/20 13:36:37 UTC
[incubator-iotdb] branch fix_oom created (now fadc454)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch fix_oom
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at fadc454 modify new memtable size estimation and limit the total memoty size of write log to up to allocate for write size / 10
This branch includes the following new commits:
new fadc454 modify new memtable size estimation and limit the total memoty size of write log to up to allocate for write size / 10
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: modify new memtable size estimation and
limit the total memoty size of write log to up to allocate for write size /
10
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch fix_oom
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fadc454475973bed14d4ac991ed6c1086a08ee30
Author: lta <li...@163.com>
AuthorDate: Mon Apr 20 21:29:52 2020 +0800
modify new memtable size estimation and limit the total memoty size of write log to up to allocate for write size / 10
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 11 ++++--
.../iotdb/db/engine/memtable/AbstractMemTable.java | 4 +--
.../engine/storagegroup/StorageGroupProcessor.java | 1 +
.../db/engine/storagegroup/TsFileProcessor.java | 2 ++
.../java/org/apache/iotdb/db/utils/MemUtils.java | 42 +++++++++++-----------
.../db/writelog/node/ExclusiveWriteLogNode.java | 8 +++--
7 files changed, 39 insertions(+), 31 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index a1dee4d..31984d1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -943,7 +943,7 @@ public class IoTDBConfig {
return walBufferSize;
}
- void setWalBufferSize(int walBufferSize) {
+ public void setWalBufferSize(int walBufferSize) {
this.walBufferSize = walBufferSize;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index 2069043..110e477 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -150,6 +150,8 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
if (canAdjust) {
CONFIG.setMaxMemtableNumber(maxMemTableNum);
+ CONFIG.setWalBufferSize(
+ (int) Math.min(Integer.MAX_VALUE, allocateMemoryForWrite / 10 / maxMemTableNum));
CONFIG.setTsFileSizeThreshold(tsFileSizeThreshold);
CONFIG.setMemtableSizeThreshold(memtableSizeInByte);
if (LOGGER.isDebugEnabled() && initialized) {
@@ -217,14 +219,16 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
@Override
public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException {
totalStorageGroup += diff;
- maxMemTableNum += IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
- if(!CONFIG.isEnableParameterAdapter()){
+ maxMemTableNum +=
+ IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
+ if (!CONFIG.isEnableParameterAdapter()) {
CONFIG.setMaxMemtableNumber(maxMemTableNum);
return;
}
if (!tryToAdaptParameters()) {
totalStorageGroup -= diff;
- maxMemTableNum -= IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
+ maxMemTableNum -=
+ IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() * diff;
throw new ConfigAdjusterException(CREATE_STORAGE_GROUP);
}
}
@@ -286,4 +290,5 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
}
}
+
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index f9de3c7..7297c61 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -93,11 +93,11 @@ public abstract class AbstractMemTable implements IMemTable {
Object value = CommonUtils.parseValue(insertPlan.getSchemas()[i].getType(),
insertPlan.getValues()[i]);
+ memSize += MemUtils.getRecordSize(insertPlan.getSchemas()[i].getType(), value);
+
write(insertPlan.getDeviceId(), insertPlan.getMeasurements()[i],
insertPlan.getSchemas()[i], insertPlan.getTime(), value);
}
- long recordSizeInByte = MemUtils.getRecordSize(insertPlan);
- memSize += recordSizeInByte;
} catch (QueryProcessException e) {
throw new WriteProcessException(e.getMessage());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 273148d..d97fdcd 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -44,6 +44,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 7e554cf..a231c36 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
@@ -88,6 +89,7 @@ public class TsFileProcessor {
*/
private volatile boolean shouldClose;
private IMemTable workMemTable;
+
private VersionController versionController;
/**
* this callback is called after the corresponding TsFile is called endFile().
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
index 5229c48..edeb1f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
@@ -19,8 +19,10 @@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.BooleanDataPoint;
@@ -43,29 +45,25 @@ public class MemUtils {
}
/**
- * function for getting the record size.
+ * function for getting the value size.
*/
- public static long getRecordSize(InsertPlan insertPlan) {
- long memSize = 0;
- for (int i = 0; i < insertPlan.getValues().length; i++) {
- switch (insertPlan.getSchemas()[i].getType()) {
- case INT32:
- memSize += 8L + 4L; break;
- case INT64:
- memSize += 8L + 8L; break;
- case FLOAT:
- memSize += 8L + 4L; break;
- case DOUBLE:
- memSize += 8L + 8L; break;
- case BOOLEAN:
- memSize += 8L + 1L; break;
- case TEXT:
- memSize += 8L + insertPlan.getValues()[i].length(); break;
- default:
- memSize += 8L + 8L;
- }
+ public static long getRecordSize(TSDataType dataType, Object value) {
+ switch (dataType) {
+ case INT32:
+ return 8L + 4L;
+ case INT64:
+ return 8L + 8L;
+ case FLOAT:
+ return 8L + 4L;
+ case DOUBLE:
+ return 8L + 8L;
+ case BOOLEAN:
+ return 8L + 1L;
+ case TEXT:
+ return 8L + RamUsageEstimator.sizeOf(value);
+ default:
+ return 8L + 8L;
}
- return memSize;
}
public static long getRecordSize(BatchInsertPlan batchInsertPlan, int start, int end) {
@@ -88,7 +86,7 @@ public class MemUtils {
case TEXT:
memSize += (end - start) * 8L;
for (int j = start; j < end; j++) {
- memSize += ((Binary[]) batchInsertPlan.getColumns()[i])[j].getLength();
+ memSize += RamUsageEstimator.sizeOf(((Binary[]) batchInsertPlan.getColumns()[i])[j]);
}
break;
default:
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index b9b5e0a..dda1a7e 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -46,7 +46,6 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
public static final String WAL_FILE_NAME = "wal";
private static final Logger logger = LoggerFactory.getLogger(ExclusiveWriteLogNode.class);
- private static int logBufferSize = IoTDBDescriptor.getInstance().getConfig().getWalBufferSize();
private String identifier;
@@ -56,7 +55,8 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private ByteBuffer logBuffer = ByteBuffer.allocate(logBufferSize);
+ private ByteBuffer logBuffer = ByteBuffer
+ .allocate(IoTDBDescriptor.getInstance().getConfig().getWalBufferSize());
private ReadWriteLock lock = new ReentrantReadWriteLock();
@@ -88,7 +88,9 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
sync();
}
} catch (BufferOverflowException e) {
- throw new IOException("Log cannot fit into buffer, please increase wal_buffer_size", e);
+ throw new IOException(
+ "Log cannot fit into buffer, if you don't enable Dynamic Parameter Adapter, please increase wal_buffer_size;"
+ + "otherwise, please increase the JVM memory", e);
} finally {
lock.writeLock().unlock();
}