You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/12/08 06:30:36 UTC
[iotdb] 01/01: [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rm_TVListAllocator
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5b9e762241dd55961ae7cd83b15ecf08d0db09c4
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Dec 8 14:29:51 2021 +0800
[IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
---
.../engine/memtable/AlignedWritableMemChunk.java | 5 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 20 ++---
.../apache/iotdb/db/rescon/TVListAllocator.java | 98 ----------------------
.../iotdb/db/rescon/TVListAllocatorMBean.java | 24 ------
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 -
.../db/utils/datastructure/AlignedTVList.java | 21 +++--
.../iotdb/db/utils/datastructure/TVList.java | 17 ++--
8 files changed, 36 insertions(+), 156 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index 6cfed35..c3539d2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
@@ -55,7 +54,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
measurementIndexMap.put(schemaList.get(i).getMeasurementId(), i);
dataTypeList.add(schemaList.get(i).getType());
}
- this.list = TVListAllocator.getInstance().allocate(dataTypeList);
+ AlignedTVList.newAlignedList(dataTypeList);
}
public Set<String> getAllMeasurements() {
@@ -339,7 +338,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
@Override
public void release() {
if (list.getReferenceCount() == 0) {
- TVListAllocator.getInstance().release(list);
+ list.clear();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index df107ce..a6c4355 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -42,7 +41,7 @@ public class WritableMemChunk implements IWritableMemChunk {
public WritableMemChunk(IMeasurementSchema schema) {
this.schema = schema;
- this.list = TVListAllocator.getInstance().allocate(schema.getType());
+ this.list = TVList.newList(schema.getType());
}
@Override
@@ -348,7 +347,7 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
public void release() {
if (list.getReferenceCount() == 0) {
- TVListAllocator.getInstance().release(list);
+ list.clear();
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index e52a43d..06b1edf 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -352,14 +352,14 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(
insertRowPlan.getMeasurements()[i], insertRowPlan.getDataTypes()[i]);
- memTableIncrement += TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i]);
+ memTableIncrement += TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i]);
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i])
+ ? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i])
: 0;
}
// TEXT data mem size
@@ -385,14 +385,14 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* insertRowPlan.getDataTypes().length;
- memTableIncrement += AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes());
+ memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes());
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ? AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes())
+ ? AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes())
: 0;
alignedMemChunk =
((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
@@ -484,19 +484,19 @@ public class TsFileProcessor {
memIncrements[2] += ChunkMetadata.calculateRamSize(measurement, dataType);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, measurement);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
long acquireArray =
(end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE))
/ PrimitiveArrayManager.ARRAY_SIZE;
memIncrements[0] +=
- acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemSize(dataType);
+ acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemCost(dataType);
}
}
// TEXT data size
@@ -523,14 +523,14 @@ public class TsFileProcessor {
* ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int currentChunkPointNum =
(int) workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int acquireArray =
(end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE))
@@ -538,7 +538,7 @@ public class TsFileProcessor {
memIncrements[0] +=
acquireArray == 0
? 0
- : acquireArray * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ : acquireArray * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
}
vectorMemChunk =
((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
deleted file mode 100644
index 6d0c4e7..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.rescon;
-
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.exception.StartupException;
-import org.apache.iotdb.db.service.IService;
-import org.apache.iotdb.db.service.JMXService;
-import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
-import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.util.ArrayDeque;
-import java.util.EnumMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-public class TVListAllocator implements TVListAllocatorMBean, IService {
-
- private Map<TSDataType, Queue<TVList>> tvListCache = new EnumMap<>(TSDataType.class);
- private String mbeanName =
- String.format(
- "%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE, getID().getJmxName());
-
- public static TVListAllocator getInstance() {
- return InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
- private static final TVListAllocator INSTANCE = new TVListAllocator();
-
- private InstanceHolder() {}
- }
-
- public synchronized TVList allocate(TSDataType dataType) {
- Queue<TVList> tvLists = tvListCache.computeIfAbsent(dataType, k -> new ArrayDeque<>());
- TVList list = tvLists.poll();
- return list != null ? list : TVList.newList(dataType);
- }
-
- public synchronized AlignedTVList allocate(List<TSDataType> dataTypes) {
- return AlignedTVList.newAlignedList(dataTypes);
- }
-
- public synchronized void release(TVList list) {
- list.clear(tvListCache);
- }
-
- @Override
- public int getNumberOfTVLists() {
- int number = 0;
- for (Queue<TVList> queue : tvListCache.values()) {
- number += queue.size();
- }
- return number;
- }
-
- @Override
- public void start() throws StartupException {
- try {
- JMXService.registerMBean(InstanceHolder.INSTANCE, mbeanName);
- } catch (Exception e) {
- throw new StartupException(this.getID().getName(), e.getMessage());
- }
- }
-
- @Override
- public void stop() {
- JMXService.deregisterMBean(mbeanName);
- for (Queue<TVList> queue : tvListCache.values()) {
- queue.clear();
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.TVLIST_ALLOCATOR_SERVICE;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
deleted file mode 100644
index fed0015..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.rescon;
-
-public interface TVListAllocatorMBean {
-
- int getNumberOfTVLists();
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 32bc1a3..93c4cf2 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -41,7 +41,6 @@ import org.apache.iotdb.db.query.udf.service.UDFClassLoaderManager;
import org.apache.iotdb.db.query.udf.service.UDFRegistrationService;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.rescon.SystemInfo;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.rest.RestService;
import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -120,7 +119,6 @@ public class IoTDB implements IoTDBMBean {
registerManager.register(FlushManager.getInstance());
registerManager.register(MultiFileLogNodeManager.getInstance());
registerManager.register(Measurement.INSTANCE);
- registerManager.register(TVListAllocator.getInstance());
registerManager.register(CacheHitRatioMonitor.getInstance());
registerManager.register(MergeManager.getINSTANCE());
registerManager.register(CompactionTaskManager.getInstance());
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 120c9f8..6f42171 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -34,10 +34,10 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Queue;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public class AlignedTVList extends TVList {
@@ -850,26 +850,29 @@ public class AlignedTVList extends TVList {
}
/**
- * Get the single alignedTVList array size by give types.
+ * Get the single alignedTVList array mem cost by give types.
*
* @param types the types in the vector
* @return AlignedTvListArrayMemSize
*/
- public static long alignedTvListArrayMemSize(TSDataType[] types) {
+ public static long alignedTvListArrayMemCost(TSDataType[] types) {
long size = 0;
- // time size
+ // time array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
- // index size
+ // index array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 4L;
- // value size
+ // value array mem size
for (TSDataType type : types) {
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
}
+ // array headers mem size
+ size += NUM_BYTES_ARRAY_HEADER * (2 + types.length);
+ // Object references size in ArrayList
+ size += NUM_BYTES_OBJECT_REF * (2 + types.length);
return size;
}
- @Override
- public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
+ public void clear() {
size = 0;
sorted = true;
minTime = Long.MAX_VALUE;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 0b2cf6f..ec383ea 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -32,11 +32,11 @@ import org.apache.iotdb.tsfile.utils.BitMap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public abstract class TVList {
@@ -84,12 +84,16 @@ public abstract class TVList {
return null;
}
- public static long tvListArrayMemSize(TSDataType type) {
+ public static long tvListArrayMemCost(TSDataType type) {
long size = 0;
- // time size
+ // time array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
- // value size
+ // value array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
+ // two array headers mem size
+ size += NUM_BYTES_ARRAY_HEADER * 2;
+ // Object references size in ArrayList
+ size += NUM_BYTES_OBJECT_REF * 2;
return size;
}
@@ -283,7 +287,7 @@ public abstract class TVList {
cloneList.minTime = minTime;
}
- public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
+ public void clear() {
size = 0;
sorted = true;
minTime = Long.MAX_VALUE;
@@ -292,7 +296,6 @@ public abstract class TVList {
clearValue();
clearSortedValue();
- tvListCache.get(getDataType()).add(this);
}
protected void clearTime() {