You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/12/08 06:30:35 UTC
[iotdb] branch rm_TVListAllocator created (now 5b9e762)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rm_TVListAllocator
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 5b9e762 [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
This branch includes the following new commits:
new 5b9e762 [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rm_TVListAllocator
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5b9e762241dd55961ae7cd83b15ecf08d0db09c4
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Dec 8 14:29:51 2021 +0800
[IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost
---
.../engine/memtable/AlignedWritableMemChunk.java | 5 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 20 ++---
.../apache/iotdb/db/rescon/TVListAllocator.java | 98 ----------------------
.../iotdb/db/rescon/TVListAllocatorMBean.java | 24 ------
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 -
.../db/utils/datastructure/AlignedTVList.java | 21 +++--
.../iotdb/db/utils/datastructure/TVList.java | 17 ++--
8 files changed, 36 insertions(+), 156 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index 6cfed35..c3539d2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
@@ -55,7 +54,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
measurementIndexMap.put(schemaList.get(i).getMeasurementId(), i);
dataTypeList.add(schemaList.get(i).getType());
}
- this.list = TVListAllocator.getInstance().allocate(dataTypeList);
+ AlignedTVList.newAlignedList(dataTypeList);
}
public Set<String> getAllMeasurements() {
@@ -339,7 +338,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
@Override
public void release() {
if (list.getReferenceCount() == 0) {
- TVListAllocator.getInstance().release(list);
+ list.clear();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index df107ce..a6c4355 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.engine.memtable;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -42,7 +41,7 @@ public class WritableMemChunk implements IWritableMemChunk {
public WritableMemChunk(IMeasurementSchema schema) {
this.schema = schema;
- this.list = TVListAllocator.getInstance().allocate(schema.getType());
+ this.list = TVList.newList(schema.getType());
}
@Override
@@ -348,7 +347,7 @@ public class WritableMemChunk implements IWritableMemChunk {
@Override
public void release() {
if (list.getReferenceCount() == 0) {
- TVListAllocator.getInstance().release(list);
+ list.clear();
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index e52a43d..06b1edf 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -352,14 +352,14 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(
insertRowPlan.getMeasurements()[i], insertRowPlan.getDataTypes()[i]);
- memTableIncrement += TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i]);
+ memTableIncrement += TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i]);
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i])
+ ? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i])
: 0;
}
// TEXT data mem size
@@ -385,14 +385,14 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* insertRowPlan.getDataTypes().length;
- memTableIncrement += AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes());
+ memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes());
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ? AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes())
+ ? AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes())
: 0;
alignedMemChunk =
((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
@@ -484,19 +484,19 @@ public class TsFileProcessor {
memIncrements[2] += ChunkMetadata.calculateRamSize(measurement, dataType);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, measurement);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
long acquireArray =
(end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE))
/ PrimitiveArrayManager.ARRAY_SIZE;
memIncrements[0] +=
- acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemSize(dataType);
+ acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemCost(dataType);
}
}
// TEXT data size
@@ -523,14 +523,14 @@ public class TsFileProcessor {
* ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int currentChunkPointNum =
(int) workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int acquireArray =
(end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE))
@@ -538,7 +538,7 @@ public class TsFileProcessor {
memIncrements[0] +=
acquireArray == 0
? 0
- : acquireArray * AlignedTVList.alignedTvListArrayMemSize(dataTypes);
+ : acquireArray * AlignedTVList.alignedTvListArrayMemCost(dataTypes);
}
vectorMemChunk =
((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId))
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
deleted file mode 100644
index 6d0c4e7..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.rescon;
-
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.exception.StartupException;
-import org.apache.iotdb.db.service.IService;
-import org.apache.iotdb.db.service.JMXService;
-import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
-import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.util.ArrayDeque;
-import java.util.EnumMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-public class TVListAllocator implements TVListAllocatorMBean, IService {
-
- private Map<TSDataType, Queue<TVList>> tvListCache = new EnumMap<>(TSDataType.class);
- private String mbeanName =
- String.format(
- "%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE, getID().getJmxName());
-
- public static TVListAllocator getInstance() {
- return InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
- private static final TVListAllocator INSTANCE = new TVListAllocator();
-
- private InstanceHolder() {}
- }
-
- public synchronized TVList allocate(TSDataType dataType) {
- Queue<TVList> tvLists = tvListCache.computeIfAbsent(dataType, k -> new ArrayDeque<>());
- TVList list = tvLists.poll();
- return list != null ? list : TVList.newList(dataType);
- }
-
- public synchronized AlignedTVList allocate(List<TSDataType> dataTypes) {
- return AlignedTVList.newAlignedList(dataTypes);
- }
-
- public synchronized void release(TVList list) {
- list.clear(tvListCache);
- }
-
- @Override
- public int getNumberOfTVLists() {
- int number = 0;
- for (Queue<TVList> queue : tvListCache.values()) {
- number += queue.size();
- }
- return number;
- }
-
- @Override
- public void start() throws StartupException {
- try {
- JMXService.registerMBean(InstanceHolder.INSTANCE, mbeanName);
- } catch (Exception e) {
- throw new StartupException(this.getID().getName(), e.getMessage());
- }
- }
-
- @Override
- public void stop() {
- JMXService.deregisterMBean(mbeanName);
- for (Queue<TVList> queue : tvListCache.values()) {
- queue.clear();
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.TVLIST_ALLOCATOR_SERVICE;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
deleted file mode 100644
index fed0015..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.rescon;
-
-public interface TVListAllocatorMBean {
-
- int getNumberOfTVLists();
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 32bc1a3..93c4cf2 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -41,7 +41,6 @@ import org.apache.iotdb.db.query.udf.service.UDFClassLoaderManager;
import org.apache.iotdb.db.query.udf.service.UDFRegistrationService;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.rescon.SystemInfo;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.rest.RestService;
import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -120,7 +119,6 @@ public class IoTDB implements IoTDBMBean {
registerManager.register(FlushManager.getInstance());
registerManager.register(MultiFileLogNodeManager.getInstance());
registerManager.register(Measurement.INSTANCE);
- registerManager.register(TVListAllocator.getInstance());
registerManager.register(CacheHitRatioMonitor.getInstance());
registerManager.register(MergeManager.getINSTANCE());
registerManager.register(CompactionTaskManager.getInstance());
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 120c9f8..6f42171 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -34,10 +34,10 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Queue;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public class AlignedTVList extends TVList {
@@ -850,26 +850,29 @@ public class AlignedTVList extends TVList {
}
/**
- * Get the single alignedTVList array size by give types.
+ * Get the single alignedTVList array mem cost by give types.
*
* @param types the types in the vector
* @return AlignedTvListArrayMemSize
*/
- public static long alignedTvListArrayMemSize(TSDataType[] types) {
+ public static long alignedTvListArrayMemCost(TSDataType[] types) {
long size = 0;
- // time size
+ // time array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
- // index size
+ // index array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 4L;
- // value size
+ // value array mem size
for (TSDataType type : types) {
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
}
+ // array headers mem size
+ size += NUM_BYTES_ARRAY_HEADER * (2 + types.length);
+ // Object references size in ArrayList
+ size += NUM_BYTES_OBJECT_REF * (2 + types.length);
return size;
}
- @Override
- public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
+ public void clear() {
size = 0;
sorted = true;
minTime = Long.MAX_VALUE;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 0b2cf6f..ec383ea 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -32,11 +32,11 @@ import org.apache.iotdb.tsfile.utils.BitMap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public abstract class TVList {
@@ -84,12 +84,16 @@ public abstract class TVList {
return null;
}
- public static long tvListArrayMemSize(TSDataType type) {
+ public static long tvListArrayMemCost(TSDataType type) {
long size = 0;
- // time size
+ // time array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
- // value size
+ // value array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize();
+ // two array headers mem size
+ size += NUM_BYTES_ARRAY_HEADER * 2;
+ // Object references size in ArrayList
+ size += NUM_BYTES_OBJECT_REF * 2;
return size;
}
@@ -283,7 +287,7 @@ public abstract class TVList {
cloneList.minTime = minTime;
}
- public void clear(Map<TSDataType, Queue<TVList>> tvListCache) {
+ public void clear() {
size = 0;
sorted = true;
minTime = Long.MAX_VALUE;
@@ -292,7 +296,6 @@ public abstract class TVList {
clearValue();
clearSortedValue();
- tvListCache.get(getDataType()).add(this);
}
protected void clearTime() {