You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/06/29 03:26:46 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated:
refactor TVList allocator: store by datatype
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature_async_close_tsfile by this push:
new d8c9117 refactor TVList allocator: store by datatype
new 986a0cc Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
d8c9117 is described below
commit d8c911743654c3b52255f65be9608af142f71e20
Author: jt <jt...@163.com>
AuthorDate: Sat Jun 29 11:25:51 2019 +0800
refactor TVList allocator: store by datatype
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 14 ++++-----
.../iotdb/db/engine/memtable/EmptyMemTable.java | 2 +-
.../db/engine/memtable/PrimitiveMemTable.java | 4 +--
.../db/utils/datastructure/TVListAllocator.java | 34 +++++++++++++++++-----
4 files changed, 36 insertions(+), 18 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 8c83bd2..a86fbd5 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -84,12 +84,12 @@ public abstract class AbstractMemTable implements IMemTable {
}
Map<String, IWritableMemChunk> memSeries = memTableMap.get(deviceId);
if (!memSeries.containsKey(measurement)) {
- memSeries.put(measurement, genMemSeries(dataType, deviceId + PATH_SEPARATOR + measurement));
+ memSeries.put(measurement, genMemSeries(dataType));
}
return memSeries.get(measurement);
}
- protected abstract IWritableMemChunk genMemSeries(TSDataType dataType, String path);
+ protected abstract IWritableMemChunk genMemSeries(TSDataType dataType);
@Override
@@ -189,7 +189,7 @@ public abstract class AbstractMemTable implements IMemTable {
if (chunk == null) {
return true;
}
- IWritableMemChunk newChunk = filterChunk(chunk, timestamp, deviceId + PATH_SEPARATOR + measurementId);
+ IWritableMemChunk newChunk = filterChunk(chunk, timestamp);
if (newChunk != null) {
deviceMap.put(measurementId, newChunk);
return newChunk.count() != chunk.count();
@@ -212,12 +212,12 @@ public abstract class AbstractMemTable implements IMemTable {
* @return A reduced copy of chunk if chunk contains data with timestamp less than 'timestamp', of
* null.
*/
- private IWritableMemChunk filterChunk(IWritableMemChunk chunk, long timestamp, String path) {
+ private IWritableMemChunk filterChunk(IWritableMemChunk chunk, long timestamp) {
if (!chunk.isEmpty() && chunk.getMinTime() <= timestamp) {
List<TimeValuePair> timeValuePairs = chunk.getSortedTimeValuePairList();
TSDataType dataType = chunk.getType();
- IWritableMemChunk newChunk = genMemSeries(dataType, path);
+ IWritableMemChunk newChunk = genMemSeries(dataType);
for (TimeValuePair pair : timeValuePairs) {
if (pair.getTimestamp() > timestamp) {
switch (dataType) {
@@ -244,7 +244,7 @@ public abstract class AbstractMemTable implements IMemTable {
}
}
}
- allocator.release(path, chunk.getTVList());
+ allocator.release(dataType, chunk.getTVList());
return newChunk;
}
return null;
@@ -278,7 +278,7 @@ public abstract class AbstractMemTable implements IMemTable {
public void release() {
for (Entry<String, Map<String, IWritableMemChunk>> entry: memTableMap.entrySet()) {
for (Entry<String, IWritableMemChunk> subEntry: entry.getValue().entrySet()) {
- allocator.release(entry.getKey() + IoTDBConstant.PATH_SEPARATOR + subEntry.getKey(), subEntry.getValue().getTVList());
+ allocator.release(subEntry.getValue().getTVList());
}
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/EmptyMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/EmptyMemTable.java
index 631fb30..75af273 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/EmptyMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/EmptyMemTable.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
public class EmptyMemTable extends AbstractMemTable {
@Override
- protected IWritableMemChunk genMemSeries(TSDataType dataType, String path) {
+ protected IWritableMemChunk genMemSeries(TSDataType dataType) {
return null;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
index 5f55b45..267b8d7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
@@ -34,8 +34,8 @@ public class PrimitiveMemTable extends AbstractMemTable {
}
@Override
- protected IWritableMemChunk genMemSeries(TSDataType dataType, String path) {
- return new WritableMemChunkV2(dataType, allocator.allocate(path, dataType));
+ protected IWritableMemChunk genMemSeries(TSDataType dataType) {
+ return new WritableMemChunkV2(dataType, allocator.allocate(dataType));
}
@Override
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/datastructure/TVListAllocator.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/datastructure/TVListAllocator.java
index cdf387e..0b8acac 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/datastructure/TVListAllocator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/datastructure/TVListAllocator.java
@@ -19,24 +19,42 @@
package org.apache.iotdb.db.utils.datastructure;
+import java.util.ArrayDeque;
+import java.util.EnumMap;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.Queue;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
public class TVListAllocator {
- private Map<String, ConcurrentLinkedQueue<TVList>> tvListCache = new ConcurrentHashMap<>();
+ private Map<TSDataType, Queue<TVList>> tvListCache = new EnumMap<>(TSDataType.class);
- public TVList allocate(String identifier, TSDataType dataType) {
- ConcurrentLinkedQueue<TVList> tvLists = tvListCache.computeIfAbsent(identifier,
- k -> new ConcurrentLinkedQueue<>());
+ public synchronized TVList allocate(TSDataType dataType) {
+ Queue<TVList> tvLists = tvListCache.computeIfAbsent(dataType,
+ k -> new ArrayDeque<>());
TVList list = tvLists.poll();
return list != null ? list : TVList.newList(dataType);
}
- public void release(String identifier, TVList list) {
+ public synchronized void release(TSDataType dataType, TVList list) {
list.reset();
- tvListCache.get(identifier).add(list);
+ tvListCache.get(dataType).add(list);
+ }
+
+ public synchronized void release(TVList list) {
+ list.reset();
+ if (list instanceof BinaryTVList) {
+ tvListCache.get(TSDataType.TEXT).add(list);
+ } else if (list instanceof BooleanTVList) {
+ tvListCache.get(TSDataType.BOOLEAN).add(list);
+ } else if (list instanceof DoubleTVList) {
+ tvListCache.get(TSDataType.DOUBLE).add(list);
+ } else if (list instanceof FloatTVList) {
+ tvListCache.get(TSDataType.FLOAT).add(list);
+ } else if (list instanceof IntTVList) {
+ tvListCache.get(TSDataType.INT32).add(list);
+ } else if (list instanceof LongTVList) {
+ tvListCache.get(TSDataType.INT64).add(list);
+ }
}
}