You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/08/04 08:25:23 UTC
[iotdb] branch upgradeMem updated: Optimize map usage
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch upgradeMem
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/upgradeMem by this push:
new d6b1cfa Optimize map usage
d6b1cfa is described below
commit d6b1cfaf30f2986e2261fd6ce30690ba0fb6b899
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Aug 4 16:12:55 2021 +0800
Optimize map usage
---
.../apache/iotdb/db/tools/TsFileRewriteTool.java | 36 +++++++++-------------
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 4 +--
2 files changed, 16 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
index d8e5215..84b075f 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileRewriteTool.java
@@ -294,26 +294,24 @@ public class TsFileRewriteTool implements AutoCloseable {
List<Boolean> needToDecodeInfoInChunk)
throws IOException, PageException {
valueDecoder = Decoder.getDecoderByType(schema.getEncodingType(), schema.getType());
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup = new HashMap<>();
+ Map<Long, ChunkWriterImpl> partitionChunkWriterMap = new HashMap<>();
for (int i = 0; i < pageDataInChunk.size(); i++) {
if (Boolean.TRUE.equals(needToDecodeInfoInChunk.get(i))) {
- decodeAndWritePage(schema, pageDataInChunk.get(i), chunkWritersInChunkGroup);
+ decodeAndWritePage(schema, pageDataInChunk.get(i), partitionChunkWriterMap);
} else {
writePage(
- schema, pageHeadersInChunk.get(i), pageDataInChunk.get(i), chunkWritersInChunkGroup);
+ schema, pageHeadersInChunk.get(i), pageDataInChunk.get(i), partitionChunkWriterMap);
}
}
- for (Entry<Long, Map<MeasurementSchema, ChunkWriterImpl>> entry :
- chunkWritersInChunkGroup.entrySet()) {
+ for (Entry<Long, ChunkWriterImpl> entry : partitionChunkWriterMap.entrySet()) {
long partitionId = entry.getKey();
TsFileIOWriter tsFileIOWriter = partitionWriterMap.get(partitionId);
if (firstChunkInChunkGroup) {
tsFileIOWriter.startChunkGroup(deviceId);
}
// write chunks to their own upgraded tsFiles
- for (IChunkWriter chunkWriter : entry.getValue().values()) {
- chunkWriter.writeToFileWriter(tsFileIOWriter);
- }
+ IChunkWriter chunkWriter = entry.getValue();
+ chunkWriter.writeToFileWriter(tsFileIOWriter);
}
}
@@ -365,42 +363,38 @@ public class TsFileRewriteTool implements AutoCloseable {
MeasurementSchema schema,
PageHeader pageHeader,
ByteBuffer pageData,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
+ Map<Long, ChunkWriterImpl> partitionChunkWriterMap)
throws PageException {
long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime());
getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
- Map<MeasurementSchema, ChunkWriterImpl> chunkWriters =
- chunkWritersInChunkGroup.getOrDefault(partitionId, new HashMap<>());
- ChunkWriterImpl chunkWriter = chunkWriters.getOrDefault(schema, new ChunkWriterImpl(schema));
+ ChunkWriterImpl chunkWriter =
+ partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema));
chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader);
- chunkWriters.put(schema, chunkWriter);
- chunkWritersInChunkGroup.put(partitionId, chunkWriters);
}
protected void decodeAndWritePage(
MeasurementSchema schema,
ByteBuffer pageData,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
+ Map<Long, ChunkWriterImpl> partitionChunkWriterMap)
throws IOException {
valueDecoder.reset();
PageReader pageReader =
new PageReader(pageData, schema.getType(), valueDecoder, defaultTimeDecoder, null);
BatchData batchData = pageReader.getAllSatisfiedPageData();
- rewritePageIntoFiles(batchData, schema, chunkWritersInChunkGroup);
+ rewritePageIntoFiles(batchData, schema, partitionChunkWriterMap);
}
protected void rewritePageIntoFiles(
BatchData batchData,
MeasurementSchema schema,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup) {
+ Map<Long, ChunkWriterImpl> partitionChunkWriterMap) {
while (batchData.hasCurrent()) {
long time = batchData.currentTime();
Object value = batchData.currentValue();
long partitionId = StorageEngine.getTimePartition(time);
- Map<MeasurementSchema, ChunkWriterImpl> chunkWriters =
- chunkWritersInChunkGroup.getOrDefault(partitionId, new HashMap<>());
- ChunkWriterImpl chunkWriter = chunkWriters.getOrDefault(schema, new ChunkWriterImpl(schema));
+ ChunkWriterImpl chunkWriter =
+ partitionChunkWriterMap.computeIfAbsent(partitionId, v -> new ChunkWriterImpl(schema));
getOrDefaultTsFileIOWriter(oldTsFile, partitionId);
switch (schema.getType()) {
case INT32:
@@ -426,8 +420,6 @@ public class TsFileRewriteTool implements AutoCloseable {
String.format("Data type %s is not supported.", schema.getType()));
}
batchData.next();
- chunkWriters.put(schema, chunkWriter);
- chunkWritersInChunkGroup.put(partitionId, chunkWriters);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
index a3222e1..c86b940 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java
@@ -285,13 +285,13 @@ public class TsFileOnlineUpgradeTool extends TsFileRewriteTool {
protected void decodeAndWritePage(
MeasurementSchema schema,
ByteBuffer pageData,
- Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup)
+ Map<Long, ChunkWriterImpl> partitionChunkWriterMap)
throws IOException {
valueDecoder.reset();
PageReaderV2 pageReader =
new PageReaderV2(pageData, schema.getType(), valueDecoder, defaultTimeDecoder, null);
BatchData batchData = pageReader.getAllSatisfiedPageData();
- rewritePageIntoFiles(batchData, schema, chunkWritersInChunkGroup);
+ rewritePageIntoFiles(batchData, schema, partitionChunkWriterMap);
}
/** check if the file to be upgraded has correct magic strings and version number */