You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/11/05 14:07:25 UTC
[iotdb] branch rel/0.11 updated: fix does not compact unseq files
bug (#1949)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 2b16984 fix does not compact unseq files bug (#1949)
2b16984 is described below
commit 2b1698424b2e82c3a60eb87a56f2e53e815da2b7
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Thu Nov 5 22:02:35 2020 +0800
fix does not compact unseq files bug (#1949)
(cherry picked from commit 55dbbcc149609897fa0e1fc70619cc8467add872)
---
.../compaction/level/LevelCompactionTsFileManagement.java | 13 +++++++++----
.../iotdb/db/engine/compaction/utils/CompactionUtils.java | 4 ++--
.../iotdb/db/engine/cache/ChunkMetadataCacheTest.java | 5 +++++
.../iotdb/db/engine/modification/DeletionFileNodeTest.java | 6 ++++++
.../iotdb/db/engine/modification/DeletionQueryTest.java | 6 ++++++
.../org/apache/iotdb/db/engine/storagegroup/TTLTest.java | 5 ++++-
6 files changed, 32 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index 979c845..cba29a5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -57,7 +57,8 @@ import org.slf4j.LoggerFactory;
*/
public class LevelCompactionTsFileManagement extends TsFileManagement {
- private static final Logger logger = LoggerFactory.getLogger(LevelCompactionTsFileManagement.class);
+ private static final Logger logger = LoggerFactory
+ .getLogger(LevelCompactionTsFileManagement.class);
private final int seqLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum();
private final int seqFileNumInEachLevel = IoTDBDescriptor.getInstance().getConfig()
@@ -390,17 +391,18 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
forkTsFileList(
forkedSequenceTsFileResources,
sequenceTsFileResources.computeIfAbsent(timePartition, this::newSequenceTsFileResources),
- seqLevelNum);
+ seqLevelNum, seqFileNumInEachLevel);
+ // we have to copy all unseq file
forkTsFileList(
forkedUnSequenceTsFileResources,
unSequenceTsFileResources
.computeIfAbsent(timePartition, this::newUnSequenceTsFileResources),
- unseqLevelNum);
+ unseqLevelNum + 1, unseqFileNumInEachLevel);
}
private void forkTsFileList(
List<List<TsFileResource>> forkedTsFileResources,
- List rawTsFileResources, int currMaxLevel) {
+ List rawTsFileResources, int currMaxLevel, int currFileNumInEachLevel) {
forkedTsFileResources.clear();
for (int i = 0; i < currMaxLevel - 1; i++) {
List<TsFileResource> forkedLevelTsFileResources = new ArrayList<>();
@@ -409,6 +411,9 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
for (TsFileResource tsFileResource : levelRawTsFileResources) {
if (tsFileResource.isClosed()) {
forkedLevelTsFileResources.add(tsFileResource);
+ if (forkedLevelTsFileResources.size() > currFileNumInEachLevel) {
+ break;
+ }
}
}
forkedTsFileResources.add(forkedLevelTsFileResources);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
index c2edcbe..a2ef1fe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/CompactionUtils.java
@@ -247,13 +247,13 @@ public class CompactionUtils {
}
}
if (isPageEnoughLarge) {
- logger.info("{} [Compaction] page enough large, use append merge", storageGroup);
+ logger.debug("{} [Compaction] page enough large, use append merge", storageGroup);
// append page in chunks, so we do not have to deserialize a chunk
maxVersion = writeByAppendMerge(maxVersion, device, compactionWriteRateLimiter,
readerChunkMetadatasMap, targetResource, writer);
} else {
logger
- .info("{} [Compaction] page enough large, use deserialize merge", storageGroup);
+ .debug("{} [Compaction] page too small, use deserialize merge", storageGroup);
// we have to deserialize chunks to merge pages
maxVersion = writeByDeserializeMerge(maxVersion, device, compactionWriteRateLimiter,
entry, targetResource, writer);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
index abf89e6..7241405 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
@@ -62,8 +62,12 @@ public class ChunkMetadataCacheTest {
private String systemDir = TestConstant.BASE_OUTPUT_PATH.concat("data")
.concat(File.separator).concat("info");
+ private int prevUnseqLevelNum = 0;
+
@Before
public void setUp() throws Exception {
+ prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2);
EnvironmentUtils.envSetUp();
MetadataManagerHelper.initMetadata();
storageGroupProcessor = new StorageGroupProcessor(systemDir, storageGroup,
@@ -77,6 +81,7 @@ public class ChunkMetadataCacheTest {
storageGroupProcessor.syncDeleteDataFiles();
EnvironmentUtils.cleanEnv();
EnvironmentUtils.cleanDir(systemDir);
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum);
}
private void insertOneRecord(long time, int num)
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 61c66c0..cd9c527 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
@@ -65,6 +66,8 @@ public class DeletionFileNodeTest {
private TSDataType dataType = TSDataType.DOUBLE;
private TSEncoding encoding = TSEncoding.PLAIN;
+ private int prevUnseqLevelNum = 0;
+
static {
for (int i = 0; i < 10; i++) {
measurements[i] = "m" + i;
@@ -73,6 +76,8 @@ public class DeletionFileNodeTest {
@Before
public void setup() throws MetadataException {
+ prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2);
EnvironmentUtils.envSetUp();
IoTDB.metaManager.setStorageGroup(new PartialPath(processorName));
@@ -85,6 +90,7 @@ public class DeletionFileNodeTest {
@After
public void teardown() throws IOException, StorageEngineException {
EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum);
}
private void insertToStorageEngine(TSRecord record)
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index 189d910..8f11698 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
@@ -57,6 +58,8 @@ public class DeletionQueryTest {
private TSEncoding encoding = TSEncoding.PLAIN;
private QueryRouter router = new QueryRouter();
+ private int prevUnseqLevelNum = 0;
+
static {
for (int i = 0; i < 10; i++) {
measurements[i] = "m" + i;
@@ -65,6 +68,8 @@ public class DeletionQueryTest {
@Before
public void setup() throws MetadataException {
+ prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2);
EnvironmentUtils.envSetUp();
IoTDB.metaManager.setStorageGroup(new PartialPath(processorName));
for (int i = 0; i < 10; i++) {
@@ -76,6 +81,7 @@ public class DeletionQueryTest {
@After
public void teardown() throws IOException, StorageEngineException {
EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum);
}
private void insertToStorageEngine(TSRecord record)
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
index 985bd319..45e6619 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TTLTest.java
@@ -81,11 +81,13 @@ public class TTLTest {
private String s1 = "s1";
private String g1s1 = sg1 + IoTDBConstant.PATH_SEPARATOR + s1;
private long prevPartitionInterval;
-
+ private int prevUnseqLevelNum;
@Before
public void setUp()
throws MetadataException, IOException, StartupException, StorageGroupProcessorException {
+ prevUnseqLevelNum = IoTDBDescriptor.getInstance().getConfig().getUnseqLevelNum();
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(2);
prevPartitionInterval = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
EnvironmentUtils.envSetUp();
@@ -97,6 +99,7 @@ public class TTLTest {
storageGroupProcessor.syncCloseAllWorkingTsFileProcessors();
EnvironmentUtils.cleanEnv();
IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(prevPartitionInterval);
+ IoTDBDescriptor.getInstance().getConfig().setUnseqLevelNum(prevUnseqLevelNum);
}
private void insertToStorageGroupProcessor(InsertRowPlan insertPlan)