You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/11/21 13:33:51 UTC
[incubator-iotdb] branch dev_new_merge_strategy updated: fix
imports and tests
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_new_merge_strategy
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/dev_new_merge_strategy by this push:
new 58ff4e3 fix imports and tests
58ff4e3 is described below
commit 58ff4e3ace9d6ef24282c38101a1bf110e236afd
Author: jt2594838 <jt...@163.com>
AuthorDate: Thu Nov 21 21:33:32 2019 +0800
fix imports and tests
---
.../org/apache/iotdb/db/engine/merge/BaseFileSelector.java | 7 +++++++
.../iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java | 6 +++---
.../apache/iotdb/db/sync/sender/manage/SyncFileManager.java | 6 ++++--
.../org/apache/iotdb/db/engine/merge/MergeOverLapTest.java | 1 -
.../org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java | 10 +++++-----
.../org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java | 6 +++---
.../apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java | 6 +++---
7 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
index 7f38dd6..38ef280 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/BaseFileSelector.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.db.utils.UpgradeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,6 +110,9 @@ public abstract class BaseFileSelector implements IMergeFileSelector{
while (unseqIndex < resource.getUnseqFiles().size() && timeConsumption < timeLimit) {
// select next unseq files
TsFileResource unseqFile = resource.getUnseqFiles().get(unseqIndex);
+ if (UpgradeUtils.isNeedUpgrade(unseqFile)) {
+ continue;
+ }
selectOverlappedSeqFiles(unseqFile);
@@ -179,6 +183,9 @@ public abstract class BaseFileSelector implements IMergeFileSelector{
for (Integer seqFileIdx : tmpSelectedSeqIterable) {
TsFileResource seqFile = resource.getSeqFiles().get(seqFileIdx);
+ if (UpgradeUtils.isNeedUpgrade(seqFile)) {
+ return Long.MAX_VALUE;
+ }
fileReadCost = seqMeasurement.measure(seqFile);
if (fileReadCost > tempMaxSeqFileCost) {
// memory used when read data from a seq file:
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
index 0bcde8c..96d5b68 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/squeeze/task/SqueezeMergeTask.java
@@ -12,7 +12,7 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.MergeCallback;
import org.apache.iotdb.db.engine.merge.squeeze.recover.SqueezeMergeLogger;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.utils.MergeUtils;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
@@ -23,7 +23,7 @@ import org.slf4j.LoggerFactory;
public class SqueezeMergeTask implements Callable<Void> {
- static final String MERGE_SUFFIX = ".merge.squeeze";
+ public static final String MERGE_SUFFIX = ".merge.squeeze";
private static final Logger logger = LoggerFactory.getLogger(SqueezeMergeTask.class);
MergeResource resource;
@@ -65,7 +65,7 @@ public class SqueezeMergeTask implements Callable<Void> {
return null;
}
- private void doMerge() throws IOException, MetadataErrorException {
+ private void doMerge() throws IOException, MetadataException {
if (logger.isInfoEnabled()) {
logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName,
resource.getSeqFiles().size(), resource.getUnseqFiles().size());
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
index bb2532b..98ee75d 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/manage/SyncFileManager.java
@@ -32,7 +32,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.merge.inplace.task.InplaceMergeTask;
+import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.sync.conf.SyncSenderDescriptor;
@@ -116,7 +117,8 @@ public class SyncFileManager implements ISyncFileManager {
}
if (new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists() && !new File(
file.getAbsolutePath() + ModificationFile.FILE_SUFFIX).exists() && !new File(
- file.getAbsolutePath() + MergeTask.MERGE_SUFFIX).exists()) {
+ file.getAbsolutePath() + InplaceMergeTask.MERGE_SUFFIX).exists() && !new File(
+ file.getAbsolutePath() + SqueezeMergeTask.MERGE_SUFFIX).exists()) {
currentSealedLocalFilesMap.get(sgName).add(file);
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
index 0e83e6a..1857598 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeOverLapTest.java
@@ -170,7 +170,6 @@ public class MergeOverLapTest extends MergeTest {
}
}
assertEquals(expected, cnt);
- assertEquals(1000, cnt);
} finally {
tsFilesReader.close();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
index 2fc1be7..073221c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeUpgradeTest.java
@@ -27,8 +27,8 @@ import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.merge.inplace.selector.InplaceMaxFileSelector;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
-import org.apache.iotdb.db.engine.merge.selector.MaxFileMergeFileSelector;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -75,10 +75,10 @@ public class MergeUpgradeTest {
@Test
public void testMergeUpgradeSelect() throws MergeException {
MergeResource resource = new MergeResource(seqResources, unseqResources);
- MaxFileMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource,
- Long.MAX_VALUE);
- List[] result = mergeFileSelector.select();
- assertEquals(0, result.length);
+ IMergeFileSelector mergeFileSelector = new MaxSeriesMergeFileSelector<>(new InplaceMaxFileSelector(resource,
+ Long.MAX_VALUE));
+ mergeFileSelector.select();
+ assertEquals(0, mergeFileSelector.getSelectedSeqFiles().size());
}
private void prepareFiles() throws IOException, WriteProcessException {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
index 45b740e..a988b05 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeLogTest.java
@@ -32,9 +32,9 @@ import org.apache.iotdb.db.engine.merge.MergeTest;
import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
-import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.path.PathException;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +45,7 @@ public class MergeLogTest extends MergeTest {
private File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException, MetadataErrorException, PathErrorException {
+ public void setUp() throws IOException, WriteProcessException, MetadataException, PathException {
super.setUp();
tempSGDir = new File("tempSG");
tempSGDir.mkdirs();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
index 9e63e17..becb3f8 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/squeeze/MergeTaskTest.java
@@ -31,9 +31,9 @@ import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.merge.squeeze.task.SqueezeMergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.exception.MetadataErrorException;
-import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.path.PathException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.resourceRelated.SeqResourceIterateReader;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -48,7 +48,7 @@ public class MergeTaskTest extends MergeTest {
private File tempSGDir;
@Before
- public void setUp() throws IOException, WriteProcessException, MetadataErrorException, PathErrorException {
+ public void setUp() throws IOException, WriteProcessException, MetadataException, PathException {
super.setUp();
tempSGDir = new File("tempSG");
tempSGDir.mkdirs();