You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/07/18 14:32:59 UTC
[iotdb] branch rel/0.13 updated: [To rel/0.13][IOTDB-3822]Fix cross compaction overlap bug (#6692)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new 41f0d42f46 [To rel/0.13][IOTDB-3822]Fix cross compaction overlap bug (#6692)
41f0d42f46 is described below
commit 41f0d42f4605d856b94b1f8b6ebf342e17a49b5b
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Mon Jul 18 22:32:54 2022 +0800
[To rel/0.13][IOTDB-3822]Fix cross compaction overlap bug (#6692)
---
.../selector/RewriteCompactionFileSelector.java | 24 +-
.../db/tools/validate/TsFileValidationTool.java | 198 ++-
.../engine/compaction/AbstractCompactionTest.java | 6 +
.../cross/CrossSpaceCompactionValidationTest.java | 1764 ++++++++++++++++++++
4 files changed, 1942 insertions(+), 50 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
index 57cd06daac..9c0009a4bb 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
@@ -176,6 +176,13 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
break;
}
+ // Filter out the selected seq files
+ for (int i = 0; i < seqSelected.length; i++) {
+ if (seqSelected[i]) {
+ tmpSelectedSeqFiles.remove(i);
+ }
+ }
+
tempMaxSeqFileCost = maxSeqFileCost;
long newCost =
useTightBound
@@ -258,25 +265,20 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
boolean noMoreOverlap = false;
for (int i = 0; i < resource.getSeqFiles().size() && !noMoreOverlap; i++) {
TsFileResource seqFile = resource.getSeqFiles().get(i);
- if (tmpSelectedSeqFiles.contains(i) || !seqFile.mayContainsDevice(deviceId)) {
+ if (!seqFile.mayContainsDevice(deviceId)) {
continue;
}
long seqEndTime = seqFile.getEndTime(deviceId);
long seqStartTime = seqFile.getStartTime(deviceId);
- if (unseqEndTime < seqStartTime) {
- // Suppose the time range in unseq file is 10-20, seq file is 30-40. If this unseq file
- // has no overlapped seq files, then select this seq file. Otherwise, skip this seq file.
- // There is no more overlap later.
- if (tmpSelectedSeqFiles.size() == 0) {
+ if (!seqFile.isClosed()) {
+ // for unclosed file, only select those that overlap with the unseq file
+ if (unseqEndTime >= seqStartTime) {
tmpSelectedSeqFiles.add(i);
}
- noMoreOverlap = true;
- } else if (!seqFile.isClosed()) {
- // we cannot make sure whether unclosed file has overlap or not, so we just add it.
- tmpSelectedSeqFiles.add(i);
} else if (unseqEndTime <= seqEndTime) {
- // if time range in unseq file is 10-20, seq file is 15-25, then select this seq file and
+ // if time range in unseq file is 10-20, seq file is 30-40, or
+ // time range in unseq file is 10-20, seq file is 15-25, then select this seq file and
// there is no more overlap later.
tmpSelectedSeqFiles.add(i);
noMoreOverlap = true;
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java b/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
index fff7ea74c1..09c12e45bc 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/validate/TsFileValidationTool.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.read.reader.page.TimePageReader;
+import org.apache.iotdb.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +50,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@@ -80,7 +83,7 @@ public class TsFileValidationTool {
private static final Logger logger = LoggerFactory.getLogger(TsFileValidationTool.class);
private static final List<File> seqDataDirList = new ArrayList<>();
private static final List<File> fileList = new ArrayList<>();
- private static int badFileNum = 0;
+ public static int badFileNum = 0;
/**
* The form of param is: [path of data dir or tsfile] [-pd = print details or not] [-f = path of
@@ -95,7 +98,9 @@ public class TsFileValidationTool {
if (printToFile) {
pw = new PrintWriter(new FileWriter(outFilePath));
}
- printBoth("Start checking seq files ...");
+ if (printDetails) {
+ printBoth("Start checking seq files ...");
+ }
// check tsfile, which will only check for correctness inside a single tsfile
for (File f : fileList) {
@@ -114,7 +119,9 @@ public class TsFileValidationTool {
if (!checkIsDirectory(sgDir)) {
continue;
}
- printBoth("- Check files in storage group: " + sgDir.getAbsolutePath());
+ if (printDetails) {
+ printBoth("- Check files in storage group: " + sgDir.getAbsolutePath());
+ }
// get data region dirs
File[] dataRegionDirs = sgDir.listFiles();
for (File dataRegionDir : Objects.requireNonNull(dataRegionDirs)) {
@@ -123,7 +130,9 @@ public class TsFileValidationTool {
}
// get time partition dirs and sort them
List<File> timePartitionDirs =
- Arrays.asList(Objects.requireNonNull(dataRegionDir.listFiles()));
+ Arrays.asList(Objects.requireNonNull(dataRegionDir.listFiles())).stream()
+ .filter(file -> Pattern.compile("[0-9]*").matcher(file.getName()).matches())
+ .collect(Collectors.toList());
timePartitionDirs.sort(
(f1, f2) ->
Long.compareUnsigned(Long.parseLong(f1.getName()), Long.parseLong(f2.getName())));
@@ -148,19 +157,24 @@ public class TsFileValidationTool {
}
}
}
- printBoth("Finish checking successfully, totally find " + badFileNum + " bad files.");
+ if (printDetails) {
+ printBoth("Finish checking successfully, totally find " + badFileNum + " bad files.");
+ }
if (printToFile) {
pw.close();
}
}
- private static void findUncorrectFiles(List<File> tsFiles) {
- // measurementID -> [lastTime, endTimeInLastFile]
- Map<String, long[]> measurementLastTime = new HashMap<>();
- // deviceID -> endTime, the endTime of device in the last seq file
- Map<String, Long> deviceEndTime = new HashMap<>();
+ public static void findUncorrectFiles(List<File> tsFiles) {
+ // measurementID -> <fileName, [lastTime, endTimeInLastFile]>
+ Map<String, Pair<String, long[]>> measurementLastTime = new HashMap<>();
+ // deviceID -> <fileName, endTime>, the endTime of device in the last seq file
+ Map<String, Pair<String, Long>> deviceEndTime = new HashMap<>();
+ // fileName -> isBadFile
+ Map<String, Boolean> isBadFileMap = new HashMap<>();
for (File tsFile : tsFiles) {
+ List<String> previousBadFileMsgs = new ArrayList<>();
try {
TsFileResource resource = new TsFileResource(tsFile);
if (!new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists()) {
@@ -172,7 +186,8 @@ public class TsFileValidationTool {
} else {
resource.deserialize();
}
- boolean isBadFile = false;
+ isBadFileMap.put(tsFile.getName(), false);
+
try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
// deviceID -> has checked overlap or not
Map<String, Boolean> hasCheckedDeviceOverlap = new HashMap<>();
@@ -206,7 +221,7 @@ public class TsFileValidationTool {
k -> {
long[] arr = new long[2];
Arrays.fill(arr, Long.MIN_VALUE);
- return arr;
+ return new Pair<>("", arr);
});
Decoder defaultTimeDecoder =
Decoder.getDecoderByType(
@@ -233,20 +248,47 @@ public class TsFileValidationTool {
long[] timeBatch = timePageReader.getNextTimeBatch();
for (int i = 0; i < timeBatch.length; i++) {
long timestamp = timeBatch[i];
- if (timestamp <= measurementLastTime.get(measurementID)[0]) {
+ if (timestamp <= measurementLastTime.get(measurementID).right[0]) {
// find bad file
- if (!isBadFile) {
- printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
- isBadFile = true;
+ if (timestamp <= measurementLastTime.get(measurementID).right[1]) {
+ // overlap between file, then add previous bad file path to list
+ String lastBadFile = measurementLastTime.get(measurementID).left;
+ if (!isBadFileMap.get(lastBadFile)) {
+ if (printDetails) {
+ previousBadFileMsgs.add(
+ "-- Find the bad file "
+ + tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + lastBadFile
+ + ", overlap with later files.");
+ } else {
+ previousBadFileMsgs.add(
+ tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + lastBadFile);
+ }
+ isBadFileMap.put(lastBadFile, true);
+ badFileNum++;
+ }
+ }
+
+ if (!isBadFileMap.get(tsFile.getName())) {
+ if (printDetails) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFileMap.put(tsFile.getName(), true);
badFileNum++;
}
if (printDetails) {
- if (timestamp <= measurementLastTime.get(measurementID)[1]) {
+ if (timestamp <= measurementLastTime.get(measurementID).right[1]) {
if (!hasMeasurementPrintedDetails.get(measurementID)[0]) {
printBoth(
"-------- Timeseries "
+ measurementID
- + " overlap between files");
+ + " overlap between files, with previous file "
+ + measurementLastTime.get(measurementID).left);
hasMeasurementPrintedDetails.get(measurementID)[0] = true;
}
} else if (timestamp
@@ -277,7 +319,7 @@ public class TsFileValidationTool {
}
}
} else {
- measurementLastTime.get(measurementID)[0] = timestamp;
+ measurementLastTime.get(measurementID).right[0] = timestamp;
currentPageEndTime = timestamp;
currentChunkEndTime = timestamp;
}
@@ -293,20 +335,45 @@ public class TsFileValidationTool {
BatchData batchData = pageReader.getAllSatisfiedPageData();
while (batchData.hasCurrent()) {
long timestamp = batchData.currentTime();
- if (timestamp <= measurementLastTime.get(measurementID)[0]) {
+ if (timestamp <= measurementLastTime.get(measurementID).right[0]) {
// find bad file
- if (!isBadFile) {
- printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
- isBadFile = true;
+ if (timestamp <= measurementLastTime.get(measurementID).right[1]) {
+ // overlap between file, then add previous bad file path to list
+ if (!isBadFileMap.get(measurementLastTime.get(measurementID).left)) {
+ if (printDetails) {
+ previousBadFileMsgs.add(
+ "-- Find the bad file "
+ + tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + measurementLastTime.get(measurementID).left
+ + ", overlap with later files.");
+ } else {
+ previousBadFileMsgs.add(
+ tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + measurementLastTime.get(measurementID).left);
+ }
+ badFileNum++;
+ isBadFileMap.put(measurementLastTime.get(measurementID).left, true);
+ }
+ }
+ if (!isBadFileMap.get(tsFile.getName())) {
+ if (printDetails) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFileMap.put(tsFile.getName(), true);
badFileNum++;
}
if (printDetails) {
- if (timestamp <= measurementLastTime.get(measurementID)[1]) {
+ if (timestamp <= measurementLastTime.get(measurementID).right[1]) {
if (!hasMeasurementPrintedDetails.get(measurementID)[0]) {
printBoth(
"-------- Timeseries "
+ measurementID
- + " overlap between files");
+ + " overlap between files, with previous file "
+ + measurementLastTime.get(measurementID).left);
hasMeasurementPrintedDetails.get(measurementID)[0] = true;
}
} else if (timestamp
@@ -337,7 +404,7 @@ public class TsFileValidationTool {
}
}
} else {
- measurementLastTime.get(measurementID)[0] = timestamp;
+ measurementLastTime.get(measurementID).right[0] = timestamp;
currentPageEndTime = timestamp;
currentChunkEndTime = timestamp;
}
@@ -357,23 +424,61 @@ public class TsFileValidationTool {
if (!deviceID.equals("")) {
// record the end time of last device in current file
if (resource.getEndTime(deviceID)
- > deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
- deviceEndTime.put(deviceID, resource.getEndTime(deviceID));
+ > deviceEndTime.computeIfAbsent(
+ deviceID,
+ k -> {
+ return new Pair<>("", Long.MIN_VALUE);
+ })
+ .right) {
+ deviceEndTime.get(deviceID).left = tsFile.getName();
+ deviceEndTime.get(deviceID).right = resource.getEndTime(deviceID);
}
}
ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
deviceID = chunkGroupHeader.getDeviceID();
if (!hasCheckedDeviceOverlap.getOrDefault(deviceID, false)
&& resource.getStartTime(deviceID)
- <= deviceEndTime.getOrDefault(deviceID, Long.MIN_VALUE)) {
+ <= deviceEndTime.computeIfAbsent(
+ deviceID,
+ k -> {
+ return new Pair<>("", Long.MIN_VALUE);
+ })
+ .right) {
// find bad file
- if (!isBadFile) {
- printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
- isBadFile = true;
+ // add prevous bad file msg to list
+ if (!isBadFileMap.get(deviceEndTime.get(deviceID).left)) {
+ if (printDetails) {
+ previousBadFileMsgs.add(
+ "-- Find the bad file "
+ + tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + deviceEndTime.get(deviceID).left
+ + ", overlap with later files.");
+ } else {
+ previousBadFileMsgs.add(
+ tsFile.getParentFile().getAbsolutePath()
+ + File.separator
+ + deviceEndTime.get(deviceID).left);
+ }
+ isBadFileMap.put(deviceEndTime.get(deviceID).left, true);
+ badFileNum++;
+ }
+ // print current file
+ if (!isBadFileMap.get(tsFile.getName())) {
+ if (printDetails) {
+ printBoth("-- Find the bad file " + tsFile.getAbsolutePath());
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFileMap.put(tsFile.getName(), true);
badFileNum++;
}
if (printDetails) {
- printBoth("---- Device " + deviceID + " overlap between files");
+ printBoth(
+ "---- Device "
+ + deviceID
+ + " overlap between files, with previous file "
+ + deviceEndTime.get(deviceID).left);
}
}
hasCheckedDeviceOverlap.put(deviceID, true);
@@ -388,16 +493,31 @@ public class TsFileValidationTool {
// record the end time of each timeseries in current file
for (Map.Entry<String, Long> entry : lashChunkEndTime.entrySet()) {
- Long endTime = Math.max(measurementLastTime.get(entry.getKey())[1], entry.getValue());
- measurementLastTime.get(entry.getKey())[1] = endTime;
+ if (measurementLastTime.get(entry.getKey()).right[1] <= entry.getValue()) {
+ measurementLastTime.get(entry.getKey()).right[1] = entry.getValue();
+ measurementLastTime.get(entry.getKey()).left = tsFile.getName();
+ }
}
}
} catch (Throwable e) {
logger.error("Meet errors in reading file {} , skip it.", tsFile.getAbsolutePath(), e);
- printBoth(
- "-- Meet errors in reading file "
- + tsFile.getAbsolutePath()
- + ", tsfile may be corrupted.");
+ if (!isBadFileMap.get(tsFile.getName())) {
+ if (printDetails) {
+ printBoth(
+ "-- Meet errors in reading file "
+ + tsFile.getAbsolutePath()
+ + ", tsfile may be corrupted.");
+ } else {
+ printBoth(tsFile.getAbsolutePath());
+ }
+ isBadFileMap.put(tsFile.getName(), true);
+ badFileNum++;
+ }
+
+ } finally {
+ for (String msg : previousBadFileMsgs) {
+ printBoth(msg);
+ }
}
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index 8892dc161b..beb2ee910c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -65,6 +65,9 @@ public class AbstractCompactionTest {
private static final int oldPagePointSize =
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
+ private static final int oldMaxCrossCompactionFileNum =
+ IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
+
protected static File STORAGE_GROUP_DIR =
new File(
TestConstant.BASE_OUTPUT_PATH
@@ -260,6 +263,9 @@ public class AbstractCompactionTest {
unseqResources.clear();
IoTDB.metaManager.clear();
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(oldTargetChunkSize);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMaxCrossCompactionCandidateFileNum(oldMaxCrossCompactionFileNum);
TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(oldChunkGroupSize);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(oldPagePointSize);
EnvironmentUtils.cleanEnv();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java
new file mode 100644
index 0000000000..afc41395d8
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java
@@ -0,0 +1,1764 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.compaction.cross;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
+import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource;
+import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector;
+import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector;
+import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
+ TsFileManager tsFileManager =
+ new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath());
+
+ private final String oldThreadName = Thread.currentThread().getName();
+
+ @Before
+ public void setUp() throws IOException, WriteProcessException, MetadataException {
+ super.setUp();
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
+ TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ Thread.currentThread().setName("pool-1-IoTDB-Compaction-1");
+ }
+
+ @After
+ public void tearDown() throws IOException, StorageEngineException {
+ super.tearDown();
+ Thread.currentThread().setName(oldThreadName);
+ FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+ }
+
+ /**
+ * 4 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * Selected seq file index: 3<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void test1() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 10, 10, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 5300, 5300, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(1, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 4 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * Selected seq file index: 3<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void test2() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 5, 10, 1500, 3200, 3200, 100, 100, true, false);
+ createFiles(2, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(1, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 4 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * Selected seq file index: 2, 3<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void test3() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 1500, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 5, 10, 1000, 3100, 3100, 100, 100, true, false);
+ createFiles(2, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6500 ~ 7500<br>
+ * 5 Unseq files: 1500 ~ 2500, 1700 ~ 2000, 2400 ~ 3400, 3300 ~ 4500, 6301 ~ 7301<br>
+ * Notice: the last seq file is unsealed<br>
+ * Selected seq file index: 2, 3<br>
+ * Selected unseq file index: 1, 2, 3, 4
+ */
+ @Test
+ public void test4() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 5, 10, 300, 1700, 1700, 100, 100, true, false);
+ createFiles(1, 5, 10, 1000, 2400, 2400, 100, 100, true, false);
+ createFiles(1, 5, 10, 1200, 3300, 3300, 100, 100, true, false);
+ createFiles(2, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 6500, 6500, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 6301, 6301, 100, 100, true, false);
+ seqResources.get(4).setStatus(TsFileResourceStatus.UNCLOSED);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(4, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+ Assert.assertEquals(result[1].get(2), unseqResources.get(2));
+ Assert.assertEquals(result[1].get(3), unseqResources.get(3));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 7 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400, 5500 ~ 6500, 6600 ~
+ * 7600<br>
+ * 2 Unseq files: 2150 ~ 5450, 1150 ～ 5550<br>
+ * Selected seq file index: 2, 3, 4, 5, 6<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void test5() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(7);
+ createFiles(7, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 3300, 2150, 2150, 100, 100, true, false);
+ createFiles(1, 5, 10, 4400, 1150, 1150, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[0].get(3), seqResources.get(4));
+ Assert.assertEquals(result[0].get(4), seqResources.get(5));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 4 Seq files: 0 ~ 1000, 1100 ~ 4100, 4200 ~ 5200, 5300 ~ 6300<br>
+ * 3 Unseq files: 1500 ~ 2500, 2600 ～ 3600, 2000 ~ 3000<br>
+ * Selected seq file index: 2<br>
+ * Selected unseq file index: 1, 2, 3
+ */
+ @Test
+ public void test6() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 3000, 1100, 1100, 100, 100, true, true);
+ createFiles(2, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(2, 5, 10, 1000, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 5, 10, 1000, 2000, 2000, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(1, result[0].size());
+ Assert.assertEquals(3, result[1].size());
+ for (TsFileResource selectedResource : (List<TsFileResource>) result[0]) {
+ Assert.assertEquals(selectedResource, seqResources.get(1));
+ }
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+ Assert.assertEquals(result[1].get(2), unseqResources.get(2));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 4100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 3 Unseq files: 1500 ~ 2500, 2600 ～ 3600, 2000 ~ 3000, 1200 ~ 5250<br>
+ * Selected seq file index: 2, 3, 4<br>
+ * Selected unseq file index: 1, 2, 3, 4
+ */
+ @Test
+ public void test7() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 3000, 1100, 1100, 100, 100, true, true);
+ createFiles(3, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(2, 5, 10, 1000, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 5, 10, 1000, 2000, 2000, 100, 100, true, false);
+ createFiles(1, 5, 10, 4050, 1200, 1200, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(4, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+ Assert.assertEquals(result[1].get(2), unseqResources.get(2));
+ Assert.assertEquals(result[1].get(3), unseqResources.get(3));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 4 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300<br>
+ * 4 Unseq files: 1500 ~ 2500, 1700 ~ 2000, 2400 ~ 3400, 3300 ~ 4500<br>
+ * Selected seq file index: 2, 3<br>
+ * Selected unseq file index: 1, 2, 3, 4
+ */
+ @Test
+ public void test8() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 5, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 5, 10, 300, 1700, 1700, 100, 100, true, false);
+ createFiles(1, 5, 10, 1000, 2400, 2400, 100, 100, true, false);
+ createFiles(1, 5, 10, 1200, 3300, 3300, 100, 100, true, false);
+ createFiles(2, 5, 10, 1000, 4200, 4200, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(4, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+ Assert.assertEquals(result[1].get(2), unseqResources.get(2));
+ Assert.assertEquals(result[1].get(3), unseqResources.get(3));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile10() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile11() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile12() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile20() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile21() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile22() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, true, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, true, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, true, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile30() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, true, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile31() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, true, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, true, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile32() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, true, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, true, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, true, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, true, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile50() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, true, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, true, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, true, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 6600, 6600, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 7700, 7700, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, true, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile51() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, true, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, true, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, true, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, true, true);
+ createFiles(1, 10, 5, 1000, 6600, 6600, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 7700, 7700, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, true, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7, 8<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile52() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, true, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, true, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, true, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 6600, 6600, 100, 100, true, true);
+ createFiles(1, 10, 7, 1000, 7700, 7700, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, true, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(6, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+ Assert.assertEquals(result[0].get(5), seqResources.get(7));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 8<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewAlignedDeviceAndSensorInUnseqFile53() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, true, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, true, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, true, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, true, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, true, true);
+ createFiles(1, 5, 2, 1000, 5500, 5500, 100, 100, true, true);
+ createFiles(1, 5, 10, 1000, 6600, 6600, 100, 100, true, true);
+ createFiles(1, 10, 7, 1000, 7700, 7700, 100, 100, true, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, true, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, true, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(7));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile10() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile11() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4200<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile12() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile20() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile21() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 2100 ~ 3100, 3200 ~ 4700<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [10,10], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile22() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(2, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2100, 2100, 100, 100, false, false);
+ createFiles(1, 10, 10, 1500, 3200, 3200, 100, 100, false, false);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, false, true);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,10], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile30() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(2, 10, 10, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, false, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, false, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,5], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile31() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 10, 5, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 6400, 6400, 100, 100, false, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, false, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, false, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * [Time range]:<br>
+ * 6 Seq files: 0 ~ 1000, 1100 ~ 2100, 4200 ~ 5200, 5300 ~ 6300, 6400 ~ 7400, 7500 ~ 8500<br>
+ * 2 Unseq files: 1500 ~ 3000, 3100 ~ 4100<br>
+ * [DeviceNum, SensorNum]:<br>
+ * seq files: [10,10], [5,5], [5,5], [5,10], [10,7], [10,10]<br>
+ * unseq files: [10,10], [10,10]<br>
+ * Selected seq file index: 2, 3, 5<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile32() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 4200, 4200, 100, 100, false, true);
+ createFiles(1, 5, 10, 1000, 5300, 5300, 100, 100, false, true);
+ createFiles(1, 10, 7, 1000, 6400, 6400, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 7500, 7500, 100, 100, false, true);
+ createFiles(1, 10, 10, 1500, 1500, 1500, 100, 100, false, false);
+ createFiles(1, 10, 10, 1000, 3100, 3100, 100, 100, false, false);
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(3, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+
+ Assert.assertEquals(result[0].get(0), seqResources.get(1));
+ Assert.assertEquals(result[0].get(1), seqResources.get(2));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile50() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, false, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, false, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, false, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 6600, 6600, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 7700, 7700, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, false, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile51() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, false, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, false, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, false, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, false, true);
+ createFiles(1, 10, 5, 1000, 6600, 6600, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 7700, 7700, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, false, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 7, 8<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile52() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, false, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, false, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, false, true);
+ createFiles(1, 2, 2, 1000, 5500, 5500, 100, 100, false, true);
+ createFiles(1, 5, 10, 1000, 6600, 6600, 100, 100, false, true);
+ createFiles(1, 10, 7, 1000, 7700, 7700, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, false, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(6, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(6));
+ Assert.assertEquals(result[0].get(5), seqResources.get(7));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 9 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400,......<br>
+ * 1 Unseq files: 2150 ~ 5450<br>
+ * Selected seq file index: 3, 4, 5, 6, 8<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithNewDeviceAndSensorInUnseqFile53() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(1, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 6, 6, 1000, 1100, 1100, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 2200, 2200, 100, 100, false, true);
+ createFiles(1, 4, 4, 1000, 3300, 3300, 100, 100, false, true);
+ createFiles(1, 3, 3, 1000, 4400, 4400, 100, 100, false, true);
+ createFiles(1, 5, 2, 1000, 5500, 5500, 100, 100, false, true);
+ createFiles(1, 5, 10, 1000, 6600, 6600, 100, 100, false, true);
+ createFiles(1, 10, 7, 1000, 7700, 7700, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 8800, 8800, 100, 100, false, true);
+ createFiles(1, 10, 10, 3300, 2150, 2150, 100, 100, false, false);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(5, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[0].get(2), seqResources.get(4));
+ Assert.assertEquals(result[0].get(3), seqResources.get(5));
+ Assert.assertEquals(result[0].get(4), seqResources.get(7));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400<br>
+ * 1 Unseq files: 2500 ~ 3500<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithUnclosedSeqFile() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(5, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2500, 2500, 100, 100, false, false);
+
+ TsFileResource unclosedSeqResource = new TsFileResource(seqResources.get(4).getTsFile());
+ unclosedSeqResource.setStatus(TsFileResourceStatus.UNCLOSED);
+ TsFileResource lastSeqResource = seqResources.get(4);
+ for (String deviceID : lastSeqResource.getDevices()) {
+ unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID));
+ }
+ seqResources.remove(4);
+ seqResources.add(4, unclosedSeqResource);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400<br>
+ * 1 Unseq files: 2500 ~ 3500<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithUnclosedSeqFileAndNewSensorInUnseqFile() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(3, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 5, 5, 1000, 3300, 3300, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 4400, 4400, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2500, 2500, 100, 100, false, false);
+
+ TsFileResource unclosedSeqResource = new TsFileResource(seqResources.get(4).getTsFile());
+ unclosedSeqResource.setStatus(TsFileResourceStatus.UNCLOSED);
+ TsFileResource lastSeqResource = seqResources.get(4);
+ for (String deviceID : lastSeqResource.getDevices()) {
+ unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID));
+ }
+ seqResources.remove(4);
+ seqResources.add(4, unclosedSeqResource);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400<br>
+ * 2 Unseq files: 2500 ~ 3500, 1500 ~ 4500<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testUnseqFileOverlapWithUnclosedSeqFile() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(5, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2500, 2500, 100, 100, false, false);
+ createFiles(1, 5, 5, 3000, 1500, 1500, 100, 100, false, false);
+
+ TsFileResource unclosedSeqResource = new TsFileResource(seqResources.get(4).getTsFile());
+ unclosedSeqResource.setStatus(TsFileResourceStatus.UNCLOSED);
+ TsFileResource lastSeqResource = seqResources.get(4);
+ for (String deviceID : lastSeqResource.getDevices()) {
+ unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID));
+ }
+ seqResources.remove(4);
+ seqResources.add(4, unclosedSeqResource);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400<br>
+ * 2 Unseq files: 2500 ~ 3500, 4310 ~ 4360<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1, 2
+ */
+ @Test
+ public void testUnseqFileOverlapWithUnclosedSeqFile2() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(5, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2500, 2500, 100, 100, false, false);
+ createFiles(1, 5, 5, 50, 4310, 4310, 100, 100, false, false);
+
+ TsFileResource unclosedSeqResource = new TsFileResource(seqResources.get(4).getTsFile());
+ unclosedSeqResource.setStatus(TsFileResourceStatus.UNCLOSED);
+ TsFileResource lastSeqResource = seqResources.get(4);
+ for (String deviceID : lastSeqResource.getDevices()) {
+ unclosedSeqResource.updateStartTime(deviceID, lastSeqResource.getStartTime(deviceID));
+ }
+ seqResources.remove(4);
+ seqResources.add(4, unclosedSeqResource);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(2, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+ Assert.assertEquals(result[1].get(1), unseqResources.get(1));
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ /**
+ * 5 Seq files: 0 ~ 1000, 1100 ~ 2100, 2200 ~ 3200, 3300 ~ 4300, 4400 ~ 5400<br>
+ * 2 Unseq files: 2500 ~ 3500, 1500 ~ 4500<br>
+ * Selected seq file index: 3, 4<br>
+ * Selected unseq file index: 1
+ */
+ @Test
+ public void testWithUnclosedUnSeqFile() throws Exception {
+ registerTimeseriesInMManger(5, 10, true);
+ createFiles(5, 10, 10, 1000, 0, 0, 100, 100, false, true);
+ createFiles(1, 10, 10, 1000, 2500, 2500, 100, 100, false, false);
+ createFiles(1, 5, 5, 3000, 1500, 1500, 100, 100, false, false);
+
+ TsFileResource unclosedUnSeqResource = new TsFileResource(unseqResources.get(1).getTsFile());
+ unclosedUnSeqResource.setStatus(TsFileResourceStatus.UNCLOSED);
+ TsFileResource lastUnSeqResource = unseqResources.get(1);
+ for (String deviceID : lastUnSeqResource.getDevices()) {
+ unclosedUnSeqResource.updateStartTime(deviceID, lastUnSeqResource.getStartTime(deviceID));
+ unclosedUnSeqResource.updateEndTime(deviceID, lastUnSeqResource.getEndTime(deviceID));
+ }
+ unseqResources.remove(1);
+ unseqResources.add(1, unclosedUnSeqResource);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(2, result.length);
+ Assert.assertEquals(2, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ Assert.assertEquals(result[0].get(0), seqResources.get(2));
+ Assert.assertEquals(result[0].get(1), seqResources.get(3));
+ Assert.assertEquals(result[1].get(0), unseqResources.get(0));
+
+ new RewriteCrossSpaceCompactionTask(
+ "0", COMPACTION_TEST_SG, 0, tsFileManager, result[0], result[1], new AtomicInteger(0))
+ .call();
+
+ validateSeqFiles();
+ }
+
+ private void validateSeqFiles() {
+ List<File> files = new ArrayList<>();
+ for (TsFileResource resource : tsFileManager.getTsFileList(true)) {
+ files.add(resource.getTsFile());
+ }
+ TsFileValidationTool.findUncorrectFiles(files);
+ Assert.assertEquals(0, TsFileValidationTool.badFileNum);
+ }
+}