You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/07/15 11:27:53 UTC
[incubator-iotdb] branch dev_merge updated: add more test
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_merge
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/dev_merge by this push:
new 7d3f076 add more test
7d3f076 is described below
commit 7d3f076986a08a6689c9708c4838222e27e35b94
Author: 江天 <jt...@163.com>
AuthorDate: Mon Jul 15 19:25:32 2019 +0800
add more test
---
.../apache/iotdb/db/engine/merge/MergeTask.java | 5 +-
.../iotdb/db/engine/merge/MergePerfTest.java | 127 +++++++++++++++++++++
.../iotdb/db/engine/merge/MergeTaskTest.java | 30 +++++
.../apache/iotdb/db/engine/merge/MergeTest.java | 16 ++-
4 files changed, 171 insertions(+), 7 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
index e38ac7b..d656069 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
@@ -130,7 +130,10 @@ public class MergeTask implements Callable<Void> {
}
private void doMerge() throws IOException {
- logger.info("{} starts", taskName);
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName, seqFiles.size(),
+ unseqFiles.size());
+ }
long startTime = System.currentTimeMillis();
this.mergeLogger = new MergeLogger(storageGroupDir);
logFiles();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java
new file mode 100644
index 0000000..fdf4596
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.junit.Before;
+
+public class MergePerfTest extends MergeTaskTest{
+
+ private int seqFileNum;
+ private int unseqFileNum;
+ private double unseqRatio;
+
+ private Random random = new Random(System.currentTimeMillis());
+
+ private long timeConsumption;
+
+ @Before
+ @Override
+ public void setUp() throws IOException, WriteProcessException {
+ tempSGDir = new File("tempSG");
+ tempSGDir.mkdirs();
+ prepareSeries();
+ prepareFiles();
+ }
+
+ private void prepareFiles() throws IOException, WriteProcessException {
+ for (int i = 0; i < seqFileNum; i++) {
+ File file = new File(i + "seq.tsfile");
+ TsFileResource tsFileResource = new TsFileResource(file);
+ seqResources.add(tsFileResource);
+ prepareFile(tsFileResource, i * ptNum, ptNum, 0);
+ }
+ long timeRange = seqFileNum * ptNum;
+ long unseqLength = (long) (timeRange * unseqRatio);
+ for (int i = 0; i < unseqFileNum; i++) {
+ long unseqOffset = (long) ((1.0 - unseqRatio) * random.nextDouble() * timeRange);
+ System.out.println(unseqOffset + " " + unseqLength);
+ File file = new File(i + "unseq.tsfile");
+ TsFileResource tsFileResource = new TsFileResource(file);
+ unseqResources.add(tsFileResource);
+ prepareFile(tsFileResource, unseqOffset, unseqLength, 10000);
+ }
+ }
+
+ public void test() throws Exception {
+ setUp();
+ timeConsumption = System.currentTimeMillis();
+ MergeTask mergeTask =
+ new MergeTask(seqResources, unseqResources, tempSGDir.getPath(), (k, v, l) -> {}, "test");
+ mergeTask.call();
+ timeConsumption = System.currentTimeMillis() - timeConsumption;
+ tearDown();
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ List<Long> timeConsumptions = new ArrayList<>();
+ MergePerfTest perfTest = new MergePerfTest();
+
+ perfTest.seqFileNum = 5;
+ perfTest.unseqFileNum = 5;
+ perfTest.measurementNum = 100;
+ perfTest.deviceNum = 10;
+ perfTest.unseqRatio = 0.2;
+ perfTest.ptNum = 10000;
+ perfTest.flushInterval = 1000;
+
+ for (int i = 0; i < 3; i++) {
+ // cache warm-up
+ perfTest.test();
+ }
+
+// int[] intParameters = new int[10];
+// for (int i = 1; i <= 10; i++) {
+// intParameters[i-1] = i;
+// }
+// for (int param : intParameters) {
+// perfTest.seqFileNum = param;
+// perfTest.test();
+// timeConsumptions.add(perfTest.timeConsumption);
+// }
+// long[] longParameters = new long[10];
+// for (int i = 1; i <= 10; i++) {
+// longParameters[i-1] = i * 2000;
+// }
+// for (long param : longParameters) {
+// perfTest.ptNum = param;
+// perfTest.test();
+// timeConsumptions.add(perfTest.timeConsumption);
+// }
+ double[] doubleParameters = new double[10];
+ for (int i = 1; i <= 10; i++) {
+ doubleParameters[i-1] = 0.1 * i;
+ }
+ for (double param : doubleParameters) {
+ perfTest.unseqRatio = param;
+ perfTest.test();
+ timeConsumptions.add(perfTest.timeConsumption);
+ }
+
+ System.out.println(timeConsumptions);
+ }
+}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
index 547f19f..2b4d635 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
@@ -25,6 +25,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.sequence.SequenceSeriesReader;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
@@ -49,6 +50,7 @@ public class MergeTaskTest extends MergeTest {
public void tearDown() throws IOException {
super.tearDown();
FileUtils.deleteDirectory(tempSGDir);
+
}
@Test
@@ -118,4 +120,32 @@ public class MergeTaskTest extends MergeTest {
}
tsFilesReader.close();
}
+
+ @Test
+ public void mergeWithDeletionTest() throws Exception {
+ seqResources.get(0).getModFile().write(new Deletion(new Path(deviceIds[0],
+ measurementSchemas[0].getMeasurementId()), 10000, 50));
+ seqResources.get(0).getModFile().close();
+
+
+ MergeTask mergeTask =
+ new MergeTask(seqResources, unseqResources, tempSGDir.getPath(), (k, v, l) -> {}, "test");
+ mergeTask.call();
+
+ QueryContext context = new QueryContext();
+ Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId());
+ SequenceSeriesReader tsFilesReader = new SequenceSeriesReader(path,
+ Collections.singletonList(seqResources.get(0)),
+ null, context);
+ int count = 0;
+ while (tsFilesReader.hasNext()) {
+ BatchData batchData = tsFilesReader.nextBatch();
+ for (int i = 0; i < batchData.length(); i++) {
+ assertEquals(batchData.getTimeByIndex(i) + 20000.0, batchData.getDoubleByIndex(i), 0.001);
+ count ++;
+ }
+ }
+ assertEquals(49, count);
+ tsFilesReader.close();
+ }
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 39e6dd5..8972d9f 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -37,10 +38,10 @@ import org.junit.Before;
abstract class MergeTest {
- private int measurementNum = 10;
- private int deviceNum = 10;
- private long ptNum = 100;
- private long flushInterval = 20;
+ int measurementNum = 10;
+ int deviceNum = 10;
+ long ptNum = 100;
+ long flushInterval = 20;
String[] deviceIds;
MeasurementSchema[] measurementSchemas;
@@ -57,9 +58,11 @@ abstract class MergeTest {
@After
public void tearDown() throws IOException {
removeFiles();
+ seqResources.clear();
+ unseqResources.clear();
}
- private void prepareSeries() {
+ void prepareSeries() {
measurementSchemas = new MeasurementSchema[measurementNum];
for (int i = 0; i < measurementNum; i++) {
measurementSchemas[i] = new MeasurementSchema("sensor" + i, TSDataType.DOUBLE,
@@ -97,9 +100,10 @@ abstract class MergeTest {
for (TsFileResource tsFileResource : unseqResources) {
tsFileResource.remove();
}
+ FileReaderManager.getInstance().stop();
}
- private void prepareFile(TsFileResource tsFileResource, long timeOffset, long ptNum,
+ void prepareFile(TsFileResource tsFileResource, long timeOffset, long ptNum,
long valueOffset)
throws IOException, WriteProcessException {
TsFileWriter fileWriter = new TsFileWriter(tsFileResource.getFile());