You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/25 04:05:27 UTC
[incubator-iotdb] 01/03: add test and log in memtable flush task
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit bc334d13455f5eecf739ec67a7a1a1985c680ee3
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Jun 25 11:59:12 2019 +0800
add test and log in memtable flush task
---
.../db/engine/memtable/MemTableFlushTaskV2.java | 12 +++
.../filenodeV2/FileNodeProcessorV2Test2.java | 109 +++++++++++++++++++++
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 68 +++++++++++++
3 files changed, 189 insertions(+)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
index 13c45a5..b8825d1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
@@ -60,6 +60,7 @@ public class MemTableFlushTaskV2 {
private Thread memoryFlushThread = new Thread(() -> {
long memSerializeTime = 0;
LOGGER.info("Storage group {},start serialize data into mem.", storageGroup);
+ long waitTime = 0;
while (!stop) {
Object task = memoryTaskQueue.poll();
if (task == null) {
@@ -68,7 +69,12 @@ public class MemTableFlushTaskV2 {
} catch (InterruptedException e) {
LOGGER.error("Storage group {}, io flush task is interrupted.", storageGroup, e);
}
+ waitTime += 10;
+ if (waitTime % 1000 == 0) {
+ LOGGER.info("encoding thread has waited {}ms", waitTime);
+ }
} else {
+ waitTime = 0;
if (task instanceof String) {
ioTaskQueue.add(task);
} else if (task instanceof ChunkGroupIoTask) {
@@ -101,6 +107,7 @@ public class MemTableFlushTaskV2 {
private Thread ioFlushThread = new Thread(() -> {
long ioTime = 0;
LOGGER.info("Storage group {}, start io cost.", storageGroup);
+ long waitTime = 0;
while (!stop) {
Object seriesWriterOrEndChunkGroupTask = ioTaskQueue.poll();
if (seriesWriterOrEndChunkGroupTask == null) {
@@ -109,7 +116,12 @@ public class MemTableFlushTaskV2 {
} catch (InterruptedException e) {
LOGGER.error("Storage group {}, io flush task is interrupted.", storageGroup, e);
}
+ waitTime += 10;
+ if (waitTime % 1000 == 0) {
+ LOGGER.info("io thread has waited {}ms", waitTime);
+ }
} else {
+ waitTime = 0;
long starTime = System.currentTimeMillis();
try {
if (seriesWriterOrEndChunkGroupTask instanceof IChunkWriter) {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java
new file mode 100644
index 0000000..e53c351
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.filenodeV2;
+
+import org.apache.iotdb.db.engine.MetadataManagerHelper;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FileNodeProcessorV2Test {
+
+ private String storageGroup = "storage_group1";
+ private String systemDir = "data/info";
+ private String deviceId = "root.vehicle.d0";
+ private String measurementId = "s0";
+ private FileNodeProcessorV2 processor;
+
+ @Before
+ public void setUp() throws Exception {
+ MetadataManagerHelper.initMetadata();
+ EnvironmentUtils.envSetUp();
+ processor = new FileNodeProcessorV2(systemDir, storageGroup);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+
+ @Test
+ public void testSequenceSyncClose() {
+ for (int j = 1; j <= 100; j++) {
+ TSRecord record = new TSRecord(j, deviceId);
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ processor.insert(new InsertPlan(record));
+ processor.asyncForceClose();
+ }
+
+ processor.syncCloseFileNode();
+ QueryDataSourceV2 queryDataSource = null;
+ try {
+ queryDataSource = processor.query(deviceId, measurementId);
+ } catch (FileNodeProcessorException e) {
+ e.printStackTrace();
+ }
+ Assert.assertEquals(queryDataSource.getSeqResources().size(), 100);
+ for (TsFileResourceV2 resource : queryDataSource.getSeqResources()) {
+ Assert.assertTrue(resource.isClosed());
+ }
+ }
+
+
+ @Test
+ public void testSeqAndUnSeqSyncClose() throws FileNodeProcessorException {
+
+ for (int j = 21; j <= 30; j++) {
+ TSRecord record = new TSRecord(j, deviceId);
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ processor.insert(new InsertPlan(record));
+ processor.asyncForceClose();
+ }
+ processor.syncCloseFileNode();
+
+ for (int j = 10; j >= 1; j--) {
+ TSRecord record = new TSRecord(j, deviceId);
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ processor.insert(new InsertPlan(record));
+ processor.asyncForceClose();
+ }
+
+ processor.syncCloseFileNode();
+
+ QueryDataSourceV2 queryDataSource = processor.query(deviceId, measurementId);
+ Assert.assertEquals(10, queryDataSource.getSeqResources().size());
+ Assert.assertEquals(10, queryDataSource.getUnseqResources().size());
+ for (TsFileResourceV2 resource : queryDataSource.getSeqResources()) {
+ Assert.assertTrue(resource.isClosed());
+ }
+ for (TsFileResourceV2 resource : queryDataSource.getUnseqResources()) {
+ Assert.assertTrue(resource.isClosed());
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java
new file mode 100644
index 0000000..b743207
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.db.engine.memtable;
+
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MemTablePoolTest {
+
+ private ConcurrentLinkedQueue<IMemTable> memTables;
+
+ @Before
+ public void setUp() throws Exception {
+ memTables = new ConcurrentLinkedQueue();
+ new ReturnThread().start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testGetAndRelease() {
+ for (int i = 0; i < 100; i++) {
+ IMemTable memTable = MemTablePool.getInstance().getEmptyMemTable("test case");
+ memTables.add(memTable);
+ }
+ }
+
+ @Test
+ public void testSort() {
+ long start = System.currentTimeMillis();
+ TreeMap<Long, Long> treeMap = new TreeMap<>();
+ for (int i = 0; i < 1000000; i++) {
+ treeMap.put((long)i, (long)i);
+ }
+ start = System.currentTimeMillis() - start;
+ System.out.println("time cost: " + start);
+ }
+
+
+ class ReturnThread extends Thread{
+
+ @Override
+ public void run() {
+ while (true) {
+ IMemTable memTable = memTables.poll();
+ if (memTable == null) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ continue;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ memTables.remove(memTable);
+ MemTablePool.getInstance().putBack(memTable, "test case");
+ }
+ }
+ }
+
+}
\ No newline at end of file