You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/25 04:05:27 UTC

[incubator-iotdb] 01/03: add test and log in memtable flush task

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git

commit bc334d13455f5eecf739ec67a7a1a1985c680ee3
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Jun 25 11:59:12 2019 +0800

    add test and log in memtable flush task
---
 .../db/engine/memtable/MemTableFlushTaskV2.java    |  12 +++
 .../filenodeV2/FileNodeProcessorV2Test2.java       | 109 +++++++++++++++++++++
 .../iotdb/db/engine/memtable/MemTablePoolTest.java |  68 +++++++++++++
 3 files changed, 189 insertions(+)

diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
index 13c45a5..b8825d1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
@@ -60,6 +60,7 @@ public class MemTableFlushTaskV2 {
   private Thread memoryFlushThread = new Thread(() -> {
     long memSerializeTime = 0;
     LOGGER.info("Storage group {},start serialize data into mem.", storageGroup);
+    long waitTime = 0;
     while (!stop) {
       Object task = memoryTaskQueue.poll();
       if (task == null) {
@@ -68,7 +69,12 @@ public class MemTableFlushTaskV2 {
         } catch (InterruptedException e) {
           LOGGER.error("Storage group {}, io flush task is interrupted.", storageGroup, e);
         }
+        waitTime += 10;
+        if (waitTime % 1000 == 0) {
+          LOGGER.info("encoding thread has waited {}ms", waitTime);
+        }
       } else {
+        waitTime = 0;
         if (task instanceof String) {
           ioTaskQueue.add(task);
         } else if (task instanceof ChunkGroupIoTask) {
@@ -101,6 +107,7 @@ public class MemTableFlushTaskV2 {
   private Thread ioFlushThread = new Thread(() -> {
     long ioTime = 0;
     LOGGER.info("Storage group {}, start io cost.", storageGroup);
+    long waitTime = 0;
     while (!stop) {
       Object seriesWriterOrEndChunkGroupTask = ioTaskQueue.poll();
       if (seriesWriterOrEndChunkGroupTask == null) {
@@ -109,7 +116,12 @@ public class MemTableFlushTaskV2 {
         } catch (InterruptedException e) {
           LOGGER.error("Storage group {}, io flush task is interrupted.", storageGroup, e);
         }
+        waitTime += 10;
+        if (waitTime % 1000 == 0) {
+          LOGGER.info("io thread has waited {}ms", waitTime);
+        }
       } else {
+        waitTime = 0;
         long starTime = System.currentTimeMillis();
         try {
           if (seriesWriterOrEndChunkGroupTask instanceof IChunkWriter) {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java
new file mode 100644
index 0000000..e53c351
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test2.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.filenodeV2;
+
+import org.apache.iotdb.db.engine.MetadataManagerHelper;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FileNodeProcessorV2Test {
+
+  private String storageGroup = "storage_group1";
+  private String systemDir = "data/info";
+  private String deviceId = "root.vehicle.d0";
+  private String measurementId = "s0";
+  private FileNodeProcessorV2 processor;
+
+  @Before
+  public void setUp() throws Exception {
+    MetadataManagerHelper.initMetadata();
+    EnvironmentUtils.envSetUp();
+    processor = new FileNodeProcessorV2(systemDir, storageGroup);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    EnvironmentUtils.cleanEnv();
+  }
+
+
+  @Test
+  public void testSequenceSyncClose() {
+    for (int j = 1; j <= 100; j++) {
+      TSRecord record = new TSRecord(j, deviceId);
+      record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+      processor.insert(new InsertPlan(record));
+      processor.asyncForceClose();
+    }
+
+    processor.syncCloseFileNode();
+    QueryDataSourceV2 queryDataSource = null;
+    try {
+      queryDataSource = processor.query(deviceId, measurementId);
+    } catch (FileNodeProcessorException e) {
+      e.printStackTrace();
+    }
+    Assert.assertEquals(queryDataSource.getSeqResources().size(), 100);
+    for (TsFileResourceV2 resource : queryDataSource.getSeqResources()) {
+      Assert.assertTrue(resource.isClosed());
+    }
+  }
+
+
+  @Test
+  public void testSeqAndUnSeqSyncClose() throws FileNodeProcessorException {
+
+    for (int j = 21; j <= 30; j++) {
+      TSRecord record = new TSRecord(j, deviceId);
+      record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+      processor.insert(new InsertPlan(record));
+      processor.asyncForceClose();
+    }
+    processor.syncCloseFileNode();
+
+    for (int j = 10; j >= 1; j--) {
+      TSRecord record = new TSRecord(j, deviceId);
+      record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+      processor.insert(new InsertPlan(record));
+      processor.asyncForceClose();
+    }
+
+    processor.syncCloseFileNode();
+
+    QueryDataSourceV2 queryDataSource = processor.query(deviceId, measurementId);
+    Assert.assertEquals(10, queryDataSource.getSeqResources().size());
+    Assert.assertEquals(10, queryDataSource.getUnseqResources().size());
+    for (TsFileResourceV2 resource : queryDataSource.getSeqResources()) {
+      Assert.assertTrue(resource.isClosed());
+    }
+    for (TsFileResourceV2 resource : queryDataSource.getUnseqResources()) {
+      Assert.assertTrue(resource.isClosed());
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java
new file mode 100644
index 0000000..b743207
--- /dev/null
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memtable/MemTablePoolTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.db.engine.memtable;
+
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MemTablePoolTest {
+
+  private ConcurrentLinkedQueue<IMemTable> memTables;
+
+  @Before
+  public void setUp() throws Exception {
+    memTables = new ConcurrentLinkedQueue();
+    new ReturnThread().start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  @Test
+  public void testGetAndRelease() {
+    for (int i = 0; i < 100; i++) {
+      IMemTable memTable = MemTablePool.getInstance().getEmptyMemTable("test case");
+      memTables.add(memTable);
+    }
+  }
+
+  @Test
+  public void testSort() {
+    long start = System.currentTimeMillis();
+    TreeMap<Long, Long> treeMap = new TreeMap<>();
+    for (int i = 0; i < 1000000; i++) {
+      treeMap.put((long)i, (long)i);
+    }
+    start = System.currentTimeMillis() - start;
+    System.out.println("time cost: " + start);
+  }
+
+
+  class ReturnThread extends Thread{
+
+    @Override
+    public void run() {
+      while (true) {
+        IMemTable memTable = memTables.poll();
+        if (memTable == null) {
+          try {
+            Thread.sleep(10);
+          } catch (InterruptedException e) {
+            e.printStackTrace();
+          }
+          continue;
+        }
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        memTables.remove(memTable);
+        MemTablePool.getInstance().putBack(memTable, "test case");
+      }
+    }
+  }
+
+}
\ No newline at end of file