You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/06/12 12:59:31 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated
(251cfe9 -> 011cfbd)
This is an automated email from the ASF dual-hosted git repository.
east pushed a change to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 251cfe9 Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new 95901a1 fix overflow bug
new 011cfbd Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java | 9 +++------
.../apache/iotdb/db/engine/overflow/io/OverflowProcessor.java | 5 +++--
.../apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java | 8 +++++++-
3 files changed, 13 insertions(+), 9 deletions(-)
[incubator-iotdb] 02/02: Merge remote-tracking branch
'origin/feature_async_close_tsfile' into feature_async_close_tsfile
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 011cfbd66b087c5ae9f422df5a0fe53ec8358cbd
Merge: 95901a1 251cfe9
Author: mdf369 <95...@qq.com>
AuthorDate: Wed Jun 12 20:55:49 2019 +0800
Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
# Conflicts:
# iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
.../iotdb/db/conf/directories/Directories.java | 5 +
.../engine/bufferwrite/BufferWriteProcessor.java | 120 +++++--------------
.../iotdb/db/engine/filenode/FileNodeManager.java | 18 ++-
.../db/engine/filenode/FileNodeProcessor.java | 17 +--
.../db/engine/memtable/MemTableFlushTask.java | 64 +++++++----
.../iotdb/db/engine/memtable/MemTablePool.java | 21 +++-
.../db/engine/overflow/io/OverflowProcessor.java | 14 +--
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 13 +++
.../physical/transfer/PhysicalPlanLogTransfer.java | 2 +-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 4 +
.../writelog/manager/MultiFileLogNodeManager.java | 36 +-----
.../db/writelog/manager/WriteLogNodeManager.java | 5 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 127 +++++++++------------
.../iotdb/db/writelog/node/WriteLogNode.java | 18 +--
.../bufferwrite/BufferWriteProcessorNewTest.java | 1 -
.../bufferwrite/BufferWriteProcessorTest.java | 3 -
.../transfer/PhysicalPlanLogTransferTest.java | 29 +++--
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 6 +-
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 6 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 17 +--
.../org/apache/iotdb/db/writelog/RecoverTest.java | 14 +--
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 4 +-
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 20 ++--
.../iotdb/db/writelog/io/LogWriterReaderTest.java | 5 +-
24 files changed, 247 insertions(+), 322 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index eb13d1a,7c67303..173f615
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@@ -587,8 -587,7 +587,8 @@@ public class OverflowProcessor extends
//add mmd
overflowFlushMemTables.add(workSupport);
IMemTable tmpMemTableToFlush = workSupport;
- workSupport = MemTablePool.getInstance().getEmptyMemTable();
+ workSupport = MemTablePool.getInstance().getEmptyMemTable(this);
+ flushId++;
flushFuture = FlushManager.getInstance().submit(() -> flushTask("asynchronously",
tmpMemTableToFlush, walTaskId, flushId, this::removeFlushedMemTable));
[incubator-iotdb] 01/02: fix overflow bug
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 95901a1c0e1a76d81e4b8048946d23810fe11ce6
Author: mdf369 <95...@qq.com>
AuthorDate: Wed Jun 12 20:53:27 2019 +0800
fix overflow bug
---
.../org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java | 9 +++------
.../apache/iotdb/db/engine/overflow/io/OverflowProcessor.java | 1 +
.../apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java | 8 +++++++-
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
index 91ef235..f7b3246 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
@@ -18,18 +18,13 @@
*/
package org.apache.iotdb.db.engine.overflow.io;
-import static org.apache.iotdb.db.conf.IoTDBConstant.PATH_SEPARATOR;
-
import java.util.HashMap;
import java.util.Map;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
-import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
@@ -53,6 +48,7 @@ public class OverflowMemtable extends PrimitiveMemTable {
memTable = new PrimitiveMemTable();
}
+ @Override
public void insert(TSRecord tsRecord) {
for (DataPoint dataPoint : tsRecord.dataPointList) {
memTable.write(tsRecord.deviceId, dataPoint.getMeasurementId(), dataPoint.getType(),
@@ -99,7 +95,7 @@ public class OverflowMemtable extends PrimitiveMemTable {
}
public IMemTable getMemTabale() {
- return this;
+ return memTable;
}
public long getSize() {
@@ -107,6 +103,7 @@ public class OverflowMemtable extends PrimitiveMemTable {
return 0;
}
+ @Override
public void clear() {
// indexTrees.clear();
super.clear();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index 4f77c06..eb13d1a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@ -588,6 +588,7 @@ public class OverflowProcessor extends Processor {
overflowFlushMemTables.add(workSupport);
IMemTable tmpMemTableToFlush = workSupport;
workSupport = MemTablePool.getInstance().getEmptyMemTable();
+ flushId++;
flushFuture = FlushManager.getInstance().submit(() -> flushTask("asynchronously",
tmpMemTableToFlush, walTaskId, flushId, this::removeFlushedMemTable));
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
index 24509f1..49457cf 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
@@ -25,10 +25,12 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -55,12 +57,16 @@ public class OverflowResourceTest {
EnvironmentUtils.cleanDir(folderPath);
}
+ private void removeFlushedMemTable(IMemTable memTable, TsFileIOWriter overflowIOWriter) {
+ overflowIOWriter.mergeChunkGroupMetaData();
+ }
+
@Test
public void testOverflowInsert() throws IOException {
OverflowTestUtils.produceInsertData(memtable);
QueryContext context = new QueryContext();
work.flush(OverflowTestUtils.getFileSchema(), memtable.getMemTabale(),
- "processorName", 0, (k,v)->{});
+ "processorName", 0, this::removeFlushedMemTable);
List<ChunkMetaData> chunkMetaDatas = work.getInsertMetadatas(OverflowTestUtils.deviceId1,
OverflowTestUtils.measurementId1, OverflowTestUtils.dataType2, context);
assertEquals(0, chunkMetaDatas.size());