You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/13 12:48:16 UTC
[incubator-iotdb] 02/02: resolve conflict in overflow
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 5ffd9051e925caf362738ba8bee313d3c9fa5603
Merge: 21e4980 3b57abf
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Jun 13 20:47:57 2019 +0800
resolve conflict in overflow
.../iotdb/db/conf/directories/Directories.java | 5 +
.../engine/bufferwrite/BufferWriteProcessor.java | 6 +-
.../bufferwrite/RestorableTsFileIOWriter.java | 2 +
.../db/engine/overflow/io/OverflowMemtable.java | 9 +-
.../db/engine/overflow/io/OverflowProcessor.java | 25 +--
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 13 ++
.../physical/transfer/PhysicalPlanLogTransfer.java | 2 +-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 4 +
.../writelog/manager/MultiFileLogNodeManager.java | 36 +---
.../db/writelog/manager/WriteLogNodeManager.java | 5 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 181 +++++++++++----------
.../iotdb/db/writelog/node/WriteLogNode.java | 14 +-
.../engine/overflow/io/OverflowResourceTest.java | 8 +-
.../transfer/PhysicalPlanLogTransferTest.java | 29 ++--
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 6 +-
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 6 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 17 +-
.../org/apache/iotdb/db/writelog/RecoverTest.java | 12 +-
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 4 +-
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 18 +-
.../iotdb/db/writelog/io/LogWriterReaderTest.java | 5 +-
21 files changed, 192 insertions(+), 215 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
index cb4007a,c327cae..6a492c9
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
@@@ -150,9 -150,10 +150,10 @@@ public class RestorableTsFileIOWriter e
}
}
- private void writeRestoreInfo() throws IOException {
+ public void writeRestoreInfo() throws IOException {
+ long start = System.currentTimeMillis();
long lastPosition = this.getPos();
- // TODO: no need to create a TsRowGroupBlockMetadata, flush RowGroupMetadata one by one is ok
+ // TODO: no need to create a TsRowGroupBlockMetadata, flushMetadata RowGroupMetadata one by one is ok
TsDeviceMetadata tsDeviceMetadata = new TsDeviceMetadata();
this.getAppendedRowGroupMetadata();
tsDeviceMetadata.setChunkGroupMetadataList(this.append);
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index 2cf1294,173f615..5e3b777
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@@ -509,7 -509,7 +509,8 @@@ public class OverflowProcessor extends
filenodeFlushAction.act();
// write-ahead log
if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode.notifyEndFlush(null, walTaskId);
- logNode.notifyEndFlush(null, walTaskId, workResource.getInsertFile().getName());
++ //TODO
++// logNode.notifyEndFlush(null, walTaskId, workResource.getInsertFile().getName());
}
result = true;
} catch (IOException e) {
@@@ -570,14 -570,14 +571,15 @@@
throw new IOException(e);
}
long taskId = 0;
-- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
-- try {
- taskId = logNode.notifyStartFlush();
- taskId = logNode.notifyStartFlush(workResource.getInsertFile().getName());
-- } catch (IOException e) {
-- LOGGER.error("Overflow processor {} encountered an error when notifying log node, {}",
-- getProcessorName(), e);
-- }
-- }
++// if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
++// try {
++ //TODO
++// taskId = logNode.notifyStartFlush(workResource.getInsertFile().getName());
++// } catch (IOException e) {
++// LOGGER.error("Overflow processor {} encountered an error when notifying log node, {}",
++// getProcessorName(), e);
++// }
++// }
final long walTaskId = taskId;
BasicMemController.getInstance().releaseUsage(this, memSize.get());
memSize.set(0);
diff --cc iotdb/src/main/java/org/apache/iotdb/db/writelog/node/WriteLogNode.java
index fa27a54,0ea4982..780baf6
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/node/WriteLogNode.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/node/WriteLogNode.java
@@@ -51,24 -49,18 +49,18 @@@ public interface WriteLogNode
*/
void forceSync() throws IOException;
- /*
- * Force OS to sync all written data to disk.
- */
- void force() throws IOException;
-
/**
- * When a FileNode attempts to start a flush, this method must be called to rename log file.
+ * When a FileNode attempts to start a flushMetadata, this method must be called to rename log file.
* @return the task id ( being used in the renamed log file)
*/
long notifyStartFlush() throws IOException;
/**
- * When the flush of a FlieNode ends, this method must be called to check if log file needs
+ * When the flushMetadata of a FlieNode ends, this method must be called to check if log file needs
* cleaning.
- * @param logPositions (deprecated)
- * @param taskId the task id that notifyStartFlush() returns.
+ * @param fileId
*/
- void notifyEndFlush(List<LogPosition> logPositions, long taskId);
+ void notifyEndFlush(long fileId);
/**
* return identifier of the log node.
diff --cc iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
index 24509f1,49457cf..3a4e013
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowResourceTest.java
@@@ -55,6 -57,10 +57,10 @@@ public class OverflowResourceTest
EnvironmentUtils.cleanDir(folderPath);
}
+ private void removeFlushedMemTable(IMemTable memTable, TsFileIOWriter overflowIOWriter) {
- overflowIOWriter.mergeChunkGroupMetaData();
++// overflowIOWriter.mergeChunkGroupMetaData();
+ }
+
@Test
public void testOverflowInsert() throws IOException {
OverflowTestUtils.produceInsertData(memtable);