You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/06/20 05:15:15 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated:
refactor UnseqTsFileRecoverPerformer
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature_async_close_tsfile by this push:
new 6e7c375 refactor UnseqTsFileRecoverPerformer
6e7c375 is described below
commit 6e7c3754ee8b33f508c1836703025dfb61b3bd12
Author: 江天 <jt...@163.com>
AuthorDate: Thu Jun 20 13:13:09 2019 +0800
refactor UnseqTsFileRecoverPerformer
---
.../db/engine/overflow/io/OverflowProcessor.java | 7 +--
.../db/engine/overflow/io/OverflowResource.java | 4 ++
.../recover/UnseqTsFileRecoverPerformer.java | 65 ++++------------------
3 files changed, 17 insertions(+), 59 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index 43f2700..3c3582c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@ -35,7 +35,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.bufferwrite.Action;
@@ -63,9 +62,7 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.ImmediateFuture;
import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.db.writelog.recover.LogReplayer;
import org.apache.iotdb.db.writelog.recover.UnseqTsFileRecoverPerformer;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
@@ -198,8 +195,8 @@ public class OverflowProcessor extends Processor {
UnseqTsFileRecoverPerformer recoverPerformer =
new UnseqTsFileRecoverPerformer(workResource.logNodePrefix(),
workResource.getInsertFilePath(),
- workResource.getModificationFile(), versionController, fileSchema,
- workResource.getInsertIO(),workResource.getAppendInsertMetadatas());
+ workResource.getModificationFile(), versionController, fileSchema
+ );
recoverPerformer.recover();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
index 8722db2..27c216c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
@@ -365,4 +365,8 @@ public class OverflowResource {
public List<ChunkGroupMetaData> getAppendInsertMetadatas() {
return appendInsertMetadatas;
}
+
+ public VersionController getVersionController() {
+ return versionController;
+ }
}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverPerformer.java
index d0a046c..761a738 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/UnseqTsFileRecoverPerformer.java
@@ -21,82 +21,39 @@ package org.apache.iotdb.db.writelog.recover;
import java.io.File;
import java.io.IOException;
-import java.util.List;
-import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.memtable.IMemTable;
-import org.apache.iotdb.db.engine.memtable.MemTableFlushCallBack;
-import org.apache.iotdb.db.engine.memtable.MemTableFlushTask;
import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.overflow.io.OverflowIO;
-import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.engine.overflow.io.OverflowResource;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
public class UnseqTsFileRecoverPerformer {
- private String logNodePrefix;
- private String insertFilePath;
- private ModificationFile modFile;
- private VersionController versionController;
private FileSchema fileSchema;
- private OverflowIO overflowIO;
- private List<ChunkGroupMetaData> appendInsertMetadatas;
+ private OverflowResource resource;
- public UnseqTsFileRecoverPerformer(String logNodePrefix, String insertFilePath,
- ModificationFile modFile,
- VersionController versionController, FileSchema fileSchema,
- OverflowIO overflowIO,
- List<ChunkGroupMetaData> appendInsertMetadatas) {
- this.logNodePrefix = logNodePrefix;
- this.insertFilePath = insertFilePath;
- this.modFile = modFile;
- this.versionController = versionController;
+
+ public UnseqTsFileRecoverPerformer(OverflowResource resource, FileSchema fileSchema) {
+ this.resource = resource;
this.fileSchema = fileSchema;
- this.overflowIO = overflowIO;
- this.appendInsertMetadatas = appendInsertMetadatas;
}
public void recover() throws ProcessorException {
IMemTable memTable = new PrimitiveMemTable();
+ String logNodePrefix = resource.logNodePrefix();
+ String insertFilePath = resource.getInsertFilePath();
LogReplayer replayer = new LogReplayer(logNodePrefix, insertFilePath,
- modFile, versionController, null, fileSchema, memTable);
+ resource.getModificationFile(), resource.getVersionController(), null,
+ fileSchema, memTable);
replayer.replayLogs();
try {
- flush(memTable);
+ resource.flush(fileSchema, memTable, logNodePrefix, 0, (a,b) -> {});
+ resource.appendMetadatas();
MultiFileLogNodeManager.getInstance().deleteNode(logNodePrefix + new File(insertFilePath).getName());
} catch (IOException e) {
throw new ProcessorException(e);
}
}
- public void flush(IMemTable memTable) throws IOException {
- if (memTable != null && !memTable.isEmpty()) {
- overflowIO.toTail();
- long lastPosition = overflowIO.getPos();
- MemTableFlushTask tableFlushTask = new MemTableFlushTask(overflowIO, logNodePrefix, 0,
- (a,b) -> {});
- tableFlushTask.flushMemTable(fileSchema, memTable, versionController.nextVersion());
-
- List<ChunkGroupMetaData> rowGroupMetaDatas = overflowIO.getChunkGroupMetaDatas();
- appendInsertMetadatas.addAll(rowGroupMetaDatas);
- if (!rowGroupMetaDatas.isEmpty()) {
- overflowIO.getWriter().write(BytesUtils.longToBytes(lastPosition));
- TsDeviceMetadata tsDeviceMetadata = new TsDeviceMetadata();
- tsDeviceMetadata.setChunkGroupMetadataList(rowGroupMetaDatas);
- long start = overflowIO.getPos();
- tsDeviceMetadata.serializeTo(overflowIO.getOutputStream());
- long end = overflowIO.getPos();
- overflowIO.getWriter().write(BytesUtils.intToBytes((int) (end - start)));
- // clear the meta-data of insert IO
- overflowIO.clearRowGroupMetadatas();
- }
- }
- }
}