You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/18 14:40:09 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated:
complete async flush and close and add TsFileResourceV2,
merge RestorableTsFileIOWriter into NativeRestorableTsFileIOWriter
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/feature_async_close_tsfile by this push:
new d074ecb complete async flush and close and add TsFileResourceV2, merge RestorableTsFileIOWriter into NativeRestorableTsFileIOWriter
d074ecb is described below
commit d074ecb22171e5dcb34aacd3036c100c0e949068
Author: qiaojialin <64...@qq.com>
AuthorDate: Tue Jun 18 22:39:52 2019 +0800
complete async flush and close and add TsFileResourceV2, merge RestorableTsFileIOWriter into NativeRestorableTsFileIOWriter
---
.../bufferwrite/RestorableTsFileIOWriter.java | 1 +
.../bufferwriteV2/BufferWriteProcessorV2.java | 92 +++++++-
.../db/engine/bufferwriteV2/FlushManager.java | 6 +-
.../filenodeV2/FileNodeProcessorStoreV2.java | 152 +++++++++++++
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 153 +++++++++++--
.../db/engine/filenodeV2/TsFileResourceV2.java | 101 ++++++++-
.../apache/iotdb/db/engine/memtable/Callback.java | 7 +
.../db/engine/memtable/MemTableFlushTaskV2.java | 241 +++++++++++++++++++++
.../db/engine/overflow/io/OverflowProcessor.java | 36 +--
.../write/writer/NativeRestorableIOWriter.java | 86 +++++++-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 3 +-
11 files changed, 822 insertions(+), 56 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
index 6a492c9..b3fef58 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
@@ -51,6 +51,7 @@ import org.slf4j.LoggerFactory;
* the tsfile loses the tsfile metadata in the end, a corresponding. restore file exists. (2) no
* .restore file, and then the class has to traverse all the data for fixing the file.
*/
+@Deprecated
public class RestorableTsFileIOWriter extends TsFileIOWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(RestorableTsFileIOWriter.class);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/BufferWriteProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/BufferWriteProcessorV2.java
index 4bfa2db..081b383 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/BufferWriteProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/BufferWriteProcessorV2.java
@@ -21,10 +21,16 @@ package org.apache.iotdb.db.engine.bufferwriteV2;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
+import org.apache.iotdb.db.engine.memtable.Callback;
import org.apache.iotdb.db.engine.memtable.IMemTable;
+import org.apache.iotdb.db.engine.memtable.MemTableFlushTaskV2;
import org.apache.iotdb.db.engine.memtable.MemTablePool;
+import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.qp.constant.DatetimeUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
@@ -44,26 +50,33 @@ public class BufferWriteProcessorV2 {
private TsFileResourceV2 tsFileResource;
- private volatile boolean isManagedByFlushManager;
+ private volatile boolean managedByFlushManager;
+
+ private ReadWriteLock lock = new ReentrantReadWriteLock();
/**
* true: to be closed
*/
- private boolean closing;
+ private volatile boolean closing;
private IMemTable workMemTable;
+ private VersionController versionController;
+
+ private Callback closeBufferWriteProcessor;
+
+ // synch this object in query() and asyncFlush()
private final ConcurrentLinkedDeque<IMemTable> flushingMemTables = new ConcurrentLinkedDeque<>();
- public BufferWriteProcessorV2(String storageGroupName, File file, FileSchema fileSchema) throws IOException {
+ public BufferWriteProcessorV2(String storageGroupName, File file, FileSchema fileSchema, VersionController versionController, Callback closeBufferWriteProcessor) throws IOException {
this.storageGroupName = storageGroupName;
this.fileSchema = fileSchema;
- tsFileResource = new TsFileResourceV2();
- // recover and get a writer
- writer = new NativeRestorableIOWriter(file);
+ this.tsFileResource = new TsFileResourceV2(file);
+ this.versionController = versionController;
+ this.writer = new NativeRestorableIOWriter(file);
+ this.closeBufferWriteProcessor = closeBufferWriteProcessor;
}
-
/**
* write a TsRecord into the workMemtable.
* If the memory usage is beyond the memTableThreshold, put it into flushing list.
@@ -91,6 +104,26 @@ public class BufferWriteProcessorV2 {
return true;
}
+
+ public TsFileResourceV2 getTsFileResource() {
+ return tsFileResource;
+ }
+
+ /**
+ * return the memtable to MemTablePool and make metadata in writer visible
+ * @param memTable
+ */
+ private void removeFlushedMemTable(Object memTable) {
+ lock.writeLock().lock();
+ writer.makeMetadataVisible();
+ try {
+ flushingMemTables.remove(memTable);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+
public boolean shouldFlush() {
return workMemTable.memSize() > TSFileDescriptor.getInstance().getConfig().groupSizeInByte;
}
@@ -98,14 +131,47 @@ public class BufferWriteProcessorV2 {
/**
* put the workMemtable into flushing list and set null
*/
- public void flush() {
+ public void asyncFlush() {
flushingMemTables.addLast(workMemTable);
+ FlushManager.getInstance().registerBWP(this);
workMemTable = null;
}
- public void flushOneMemTable(){
+ public void flushOneMemTable() {
IMemTable memTableToFlush = flushingMemTables.pollFirst();
+ MemTableFlushTaskV2 flushTask = new MemTableFlushTaskV2(writer, storageGroupName, this::removeFlushedMemTable);
+ flushTask.flushMemTable(fileSchema, memTableToFlush, versionController.nextVersion());
+
+ if (closing && flushingMemTables.isEmpty()) {
+
+ }
+ }
+ public void close() throws IOException {
+ long closeStartTime = System.currentTimeMillis();
+ writer.endFile(fileSchema);
+ //FIXME suppose the flushMetadata-thread-pool is 2.
+ // then if a flushMetadata task and a close task are running in the same time
+ // and the close task is faster, then writer == null, and the flushMetadata task will throw nullpointer
+ // exception. Add "synchronized" keyword on both flushMetadata and close may solve the issue.
+ writer = null;
+
+ // remove this processor from Closing list in FileNodeProcessor
+ closeBufferWriteProcessor.call(this);
+
+ // delete the restore for this bufferwrite processor
+ if (LOGGER.isInfoEnabled()) {
+
+ long closeEndTime = System.currentTimeMillis();
+
+ LOGGER.info(
+ "Close bufferwrite processor {}, the file name is {}, start time is {}, end time is {}, "
+ + "time consumption is {}ms",
+ storageGroupName, tsFileResource.getFile().getAbsoluteFile(),
+ DatetimeUtils.convertMillsecondToZonedDateTime(closeStartTime),
+ DatetimeUtils.convertMillsecondToZonedDateTime(closeEndTime),
+ closeEndTime - closeStartTime);
+ }
}
public boolean shouldClose() {
@@ -114,12 +180,16 @@ public class BufferWriteProcessorV2 {
return fileSize > fileSizeThreshold;
}
+ public void setClosing() {
+ closing = true;
+ }
+
public boolean isManagedByFlushManager() {
- return isManagedByFlushManager;
+ return managedByFlushManager;
}
public void setManagedByFlushManager(boolean managedByFlushManager) {
- isManagedByFlushManager = managedByFlushManager;
+ this.managedByFlushManager = managedByFlushManager;
}
public int getFlushingMemTableSize() {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/FlushManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/FlushManager.java
index 57e81c2..55477f9 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/FlushManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwriteV2/FlushManager.java
@@ -31,13 +31,13 @@ public class FlushManager {
BufferWriteProcessorV2 bwp = bwpQueue.poll();
bwp.flushOneMemTable();
bwp.setManagedByFlushManager(false);
- addBWP(bwp);
+ registerBWP(bwp);
};
/**
- * Add BufferWriteProcessor to flush manager
+ * Add BufferWriteProcessor to asyncFlush manager
*/
- private boolean addBWP(BufferWriteProcessorV2 bwp) {
+ public boolean registerBWP(BufferWriteProcessorV2 bwp) {
synchronized (bwp) {
if (!bwp.isManagedByFlushManager() && bwp.getFlushingMemTableSize() > 0) {
bwpQueue.add(bwp);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorStoreV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorStoreV2.java
new file mode 100644
index 0000000..7ae9776
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorStoreV2.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.filenodeV2;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.iotdb.db.engine.filenode.FileNodeProcessorStatus;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+/**
+ * FileNodeProcessorStore is used to store information about FileNodeProcessor's status.
+ * lastUpdateTime is changed and stored by BufferWrite flushMetadata or BufferWrite close.
+ * emptyTsFileResource and newFileNodes are changed and stored by Overflow flushMetadata and
+ * Overflow close. fileNodeProcessorState is changed and stored by the change of FileNodeProcessor's
+ * status such as "work->merge merge->wait wait->work". numOfMergeFile is changed
+ * and stored when FileNodeProcessor's status changes from work to merge.
+ */
+public class FileNodeProcessorStoreV2 implements Serializable {
+
+ private static final long serialVersionUID = -54525372941897565L;
+
+ private boolean isOverflowed;
+ private Map<String, Long> lastUpdateTimeMap;
+ private List<TsFileResourceV2> newFileNodes;
+ private int numOfMergeFile;
+ private FileNodeProcessorStatus fileNodeProcessorStatus;
+
+ /**
+ * Constructor of FileNodeProcessorStore.
+ *
+ * @param isOverflowed whether this FileNode contains unmerged Overflow operations.
+ * @param lastUpdateTimeMap the timestamp of last data point of each device in this FileNode.
+ * @param newFileNodes TsFiles in the FileNode.
+ * @param fileNodeProcessorStatus the status of the FileNode.
+ * @param numOfMergeFile the number of files already merged in one merge operation.
+ */
+ public FileNodeProcessorStoreV2(boolean isOverflowed, Map<String, Long> lastUpdateTimeMap,
+ List<TsFileResourceV2> newFileNodes,
+ FileNodeProcessorStatus fileNodeProcessorStatus,
+ int numOfMergeFile) {
+ this.isOverflowed = isOverflowed;
+ this.lastUpdateTimeMap = lastUpdateTimeMap;
+ this.newFileNodes = newFileNodes;
+ this.fileNodeProcessorStatus = fileNodeProcessorStatus;
+ this.numOfMergeFile = numOfMergeFile;
+ }
+
+ public void serialize(OutputStream outputStream) throws IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ ReadWriteIOUtils.write(this.isOverflowed, byteArrayOutputStream);
+ // lastUpdateTimeMap
+ ReadWriteIOUtils.write(lastUpdateTimeMap.size(), byteArrayOutputStream);
+ for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), byteArrayOutputStream);
+ ReadWriteIOUtils.write(entry.getValue(), byteArrayOutputStream);
+ }
+ ReadWriteIOUtils.write(this.newFileNodes.size(), byteArrayOutputStream);
+ for (TsFileResourceV2 tsFileResource : this.newFileNodes) {
+ tsFileResource.serialize(byteArrayOutputStream);
+ }
+ ReadWriteIOUtils.write(this.numOfMergeFile, byteArrayOutputStream);
+ ReadWriteIOUtils.write(this.fileNodeProcessorStatus.serialize(), byteArrayOutputStream);
+ // buffer array to outputstream
+ byteArrayOutputStream.writeTo(outputStream);
+ }
+
+ public static FileNodeProcessorStoreV2 deSerialize(InputStream inputStream) throws IOException {
+ boolean isOverflowed = ReadWriteIOUtils.readBool(inputStream);
+ Map<String, Long> lastUpdateTimeMap = new HashMap<>();
+ int size = ReadWriteIOUtils.readInt(inputStream);
+ for (int i = 0; i < size; i++) {
+ String path = ReadWriteIOUtils.readString(inputStream);
+ long time = ReadWriteIOUtils.readLong(inputStream);
+ lastUpdateTimeMap.put(path, time);
+ }
+ size = ReadWriteIOUtils.readInt(inputStream);
+ List<TsFileResourceV2> newFileNodes = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ newFileNodes.add(TsFileResourceV2.deSerialize(inputStream));
+ }
+ int numOfMergeFile = ReadWriteIOUtils.readInt(inputStream);
+ FileNodeProcessorStatus fileNodeProcessorStatus = FileNodeProcessorStatus
+ .deserialize(ReadWriteIOUtils.readShort(inputStream));
+
+ return new FileNodeProcessorStoreV2(isOverflowed, lastUpdateTimeMap,
+ newFileNodes, fileNodeProcessorStatus, numOfMergeFile);
+ }
+
+ public boolean isOverflowed() {
+ return isOverflowed;
+ }
+
+ public void setOverflowed(boolean isOverflowed) {
+ this.isOverflowed = isOverflowed;
+ }
+
+ public FileNodeProcessorStatus getFileNodeProcessorStatus() {
+ return fileNodeProcessorStatus;
+ }
+
+ public void setFileNodeProcessorStatus(FileNodeProcessorStatus fileNodeProcessorStatus) {
+ this.fileNodeProcessorStatus = fileNodeProcessorStatus;
+ }
+
+ public Map<String, Long> getLastUpdateTimeMap() {
+ return new HashMap<>(lastUpdateTimeMap);
+ }
+
+ public void setLastUpdateTimeMap(Map<String, Long> lastUpdateTimeMap) {
+ this.lastUpdateTimeMap = lastUpdateTimeMap;
+ }
+
+ public List<TsFileResourceV2> getNewFileNodes() {
+ return newFileNodes;
+ }
+
+ public void setNewFileNodes(List<TsFileResourceV2> newFileNodes) {
+ this.newFileNodes = newFileNodes;
+ }
+
+ public int getNumOfMergeFile() {
+ return numOfMergeFile;
+ }
+
+ public void setNumOfMergeFile(int numOfMergeFile) {
+ this.numOfMergeFile = numOfMergeFile;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index 085357e..ef46166 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -19,10 +19,15 @@
package org.apache.iotdb.db.engine.filenodeV2;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -30,7 +35,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.bufferwriteV2.BufferWriteProcessorV2;
import org.apache.iotdb.db.engine.filenode.CopyOnWriteLinkedList;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.filenode.FileNodeProcessorStatus;
+import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
+import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
@@ -42,6 +50,8 @@ public class FileNodeProcessorV2 {
private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeProcessorV2.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final String RESTORE_FILE_SUFFIX = ".restore";
+
private static final MManager mManager = MManager.getInstance();
private static final Directories directories = Directories.getInstance();
@@ -52,9 +62,9 @@ public class FileNodeProcessorV2 {
*/
private Map<String, List<TsFileResourceV2>> invertedIndexOfFiles = new HashMap<>();
- private List<TsFileResource> newFileNodes;
+ private List<TsFileResourceV2> newFileNodes;
- private BufferWriteProcessorV2 activeBufferWriteProcessor = null;
+ private BufferWriteProcessorV2 workBufferWriteProcessor = null;
/**
* device -> global latest timestamp of each device
@@ -62,7 +72,7 @@ public class FileNodeProcessorV2 {
private Map<String, Long> latestTimeMap = new HashMap<>();
/**
- * device -> largest timestamp of the latest memtable to be submitted to flush
+ * device -> largest timestamp of the latest memtable to be submitted to asyncFlush
*/
private Map<String, Long> latestFlushTimeMap = new HashMap<>();
@@ -72,7 +82,14 @@ public class FileNodeProcessorV2 {
private final ReadWriteLock lock;
- public FileNodeProcessorV2(String baseDir, String storageGroup) {
+ private VersionController versionController;
+
+ private String fileNodeRestoreFilePath;
+ private FileNodeProcessorStoreV2 fileNodeProcessorStore;
+ private final Object fileNodeRestoreLock = new Object();
+
+
+ public FileNodeProcessorV2(String baseDir, String storageGroup) throws FileNodeProcessorException {
this.storageGroup = storageGroup;
lock = new ReentrantReadWriteLock();
@@ -83,8 +100,75 @@ public class FileNodeProcessorV2 {
"The directory of the storage group {} doesn't exist. Create a new " +
"directory {}", storageGroup, storageGroupDir.getAbsolutePath());
}
+
+ /**
+ * restore
+ */
+ File restoreFolder = new File(baseDir + storageGroup);
+ if (!restoreFolder.exists()) {
+ restoreFolder.mkdirs();
+ LOGGER.info("The restore directory of the filenode processor {} doesn't exist. Create new " +
+ "directory {}", storageGroup, restoreFolder.getAbsolutePath());
+ }
+ fileNodeRestoreFilePath = new File(restoreFolder, storageGroup + RESTORE_FILE_SUFFIX)
+ .getPath();
+ try {
+ fileNodeProcessorStore = readStoreFromDisk();
+ } catch (FileNodeProcessorException e) {
+ LOGGER.error(
+ "The fileNode processor {} encountered an error when recoverying restore " +
+ "information.", storageGroup);
+ throw new FileNodeProcessorException(e);
+ }
+ // TODO deep clone the lastupdate time, change the getNewFileNodes to V2
+// newFileNodes = fileNodeProcessorStore.getNewFileNodes();
+ invertedIndexOfFiles = new HashMap<>();
+
+ /**
+ * version controller
+ */
+ try {
+ versionController = new SimpleFileVersionController(restoreFolder.getPath());
+ } catch (IOException e) {
+ throw new FileNodeProcessorException(e);
+ }
+
+ }
+
+
+ private FileNodeProcessorStoreV2 readStoreFromDisk() throws FileNodeProcessorException {
+
+ synchronized (fileNodeRestoreLock) {
+ File restoreFile = new File(fileNodeRestoreFilePath);
+ if (!restoreFile.exists() || restoreFile.length() == 0) {
+ return new FileNodeProcessorStoreV2(false, new HashMap<>(),
+ new ArrayList<>(), FileNodeProcessorStatus.NONE, 0);
+ }
+ try (FileInputStream inputStream = new FileInputStream(fileNodeRestoreFilePath)) {
+ return FileNodeProcessorStoreV2.deSerialize(inputStream);
+ } catch (IOException e) {
+ LOGGER.error("Failed to deserialize the FileNodeRestoreFile {}, {}", fileNodeRestoreFilePath,
+ e);
+ throw new FileNodeProcessorException(e);
+ }
+ }
}
+ private void writeStoreToDisk(FileNodeProcessorStoreV2 fileNodeProcessorStore)
+ throws FileNodeProcessorException {
+
+ synchronized (fileNodeRestoreLock) {
+ try (FileOutputStream fileOutputStream = new FileOutputStream(fileNodeRestoreFilePath)) {
+ fileNodeProcessorStore.serialize(fileOutputStream);
+ LOGGER.debug("The filenode processor {} writes restore information to the restore file",
+ storageGroup);
+ } catch (IOException e) {
+ throw new FileNodeProcessorException(e);
+ }
+ }
+ }
+
+
public boolean insert(TSRecord tsRecord) {
lock.writeLock().lock();
@@ -93,12 +177,12 @@ public class FileNodeProcessorV2 {
try {
// create a new BufferWriteProcessor
- if (activeBufferWriteProcessor == null) {
+ if (workBufferWriteProcessor == null) {
String baseDir = directories.getNextFolderForTsfile();
String filePath = Paths.get(baseDir, storageGroup, tsRecord.time + "").toString();
- activeBufferWriteProcessor = new BufferWriteProcessorV2(storageGroup, new File(filePath),
- fileSchema);
-
+ workBufferWriteProcessor = new BufferWriteProcessorV2(storageGroup, new File(filePath),
+ fileSchema, versionController, this::closeBufferWriteProcessor);
+ newFileNodes.add(workBufferWriteProcessor.getTsFileResource());
// TODO check if the disk is full
}
@@ -107,27 +191,32 @@ public class FileNodeProcessorV2 {
latestFlushTimeMap.putIfAbsent(tsRecord.deviceId, Long.MIN_VALUE);
if (tsRecord.time > latestFlushTimeMap.get(tsRecord.deviceId)) {
+
// write BufferWrite
- result = activeBufferWriteProcessor.write(tsRecord);
+ result = workBufferWriteProcessor.write(tsRecord);
// try to update the latest time of the device of this tsRecord
if (result && latestTimeMap.get(tsRecord.deviceId) < tsRecord.time) {
latestTimeMap.put(tsRecord.deviceId, tsRecord.time);
}
- // check memtable size and may flush the workMemtable
- if (activeBufferWriteProcessor.shouldFlush()) {
- activeBufferWriteProcessor.flush();
+ // check memtable size and may asyncFlush the workMemtable
+ if (workBufferWriteProcessor.shouldFlush()) {
+ workBufferWriteProcessor.asyncFlush();
- // update the largest time stamp in the last flushing memtable
- latestFlushTimeMap.put(tsRecord.deviceId, tsRecord.time);
+ // update the largest timestamp in the last flushing memtable
+ for (Entry<String, Long> entry : latestTimeMap.entrySet()) {
+ latestFlushTimeMap.put(entry.getKey(), entry.getValue());
+ }
}
// check file size and may close the BufferWrite
- if (activeBufferWriteProcessor.shouldClose()) {
- closingBufferWriteProcessor.add(activeBufferWriteProcessor);
- activeBufferWriteProcessor = null;
+ if (workBufferWriteProcessor.shouldClose()) {
+ closingBufferWriteProcessor.add(workBufferWriteProcessor);
+ workBufferWriteProcessor.setClosing();
+ workBufferWriteProcessor = null;
}
+
} else {
// TODO write to overflow
}
@@ -138,7 +227,35 @@ public class FileNodeProcessorV2 {
}
return result;
+ }
+
+ /**
+ * return the memtable to MemTablePool and make metadata in writer visible
+ */
+ private void closeBufferWriteProcessor(Object bufferWriteProcessor) {
+ closingBufferWriteProcessor.remove((BufferWriteProcessorV2) bufferWriteProcessor);
+ synchronized (fileNodeProcessorStore) {
+ fileNodeProcessorStore.setLastUpdateTimeMap(latestTimeMap);
+
+ if (!newFileNodes.isEmpty()) {
+ // end time with one start time
+ Map<String, Long> endTimeMap = new HashMap<>();
+ TsFileResourceV2 resource = workBufferWriteProcessor.getTsFileResource();
+ for (Entry<String, Long> startTime : resource.getStartTimeMap().entrySet()) {
+ String deviceId = startTime.getKey();
+ endTimeMap.put(deviceId, latestTimeMap.get(deviceId));
+ }
+ resource.setEndTimeMap(endTimeMap);
+ }
+ fileNodeProcessorStore.setNewFileNodes(newFileNodes);
+ try {
+ writeStoreToDisk(fileNodeProcessorStore);
+ } catch (FileNodeProcessorException e) {
+ LOGGER.error("write FileNodeStore info error, because {}", e.getMessage(), e);
+ }
+ }
}
+
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
index e7d981f..5741037 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
@@ -19,22 +19,117 @@
package org.apache.iotdb.db.engine.filenodeV2;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.iotdb.db.conf.directories.Directories;
+import org.apache.iotdb.db.engine.filenode.OverflowChangeType;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
public class TsFileResourceV2 {
private File file;
-// private Map<String, Long> startTimeMap;
-// private Map<String, Long> endTimeMap;
+ private Map<String, Long> startTimeMap;
+
+ private Map<String, Long> endTimeMap;
- private boolean sealed;
private transient ModificationFile modFile;
+ public TsFileResourceV2(File file) {
+ this.file = file;
+ this.startTimeMap = new HashMap<>();
+ this.endTimeMap = new HashMap<>();
+ }
+
+ public TsFileResourceV2(File file, Map<String, Long> startTimeMap, Map<String, Long> endTimeMap) {
+ this.file = file;
+ this.startTimeMap = startTimeMap;
+ this.endTimeMap = endTimeMap;
+ }
+
+ public File getFile() {
+ return file;
+ }
+
public long getFileSize() {
return file.length();
}
+ public Map<String, Long> getStartTimeMap() {
+ return startTimeMap;
+ }
+
+ public void setEndTimeMap(Map<String, Long> endTimeMap) {
+ this.endTimeMap = endTimeMap;
+ }
+
+
+ public void serialize(OutputStream outputStream) throws IOException {
+// ReadWriteIOUtils.write(this.overflowChangeType.serialize(), outputStream);
+// ReadWriteIOUtils.write(this.baseDirIndex, outputStream);
+ ReadWriteIOUtils.writeIsNull(this.file, outputStream);
+ if (this.file != null) {
+ ReadWriteIOUtils.write(file.getAbsolutePath(), outputStream);
+ }
+ ReadWriteIOUtils.write(this.startTimeMap.size(), outputStream);
+ for (Entry<String, Long> entry : this.startTimeMap.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+ ReadWriteIOUtils.write(this.endTimeMap.size(), outputStream);
+ for (Entry<String, Long> entry : this.endTimeMap.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue(), outputStream);
+ }
+// ReadWriteIOUtils.write(mergeChanged.size(), outputStream);
+// for (String mergeChangedElement : this.mergeChanged) {
+// ReadWriteIOUtils.write(mergeChangedElement, outputStream);
+// }
+ }
+
+ public static TsFileResourceV2 deSerialize(InputStream inputStream) throws IOException {
+ OverflowChangeType overflowChangeType = OverflowChangeType
+ .deserialize(ReadWriteIOUtils.readShort(inputStream));
+ int baseDirIndex = ReadWriteIOUtils.readInt(inputStream);
+ boolean hasRelativePath = ReadWriteIOUtils.readIsNull(inputStream);
+
+ File file = null;
+ if (hasRelativePath) {
+ String relativePath = ReadWriteIOUtils.readString(inputStream);
+ file = new File(Directories.getInstance().getTsFileFolder(baseDirIndex), relativePath);
+ }
+ int size = ReadWriteIOUtils.readInt(inputStream);
+ Map<String, Long> startTimes = new HashMap<>();
+ for (int i = 0; i < size; i++) {
+ String path = ReadWriteIOUtils.readString(inputStream);
+ long time = ReadWriteIOUtils.readLong(inputStream);
+ startTimes.put(path, time);
+ }
+ size = ReadWriteIOUtils.readInt(inputStream);
+ Map<String, Long> endTimes = new HashMap<>();
+ for (int i = 0; i < size; i++) {
+ String path = ReadWriteIOUtils.readString(inputStream);
+ long time = ReadWriteIOUtils.readLong(inputStream);
+ endTimes.put(path, time);
+ }
+// size = ReadWriteIOUtils.readInt(inputStream);
+// Set<String> mergeChanaged = new HashSet<>();
+// for (int i = 0; i < size; i++) {
+// String path = ReadWriteIOUtils.readString(inputStream);
+// mergeChanaged.add(path);
+// }
+ TsFileResourceV2 tsFileResource = new TsFileResourceV2(file, startTimes, endTimes);
+// tsFileResource.mergeChanged = mergeChanaged;
+ return tsFileResource;
+ }
+
+
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/Callback.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/Callback.java
new file mode 100644
index 0000000..6359574
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/Callback.java
@@ -0,0 +1,7 @@
+package org.apache.iotdb.db.engine.memtable;
+
+public interface Callback {
+
+ void call(Object... object);
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
new file mode 100644
index 0000000..ef3a255
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.memtable;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.iotdb.db.utils.TimeValuePair;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
+import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.NativeRestorableIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MemTableFlushTaskV2 {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MemTableFlushTask.class);
+ private static final int PAGE_SIZE_THRESHOLD = TSFileConfig.pageSizeInByte;
+
+ private NativeRestorableIOWriter tsFileIoWriter;
+
+ private ConcurrentLinkedQueue ioTaskQueue = new ConcurrentLinkedQueue();
+ private ConcurrentLinkedQueue memoryTaskQueue = new ConcurrentLinkedQueue();
+ private boolean stop = false;
+ private String processorName;
+
+ private Callback flushCallBack;
+ private IMemTable memTable;
+
+ public MemTableFlushTaskV2(NativeRestorableIOWriter writer, String processorName,
+ Callback callBack) {
+ this.tsFileIoWriter = writer;
+ this.processorName = processorName;
+ this.flushCallBack = callBack;
+ ioFlushThread.start();
+ memoryFlushThread.start();
+ LOGGER.info("Processor {} flushMetadata task created", processorName);
+ }
+
+
+ private Thread memoryFlushThread = new Thread(() -> {
+ long memSerializeTime = 0;
+ LOGGER.info(
+ "BufferWrite Processor {},start serialize data into mem.", processorName);
+ while (!stop) {
+ Object task = memoryTaskQueue.poll();
+ if (task == null) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ LOGGER.error("BufferWrite Processor {}, io flushMetadata task is interrupted.", processorName, e);
+ }
+ } else {
+ if (task instanceof String) {
+ ioTaskQueue.add(task);
+ } else if (task instanceof ChunkGroupIoTask) {
+ ioTaskQueue.add(task);
+ } else {
+ long starTime = System.currentTimeMillis();
+ Pair<List<TimeValuePair>, MeasurementSchema> memorySerializeTask = (Pair<List<TimeValuePair>, MeasurementSchema>) task;
+ ChunkBuffer chunkBuffer = new ChunkBuffer(memorySerializeTask.right);
+ IChunkWriter seriesWriter = new ChunkWriterImpl(memorySerializeTask.right, chunkBuffer,
+ PAGE_SIZE_THRESHOLD);
+ try {
+ writeOneSeries(memorySerializeTask.left, seriesWriter,
+ memorySerializeTask.right.getType());
+ ioTaskQueue.add(seriesWriter);
+ } catch (IOException e) {
+ LOGGER.error("BufferWrite Processor {}, io error.", processorName, e);
+ throw new RuntimeException(e);
+ }
+ memSerializeTime += System.currentTimeMillis() - starTime;
+ }
+ }
+ }
+ LOGGER.info(
+ "BufferWrite Processor {},flushing a memtable into disk: serialize data into mem cost {} ms.",
+ processorName, memSerializeTime);
+ });
+
+
+ //TODO more better way is: for each TsFile, assign it a Executors.singleThreadPool,
+ // rather than per each memtable.
+ private Thread ioFlushThread = new Thread(() -> {
+ long ioTime = 0;
+ LOGGER.info("BufferWrite Processor {}, start io cost.", processorName);
+ while (!stop) {
+ Object seriesWriterOrEndChunkGroupTask = ioTaskQueue.poll();
+ if (seriesWriterOrEndChunkGroupTask == null) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ LOGGER.error("BufferWrite Processor {}, io flushMetadata task is interrupted.", processorName, e);
+ }
+ } else {
+ long starTime = System.currentTimeMillis();
+ try {
+ if (seriesWriterOrEndChunkGroupTask instanceof IChunkWriter) {
+ ((IChunkWriter) seriesWriterOrEndChunkGroupTask).writeToFileWriter(tsFileIoWriter);
+ } else if (seriesWriterOrEndChunkGroupTask instanceof String) {
+ tsFileIoWriter.startFlushChunkGroup((String) seriesWriterOrEndChunkGroupTask);
+ } else {
+ ChunkGroupIoTask task = (ChunkGroupIoTask) seriesWriterOrEndChunkGroupTask;
+ tsFileIoWriter.endChunkGroup(task.version);
+ task.finished = true;
+ }
+ } catch (IOException e) {
+ LOGGER.error("BufferWrite Processor {}, io error.", processorName, e);
+ throw new RuntimeException(e);
+ }
+ ioTime += System.currentTimeMillis() - starTime;
+ }
+ }
+
+ MemTablePool.getInstance().release(memTable);
+ LOGGER.info("Processor {} return back a memtable to MemTablePool", processorName);
+
+ tsFileIoWriter.makeMetadataVisible();
+ flushCallBack.call(memTable);
+ });
+
+
+ private void writeOneSeries(List<TimeValuePair> tvPairs, IChunkWriter seriesWriterImpl,
+ TSDataType dataType)
+ throws IOException {
+ for (TimeValuePair timeValuePair : tvPairs) {
+ switch (dataType) {
+ case BOOLEAN:
+ seriesWriterImpl
+ .write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
+ break;
+ case INT32:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(),
+ timeValuePair.getValue().getInt());
+ break;
+ case INT64:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(),
+ timeValuePair.getValue().getLong());
+ break;
+ case FLOAT:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(),
+ timeValuePair.getValue().getFloat());
+ break;
+ case DOUBLE:
+ seriesWriterImpl
+ .write(timeValuePair.getTimestamp(),
+ timeValuePair.getValue().getDouble());
+ break;
+ case TEXT:
+ seriesWriterImpl
+ .write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
+ break;
+ default:
+ LOGGER.error("BufferWrite Processor {}, don't support data type: {}", processorName,
+ dataType);
+ break;
+ }
+ }
+ }
+
+ /**
+ * the function for flushing memtable.
+ */
+ public void flushMemTable(FileSchema fileSchema, IMemTable imemTable, long version) {
+ long sortTime = 0;
+ ChunkGroupIoTask theLastTask = EMPTY_TASK;
+ this.memTable = imemTable;
+ for (String deviceId : imemTable.getMemTableMap().keySet()) {
+ memoryTaskQueue.add(deviceId);
+ int seriesNumber = imemTable.getMemTableMap().get(deviceId).size();
+ for (String measurementId : imemTable.getMemTableMap().get(deviceId).keySet()) {
+ long startTime = System.currentTimeMillis();
+ // TODO if we can not use TSFileIO writer, then we have to redesign the class of TSFileIO.
+ IWritableMemChunk series = imemTable.getMemTableMap().get(deviceId).get(measurementId);
+ MeasurementSchema desc = fileSchema.getMeasurementSchema(measurementId);
+ List<TimeValuePair> sortedTimeValuePairs = series.getSortedTimeValuePairList();
+ sortTime += System.currentTimeMillis() - startTime;
+ memoryTaskQueue.add(new Pair<>(sortedTimeValuePairs, desc));
+ }
+ theLastTask = new ChunkGroupIoTask(seriesNumber, deviceId, version);
+ memoryTaskQueue.add(theLastTask);
+ }
+ LOGGER.info(
+ "BufferWrite Processor {}, flushing a memtable into disk: data sort time cost {} ms.",
+ processorName, sortTime);
+ while (!theLastTask.finished) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ LOGGER.error("BufferWrite Processor {}, flushMetadata memtable table thread is interrupted.",
+ processorName, e);
+ throw new RuntimeException(e);
+ }
+ }
+ stop = true;
+ while (ioFlushThread.isAlive()) {
+ // wait until the after works are done
+ }
+ }
+
+
+ static class ChunkGroupIoTask {
+
+ int seriesNumber;
+ String deviceId;
+ long version;
+ boolean finished;
+
+ public ChunkGroupIoTask(int seriesNumber, String deviceId, long version) {
+ this(seriesNumber, deviceId, version, false);
+ }
+
+ public ChunkGroupIoTask(int seriesNumber, String deviceId, long version, boolean finished) {
+ this.seriesNumber = seriesNumber;
+ this.deviceId = deviceId;
+ this.version = version;
+ this.finished = finished;
+ }
+ }
+
+ private static ChunkGroupIoTask EMPTY_TASK = new ChunkGroupIoTask(0, "", 0, true);
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index 0d8554a..80c6224 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@ -199,7 +199,7 @@ public class OverflowProcessor extends Processor {
workResource.getModificationFile(), versionController, null, fileSchema,
memTable);
replayer.replayLogs();
- flushTask("recover flush", memTable, 0, (a,b) -> {});
+ flushTask("recover asyncFlush", memTable, 0, (a,b) -> {});
try {
WriteLogNode node = MultiFileLogNodeManager.getInstance().getNode(
workResource.logNodePrefix() + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
@@ -242,7 +242,7 @@ public class OverflowProcessor extends Processor {
// write data
workSupport.insert(tsRecord);
valueCount++;
- // check flush
+ // check asyncFlush
memUage = memSize.addAndGet(memUage);
if (memUage > memThreshold) {
if (LOGGER.isWarnEnabled()) {
@@ -257,7 +257,7 @@ public class OverflowProcessor extends Processor {
// write data
workSupport.insert(tsRecord);
valueCount++;
- // flush
+ // asyncFlush
memSize.addAndGet(memUage);
flush();
break;
@@ -387,7 +387,7 @@ public class OverflowProcessor extends Processor {
}
/**
- * query insert data in memory table. while flushing, merge the work memory table with flush
+ * query insert data in memory table. while flushing, merge the work memory table with asyncFlush
* memory table.
*
* @return insert data in SeriesChunkInMemTable
@@ -533,7 +533,7 @@ public class OverflowProcessor extends Processor {
try {
LOGGER.info("The overflow processor {} starts flushing {}.", getProcessorName(),
displayMessage);
- // flush data
+ // asyncFlush data
workResource
.flush(fileSchema, currentMemTableToFlush, getProcessorName(), flushId, removeFlushedMemTable);
filenodeFlushAction.act();
@@ -551,16 +551,16 @@ public class OverflowProcessor extends Processor {
Thread.currentThread().getName(), e);
result = false;
} finally {
- // switch from flush to work.
+ // switch from asyncFlush to work.
switchFlushToWork();
}
- // log flush time
+ // log asyncFlush time
if (LOGGER.isInfoEnabled()) {
LOGGER
.info("The overflow processor {} ends flushing {}.", getProcessorName(), displayMessage);
long flushEndTime = System.currentTimeMillis();
LOGGER.info(
- "The overflow processor {} flush {}, start time is {}, flush end time is {}," +
+ "The overflow processor {} asyncFlush {}, start time is {}, asyncFlush end time is {}," +
" time consumption is {}ms",
getProcessorName(), displayMessage,
DatetimeUtils.convertMillsecondToZonedDateTime(flushStartTime),
@@ -572,7 +572,7 @@ public class OverflowProcessor extends Processor {
@Override
public synchronized Future<Boolean> flush() throws IOException {
- // statistic information for flush
+ // statistic information for asyncFlush
if (lastFlushTime > 0 && LOGGER.isInfoEnabled()) {
long thisFLushTime = System.currentTimeMillis();
ZonedDateTime lastDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lastFlushTime),
@@ -580,8 +580,8 @@ public class OverflowProcessor extends Processor {
ZonedDateTime thisDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(thisFLushTime),
IoTDBDescriptor.getInstance().getConfig().getZoneID());
LOGGER.info(
- "The overflow processor {} last flush time is {}, this flush time is {},"
- + " flush time interval is {}s",
+ "The overflow processor {} last asyncFlush time is {}, this asyncFlush time is {},"
+ + " asyncFlush time interval is {}s",
getProcessorName(), lastDateTime, thisDateTime,
(thisFLushTime - lastFlushTime) / 1000);
}
@@ -620,7 +620,7 @@ public class OverflowProcessor extends Processor {
flushFuture = FlushPoolManager.getInstance().submit(() -> flushTask("asynchronously",
tmpMemTableToFlush, flushId, this::removeFlushedMemTable));
- // switch from work to flush
+ // switch from work to asyncFlush
// switchWorkToFlush();
// flushFuture = FlushPoolManager.getInstance().submit(() ->
// flushTask("asynchronously", walTaskId));
@@ -639,7 +639,7 @@ public class OverflowProcessor extends Processor {
}
LOGGER.info("The overflow processor {} starts close operation.", getProcessorName());
long closeStartTime = System.currentTimeMillis();
- // flush data
+ // asyncFlush data
try {
flush().get();
} catch (InterruptedException | ExecutionException e) {
@@ -758,16 +758,16 @@ public class OverflowProcessor extends Processor {
/**
* used for test. We can block to wait for finishing flushing.
*
- * @return the future of the flush() task.
+ * @return the future of the asyncFlush() task.
*/
public Future<Boolean> getFlushFuture() {
return flushFuture;
}
/**
- * used for test. We can know when the flush() is called.
+ * used for test. We can know when the asyncFlush() is called.
*
- * @return the last flush() time.
+ * @return the last asyncFlush() time.
*/
public long getLastFlushTime() {
return lastFlushTime;
@@ -814,7 +814,7 @@ public class OverflowProcessor extends Processor {
// queryFlushLock.lock();
// try {
// if (flushTaskList.isEmpty()) {
-// // flushTaskList is empty, thus all flush tasks have done and switch
+// // flushTaskList is empty, thus all asyncFlush tasks have done and switch
// OverflowMemtable temp = flushSupport == null ? new OverflowMemtable() : flushSupport;
// flushSupport = workSupport;
// workSupport = temp;
@@ -825,7 +825,7 @@ public class OverflowProcessor extends Processor {
// } finally {
// queryFlushLock.unlock();
// }
-// flush(flushInfo);
+// asyncFlush(flushInfo);
// }
// }
// }
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
index 2496d42..a5fa4da 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/NativeRestorableIOWriter.java
@@ -21,9 +21,14 @@ package org.apache.iotdb.tsfile.write.writer;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileCheckStatus;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
@@ -41,6 +46,14 @@ public class NativeRestorableIOWriter extends TsFileIOWriter {
private long truncatedPosition = -1;
private Map<String, MeasurementSchema> knownSchemas = new HashMap<>();
+ private int lastFlushedChunkGroupIndex = 0;
+
+ /**
+ * all chunk group metadata which have been serialized on disk.
+ */
+ private Map<String, Map<String, List<ChunkMetaData>>> metadatas;
+
+
long getTruncatedPosition() {
return truncatedPosition;
}
@@ -80,7 +93,7 @@ public class NativeRestorableIOWriter extends TsFileIOWriter {
out.truncate(TSFileConfig.MAGIC_STRING.length());
} else {
//remove broken data
- out.truncate(truncatedPosition);
+ out.truncate(truncatedPosition + 1);
}
}
}
@@ -90,4 +103,75 @@ public class NativeRestorableIOWriter extends TsFileIOWriter {
public Map<String, MeasurementSchema> getKnownSchema() {
return knownSchemas;
}
+
+
+ /**
+ * For query.
+ *
+ * get chunks' metadata from memory.
+ *
+ * @param deviceId the device id
+ * @param measurementId the sensor id
+ * @param dataType the value type
+ * @return chunks' metadata
+ */
+ public List<ChunkMetaData> getMetadatas(String deviceId, String measurementId, TSDataType dataType) {
+ List<ChunkMetaData> chunkMetaDatas = new ArrayList<>();
+ if (metadatas.containsKey(deviceId) && metadatas.get(deviceId).containsKey(measurementId)) {
+ for (ChunkMetaData chunkMetaData : metadatas.get(deviceId).get(measurementId)) {
+ // filter: if a device'sensor is defined as float type, and data has been persistent.
+ // Then someone deletes the timeseries and recreate it with Int type. We have to ignore
+ // all the stale data.
+ if (dataType.equals(chunkMetaData.getTsDataType())) {
+ chunkMetaDatas.add(chunkMetaData);
+ }
+ }
+ }
+ return chunkMetaDatas;
+ }
+
+
+ /**
+ * add all appendChunkGroupMetadatas into memory. After calling this method, other classes can
+ * read these metadata.
+ */
+ public void makeMetadataVisible() {
+
+ List<ChunkGroupMetaData> newlyFlushedMetadatas = getAppendedRowGroupMetadata();
+
+ if (!newlyFlushedMetadatas.isEmpty()) {
+ for (ChunkGroupMetaData rowGroupMetaData : newlyFlushedMetadatas) {
+ String deviceId = rowGroupMetaData.getDeviceID();
+ for (ChunkMetaData chunkMetaData : rowGroupMetaData.getChunkMetaDataList()) {
+ String measurementId = chunkMetaData.getMeasurementUid();
+ if (!metadatas.containsKey(deviceId)) {
+ metadatas.put(deviceId, new HashMap<>());
+ }
+ if (!metadatas.get(deviceId).containsKey(measurementId)) {
+ metadatas.get(deviceId).put(measurementId, new ArrayList<>());
+ }
+ metadatas.get(deviceId).get(measurementId).add(chunkMetaData);
+
+ }
+ }
+ }
+ }
+
+
+ /**
+ * get all the chunkGroups' metadata which are appended after the last calling of this method, or
+ * after the class instance is initialized if this is the first time to call the method.
+ *
+ * @return a list of chunkgroup metadata
+ */
+ private List<ChunkGroupMetaData> getAppendedRowGroupMetadata() {
+ List<ChunkGroupMetaData> append = new ArrayList<>();
+ if (lastFlushedChunkGroupIndex < chunkGroupMetaDataList.size()) {
+ append.addAll(chunkGroupMetaDataList.subList(lastFlushedChunkGroupIndex, chunkGroupMetaDataList.size()));
+ lastFlushedChunkGroupIndex = chunkGroupMetaDataList.size();
+ }
+ return append;
+ }
+
+
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 55413ad..dfd9723 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -66,7 +66,6 @@ public class TsFileIOWriter {
}
protected TsFileOutput out;
- private AtomicLong flushID = new AtomicLong(0);
protected List<ChunkGroupMetaData> chunkGroupMetaDataList = new ArrayList<>();
private ChunkGroupMetaData currentChunkGroupMetaData;
private ChunkMetaData currentChunkMetaData;
@@ -134,7 +133,7 @@ public class TsFileIOWriter {
}
public AtomicLong getFlushID() {
- return flushID;
+ return new AtomicLong();
}
/**