You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/05/29 00:58:17 UTC
[incubator-iotdb] branch refactor_overflow updated (ffedb93 ->
57cb4a8)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from ffedb93 update merge
new 39e0889 migrate functions of FileNodeManager to StorageGroupManager
new 1ea3bff abstract StorageGroupManager as StorageEngine
new 245cc54 rename StorageEngine to DatabaseEngine
new 02647ac migrate usages of FileNodeProcessor
new 57cb4a8 rename querycontext to datasource remove unused classes
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/engine/DatabaseEngine.java | 167 ++
.../Policy.java => DatabaseEngineFactory.java} | 15 +-
...ileNodeConstants.java => EngingeConstants.java} | 16 +-
.../java/org/apache/iotdb/db/engine/Processor.java | 1 -
.../apache/iotdb/db/engine/bufferwrite/Action.java | 28 -
.../db/engine/bufferwrite/ActionException.java | 30 -
.../engine/bufferwrite/BufferWriteProcessor.java | 560 ------
.../engine/cache/RowGroupBlockMetaDataCache.java | 167 --
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 98 -
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 84 -
.../QueryDataSource.java | 2 +-
.../ReadOnlyMemChunk.java | 8 +-
.../SeriesDataSource.java | 16 +-
.../UnsealedTsFile.java | 2 +-
.../db/engine/filenode/FileNodeFlushFuture.java | 91 -
.../iotdb/db/engine/filenode/FileNodeManager.java | 1216 ------------
.../db/engine/filenode/FileNodeProcessor.java | 2064 --------------------
.../engine/filenode/FileNodeProcessorStatus.java | 51 -
.../db/engine/filenode/FileNodeProcessorStore.java | 165 --
.../db/engine/filenode/OverflowChangeType.java | 57 -
.../db/engine/memcontrol/FlushPartialPolicy.java | 6 +-
.../db/engine/memcontrol/ForceFLushAllPolicy.java | 4 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 3 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 3 +-
.../db/engine/memtable/MemSeriesLazyMerger.java | 2 +-
.../iotdb/db/engine/overflow/io/OverflowIO.java | 173 --
.../db/engine/overflow/io/OverflowMemtable.java | 116 --
.../db/engine/overflow/io/OverflowProcessor.java | 736 -------
.../db/engine/overflow/io/OverflowResource.java | 329 ----
.../overflow/io/OverflowedTsFileIOWriter.java | 36 -
.../engine/overflow/metadata/OFFileMetadata.java | 107 -
.../overflow/metadata/OFRowGroupListMetadata.java | 109 --
.../overflow/metadata/OFSeriesListMetadata.java | 107 -
.../db/engine/overflow/utils/MergeStatus.java | 29 -
.../db/engine/overflow/utils/OverflowOpType.java | 29 -
.../apache/iotdb/db/engine/pool/FlushManager.java | 56 +-
.../apache/iotdb/db/engine/pool/MergeManager.java | 52 +-
.../engine/querycontext/MergeSeriesDataSource.java | 33 -
.../db/engine/querycontext/OverflowInsertFile.java | 51 -
.../querycontext/OverflowSeriesDataSource.java | 78 -
.../querycontext/OverflowUpdateDeleteFile.java | 42 -
.../db/engine/sgmanager/StorageGroupManager.java | 534 +++--
.../db/engine/sgmanager/StorageGroupProcessor.java | 23 +-
.../{filenode => sgmanager}/TsFileResource.java | 8 +-
.../RestorableTsFileIOWriter.java | 2 +-
.../db/engine/tsfiledata/TsFileProcessor.java | 24 +-
.../org/apache/iotdb/db/monitor/IStatistic.java | 4 +-
.../apache/iotdb/db/monitor/MonitorConstants.java | 4 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 83 +-
.../iotdb/db/monitor/collector/FileSize.java | 16 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 61 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 21 +-
.../iotdb/db/query/control/JobFileManager.java | 6 +-
.../db/query/control/QueryResourceManager.java | 68 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../EngineExecutorWithoutTimeGenerator.java | 64 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 17 -
.../db/query/executor/FillEngineExecutor.java | 2 +-
.../GroupByWithOnlyTimeFilterDataSetDataSet.java | 2 +-
.../db/query/factory/SeriesReaderFactory.java | 21 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../org/apache/iotdb/db/query/fill/LinearFill.java | 2 +-
.../apache/iotdb/db/query/fill/PreviousFill.java | 2 +-
.../iotdb/db/query/reader/mem/MemChunkReader.java | 2 +-
.../query/reader/sequence/SealedTsFilesReader.java | 2 +-
.../sequence/SealedTsFilesReaderByTimestamp.java | 2 +-
.../query/reader/sequence/SequenceDataReader.java | 2 +-
.../sequence/SequenceDataReaderByTimestamp.java | 2 +-
.../reader/sequence/UnSealedTsFileReader.java | 2 +-
.../sequence/UnSealedTsFilesReaderByTimestamp.java | 2 +-
.../query/timegenerator/EngineNodeConstructor.java | 2 +-
.../apache/iotdb/db/service/CloseMergeService.java | 5 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 7 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 10 +-
.../java/org/apache/iotdb/db/sync/SyncUtils.java | 2 +-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 30 +-
.../org/apache/iotdb/db/utils/LoadDataUtils.java | 11 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 2 +-
.../recover/ExclusiveLogRecoverPerformer.java | 3 +-
.../writelog/recover/FileNodeRecoverPerformer.java | 4 +-
.../db/writelog/replay/ConcreteLogReplayer.java | 30 +-
.../engine/bufferwrite/BufferWriteBenchmark.java | 13 +-
.../bufferwrite/BufferWriteProcessorNewTest.java | 9 +-
.../bufferwrite/BufferWriteProcessorTest.java | 9 +-
.../bufferwrite/RestorableTsFileIOWriterTest.java | 1 +
.../filenodev2/FileNodeManagerBenchmark.java | 6 +-
.../memcontrol/BufferwriteFileSizeControlTest.java | 8 +-
.../memcontrol/BufferwriteMetaSizeControlTest.java | 8 +-
.../memcontrol/OverflowFileSizeControlTest.java | 10 +-
.../memcontrol/OverflowMetaSizeControlTest.java | 10 +-
.../engine/modification/DeletionFileNodeTest.java | 54 +-
.../db/engine/modification/DeletionQueryTest.java | 88 +-
.../overflow/io/OverflowProcessorBenchmark.java | 10 +-
.../engine/overflow/io/OverflowProcessorTest.java | 10 +-
.../integration/QueryDataFromUnclosedTsFileIT.java | 6 +-
.../org/apache/iotdb/db/monitor/MonitorTest.java | 20 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 10 +-
98 files changed, 758 insertions(+), 7543 deletions(-)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
copy iotdb/src/main/java/org/apache/iotdb/db/engine/{memcontrol/Policy.java => DatabaseEngineFactory.java} (77%)
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{bufferwrite/FileNodeConstants.java => EngingeConstants.java} (68%)
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/Action.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/ActionException.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{querycontext => datasource}/QueryDataSource.java (96%)
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{querycontext => datasource}/ReadOnlyMemChunk.java (96%)
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{querycontext => datasource}/SeriesDataSource.java (82%)
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{querycontext => datasource}/UnsealedTsFile.java (96%)
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeFlushFuture.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStatus.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStore.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/OverflowChangeType.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowIO.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowedTsFileIOWriter.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFFileMetadata.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFRowGroupListMetadata.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFSeriesListMetadata.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/MergeStatus.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/OverflowOpType.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/MergeSeriesDataSource.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowInsertFile.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowSeriesDataSource.java
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowUpdateDeleteFile.java
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{filenode => sgmanager}/TsFileResource.java (97%)
rename iotdb/src/main/java/org/apache/iotdb/db/engine/{bufferwrite => tsfiledata}/RestorableTsFileIOWriter.java (99%)
[incubator-iotdb] 01/05: migrate functions of FileNodeManager to
StorageGroupManager
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 39e0889107dea42466bc74e43a4d419473619f52
Author: 江天 <jt...@163.com>
AuthorDate: Tue May 28 12:17:29 2019 +0800
migrate functions of FileNodeManager to StorageGroupManager
---
.../iotdb/db/engine/filenode/FileNodeManager.java | 5 +-
.../db/engine/filenode/FileNodeProcessor.java | 13 +-
.../db/engine/sgmanager/StorageGroupManager.java | 412 ++++++++++-----------
.../db/engine/sgmanager/StorageGroupProcessor.java | 12 +-
.../db/engine/tsfiledata/TsFileProcessor.java | 4 +-
.../apache/iotdb/db/monitor/MonitorConstants.java | 4 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 4 +-
7 files changed, 220 insertions(+), 234 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index e291e88..0404a5c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@ -51,6 +51,7 @@ import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -301,10 +302,10 @@ public class FileNodeManager implements IStatistic, IService {
// Modify the insert
if (!isMonitor) {
fileNodeProcessor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
+ .get(StorageGroupProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
.addAndGet(tsRecord.dataPointList.size());
fileNodeProcessor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
+ .get(StorageGroupProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
index 508d689..a66116e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
@@ -76,6 +76,7 @@ import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
@@ -246,8 +247,8 @@ public class FileNodeProcessor extends Processor implements IStatistic {
FileNodeProcessor(String fileNodeDirPath, String processorName)
throws FileNodeProcessorException {
super(processorName);
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants statConstant :
+ StorageGroupProcessorStatConstants.values()) {
statParamsHashMap.put(statConstant.name(), new AtomicLong(0));
}
statStorageDeltaName =
@@ -327,8 +328,8 @@ public class FileNodeProcessor extends Processor implements IStatistic {
@Override
public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants statConstant :
+ StorageGroupProcessorStatConstants.values()) {
hashMap
.put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
MonitorConstants.DATA_TYPE_INT64);
@@ -339,8 +340,8 @@ public class FileNodeProcessor extends Processor implements IStatistic {
@Override
public List<String> getAllPathForStatistic() {
List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants statConstant :
+ StorageGroupProcessorStatConstants.values()) {
list.add(
statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index 51ed0e3..d313c38 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -36,15 +36,13 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.filenode.FileNodeProcessor;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.pool.FlushManager;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
+import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
@@ -80,7 +78,7 @@ public class StorageGroupManager implements IStatistic, IService {
private static final Directories directories = Directories.getInstance();
/**
- * This map is used to manage all StorageGroupProcessors,<br> the key is filenode name which is
+ * This map is used to manage all StorageGroupProcessors,<br> the key is storage group name which is
* storage group seriesPath.
* e.g. if the user executes a query "SET STORAGE GROUP TO root.beijing", then "root.beijing" will
* be a key.
@@ -88,9 +86,9 @@ public class StorageGroupManager implements IStatistic, IService {
private ConcurrentHashMap<String, StorageGroupProcessor> processorMap;
/**
- * fileNodeManagerStatus indicates whether the Manager is merging or being closed.
+ * storageGroupManagerStatus indicates whether the Manager is merging or being closed.
*/
- private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ private volatile StorageGroupManagerStatus storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
private HashMap<String, AtomicLong> statParamsHashMap;
@@ -101,9 +99,9 @@ public class StorageGroupManager implements IStatistic, IService {
private void initStat() {
statParamsHashMap = new HashMap<>();
- for (StorageGroupManagerStatConstants fileNodeManagerStatConstant :
+ for (StorageGroupManagerStatConstants StorageGroupManagerStatConstant :
StorageGroupManagerStatConstants.values()) {
- statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
+ statParamsHashMap.put(StorageGroupManagerStatConstant.name(), new AtomicLong(0));
}
if (TsFileDBConf.isEnableStatMonitor()) {
@@ -114,7 +112,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
public static StorageGroupManager getInstance() {
- return FileNodeManagerHolder.INSTANCE;
+ return StorageGroupManagerHolder.INSTANCE;
}
private void updateStatHashMapWhenFail(TSRecord tsRecord) {
@@ -174,7 +172,7 @@ public class StorageGroupManager implements IStatistic, IService {
/**
* This function is just for unit test.
*/
- public synchronized void resetFileNodeManager() {
+ public synchronized void resetStorageGroupManager() {
for (String key : statParamsHashMap.keySet()) {
statParamsHashMap.put(key, new AtomicLong());
}
@@ -182,49 +180,49 @@ public class StorageGroupManager implements IStatistic, IService {
}
/**
- * @param filenodeName storage name, e.g., root.a.b
+ * @param processorName storage name, e.g., root.a.b
*/
- private StorageGroupProcessor constructNewProcessor(String filenodeName)
+ private StorageGroupProcessor constructNewProcessor(String processorName)
throws StorageGroupManagerException {
try {
- return new StorageGroupProcessor(filenodeName);
+ return new StorageGroupProcessor(processorName);
} catch (TsFileProcessorException e) {
throw new StorageGroupManagerException(String.format("Can't construct the "
- + "StorageGroupProcessor, the StorageGroup is %s", filenodeName), e);
+ + "StorageGroupProcessor, the StorageGroup is %s", processorName), e);
}
}
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private StorageGroupProcessor getProcessor(String path, boolean isWriteLock)
throws StorageGroupManagerException {
- String filenodeName;
+ String storageGroupName;
try {
// return the stroage name
- filenodeName = MManager.getInstance().getStorageGroupByPath(path);
+ storageGroupName = MManager.getInstance().getStorageGroupByPath(path);
} catch (PathErrorException e) {
throw new StorageGroupManagerException(
String.format("MManager get StorageGroup name error, seriesPath is %s", path), e);
}
StorageGroupProcessor processor;
- processor = processorMap.get(filenodeName);
+ processor = processorMap.get(storageGroupName);
if (processor != null) {
processor.lock(isWriteLock);
} else {
- filenodeName = filenodeName.intern();
+ storageGroupName = storageGroupName.intern();
// calculate the value with same key synchronously
- synchronized (filenodeName) {
- processor = processorMap.get(filenodeName);
+ synchronized (storageGroupName) {
+ processor = processorMap.get(storageGroupName);
if (processor != null) {
processor.lock(isWriteLock);
} else {
// calculate the value with the key monitor
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
- filenodeName, Thread.currentThread().getId());
+ LOGGER.debug("construct a processor instance, the storage group is {}, Thread is {}",
+ storageGroupName, Thread.currentThread().getId());
}
- processor = constructNewProcessor(filenodeName);
+ processor = constructNewProcessor(storageGroupName);
processor.lock(isWriteLock);
- processorMap.put(filenodeName, processor);
+ processorMap.put(storageGroupName, processor);
}
}
}
@@ -235,22 +233,22 @@ public class StorageGroupManager implements IStatistic, IService {
* recover the StorageGroupProcessors.
*/
public void recover() throws StorageGroupManagerException {
- List<String> filenodeNames;
+ List<String> storageGroupNames;
try {
- filenodeNames = MManager.getInstance().getAllStorageGroups();
+ storageGroupNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
- throw new StorageGroupManagerException("Restoring all FileNodes failed.", e);
+ throw new StorageGroupManagerException("Restoring all StorageGroups failed.", e);
}
- for (String filenodeName : filenodeNames) {
- StorageGroupProcessor fileNodeProcessor = null;
+ for (String storageGroupName : storageGroupNames) {
+ StorageGroupProcessor StorageGroupProcessor = null;
try {
- fileNodeProcessor = getProcessor(filenodeName, true);
+ StorageGroupProcessor = getProcessor(storageGroupName, true);
} catch (StorageGroupManagerException e) {
- throw new StorageGroupManagerException(String.format("Restoring fileNode %s failed.",
- filenodeName), e);
+ throw new StorageGroupManagerException(String.format("Restoring StorageGroup %s failed.",
+ storageGroupName), e);
} finally {
- if (fileNodeProcessor != null) {
- fileNodeProcessor.writeUnlock();
+ if (StorageGroupProcessor != null) {
+ StorageGroupProcessor.writeUnlock();
}
}
}
@@ -287,10 +285,10 @@ public class StorageGroupManager implements IStatistic, IService {
// Modify the insert
if (!isMonitor) {
processor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
+ .get(MonitorConstants.StorageGroupProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
.addAndGet(plan.getValues().length);
processor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
+ .get(MonitorConstants.StorageGroupProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
@@ -361,7 +359,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
private void deleteProcessor(String processorName,
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
+ Iterator<Map.Entry<String, StorageGroupProcessor>> processorIterator)
throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
LOGGER.warn("The processorMap doesn't contain the StorageGroupProcessor {}.", processorName);
@@ -431,7 +429,7 @@ public class StorageGroupManager implements IStatistic, IService {
StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Get the FileNodeProcessor: storage group is {}, begin query.",
+ LOGGER.debug("Get the StorageGroupProcessor: storage group is {}, begin query.",
processor.getProcessorName());
}
return processor.addMultiPassCount();
@@ -449,7 +447,7 @@ public class StorageGroupManager implements IStatistic, IService {
StorageGroupProcessor processorrocessor = getProcessor(deviceId, true);
try {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Get the FileNodeProcessor: {} end query.",
+ LOGGER.debug("Get the StorageGroupProcessor: {} end query.",
processorrocessor.getProcessorName());
}
processorrocessor.decreaseMultiPassCount(token);
@@ -471,7 +469,7 @@ public class StorageGroupManager implements IStatistic, IService {
String deviceId = seriesExpression.getSeriesPath().getDevice();
StorageGroupProcessor processor = getProcessor(deviceId, false);
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
+ LOGGER.debug("Get the StorageGroupProcessor: storage group is {}, query.",
processor.getProcessorName());
}
try {
@@ -492,12 +490,12 @@ public class StorageGroupManager implements IStatistic, IService {
* Append one specified tsfile to the storage group. <b>This method is only provided for
* transmission module</b>
*
- * @param fileNodeName the seriesPath of storage group
+ * @param storageGroupName the seriesPath of storage group
* @param appendFile the appended tsfile information
*/
- public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
+ public boolean appendFileToStorageGroup(String storageGroupName, TsFileResource appendFile,
String appendFilePath) throws StorageGroupManagerException {
- StorageGroupProcessor processor = getProcessor(fileNodeName, true);
+ StorageGroupProcessor processor = getProcessor(storageGroupName, true);
try {
// check append file
for (Map.Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
@@ -510,7 +508,7 @@ public class StorageGroupManager implements IStatistic, IService {
// append file to storage group.
processor.appendFile(appendFile, appendFilePath);
} catch (TsFileProcessorException e) {
- LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
+ LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), storageGroupName, e);
throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
@@ -521,63 +519,79 @@ public class StorageGroupManager implements IStatistic, IService {
/**
* get all overlap tsfiles which are conflict with the appendFile.
*
- * @param fileNodeName the seriesPath of storage group
+ * @param storageGroupName the seriesPath of storage group
* @param appendFile the appended tsfile information
*/
- public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
+ public List<String> getOverlapFilesFromStorageGroup(String storageGroupName, TsFileResource appendFile,
String uuid) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
+ StorageGroupProcessor processor = getProcessor(storageGroupName, true);
List<String> overlapFiles;
try {
- overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
- } catch (FileNodeProcessorException e) {
+ overlapFiles = processor.getOverlapFiles(appendFile, uuid);
+ } catch (TsFileProcessorException e) {
throw new StorageGroupManagerException(e);
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
return overlapFiles;
}
/**
- * merge all overflowed filenode.
+ * merge all overflowed storage group.
*
* @throws StorageGroupManagerException StorageGroupManagerException
*/
public void mergeAll() throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
- fileNodeManagerStatus);
+ if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
+ LOGGER.warn("Unable to merge all storage groups when the status is {}",
+ storageGroupManagerStatus);
return;
}
- fileNodeManagerStatus = FileNodeManagerStatus.MERGE;
- LOGGER.info("Start to merge all overflowed filenode");
- List<String> allFileNodeNames;
+ storageGroupManagerStatus = StorageGroupManagerStatus.MERGE;
+ LOGGER.info("Start to merge all storage groups");
+ List<String> storageGroupNames;
try {
- allFileNodeNames = MManager.getInstance().getAllStorageGroups();
+ storageGroupNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
- LOGGER.error("Get all storage group seriesPath error,", e);
throw new StorageGroupManagerException(e);
}
- List<Future<?>> futureTasks = new ArrayList<>();
- for (String fileNodeName : allFileNodeNames) {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
+ List<Future> futureTasks = new ArrayList<>();
+ for (String storageGroupName : storageGroupNames) {
+ StorageGroupProcessor processor = getProcessor(storageGroupName, true);
try {
- Future<?> task = fileNodeProcessor.submitToMerge();
+ Future task = processor.submitToMerge();
if (task != null) {
- LOGGER.info("Submit the filenode {} to the merge pool", fileNodeName);
+ LOGGER.info("Submit the storage group {} to the merge pool", storageGroupName);
futureTasks.add(task);
}
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
+ }
+ }
+ List<Exception> mergeExceptions = new ArrayList<>();
+ waitForMergeTasks(futureTasks, mergeExceptions);
+
+ if (!mergeExceptions.isEmpty()) {
+ LOGGER.error("There are {} errors when merging storage groups:", mergeExceptions.size());
+ for (Exception exception : mergeExceptions) {
+ LOGGER.error("", exception);
}
+ throw new StorageGroupManagerException(String.format("Merge encountered %d errors,"
+ + " see previous logs for details", mergeExceptions.size()));
}
+ storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
+ LOGGER.info("End to merge all storage groups");
+ }
+
+ private void waitForMergeTasks(List<Future> tasks, List<Exception> exceptions) {
long totalTime = 0;
// loop waiting for merge to end, the longest waiting time is
// 60s.
int time = 2;
- List<Exception> mergeException = new ArrayList<>();
- for (Future<?> task : futureTasks) {
+ int singleWaitThreshold = 60;
+
+ for (Future<?> task : tasks) {
while (!task.isDone()) {
try {
LOGGER.info(
@@ -586,42 +600,36 @@ public class StorageGroupManager implements IStatistic, IService {
totalTime, time);
TimeUnit.SECONDS.sleep(time);
totalTime += time;
- time = updateWaitTime(time);
+ time = updateWaitTime(time, singleWaitThreshold);
} catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
+ LOGGER.error("Unexpected interruption when waiting for merge", e);
Thread.currentThread().interrupt();
}
}
try {
task.get();
} catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
+ LOGGER.error("Unexpected interruption when receiving merge result", e);
+ Thread.currentThread().interrupt();
} catch (ExecutionException e) {
- mergeException.add(e);
- LOGGER.error("The exception for merge: {}", e);
+ exceptions.add(e);
}
}
- if (!mergeException.isEmpty()) {
- // just throw the first exception
- throw new StorageGroupManagerException(mergeException.get(0));
- }
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- LOGGER.info("End to merge all overflowed filenode");
}
- private int updateWaitTime(int time) {
- return time < 32 ? time * 2 : 60;
+ private int updateWaitTime(int time, int threshold) {
+ return time * 2 < threshold ? time * 2 : threshold;
}
/**
- * try to close the filenode processor. The name of filenode processor is processorName
+ * try to close the storage group processor. The name of storage group processor is processorName
*/
private boolean closeOneProcessor(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
return true;
}
- Processor processor = processorMap.get(processorName);
+ StorageGroupProcessor processor = processorMap.get(processorName);
if (processor.tryWriteLock()) {
try {
if (processor.canBeClosed()) {
@@ -630,8 +638,8 @@ public class StorageGroupManager implements IStatistic, IService {
} else {
return false;
}
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
+ } catch (TsFileProcessorException e) {
+ LOGGER.error("Close the storage group processor {} error.", processorName, e);
throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
@@ -642,87 +650,91 @@ public class StorageGroupManager implements IStatistic, IService {
}
/**
- * delete one filenode.
+ * delete one storage group.
*/
- public void deleteOneFileNode(String processorName) throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
+ public void deleteOneStorageGroup(String processorName) throws StorageGroupManagerException {
+ if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
return;
}
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.CLOSE;
try {
if (processorMap.containsKey(processorName)) {
- deleteFileNodeBlocked(processorName);
+ deleteStorageGroupBlocked(processorName);
}
- String fileNodePath = TsFileDBConf.getFileNodeDir();
- fileNodePath = standardizeDir(fileNodePath) + processorName;
- FileUtils.deleteDirectory(new File(fileNodePath));
- cleanBufferWrite(processorName);
+ cleanDataFiles(processorName);
MultiFileLogNodeManager.getInstance()
.deleteNode(processorName + IoTDBConstant.BUFFERWRITE_LOG_NODE_SUFFIX);
MultiFileLogNodeManager.getInstance()
.deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
} catch (IOException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
+ LOGGER.error("Delete the storage group processor {} error.", processorName, e);
throw new StorageGroupManagerException(e);
} finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
}
}
- private void cleanBufferWrite(String processorName) throws IOException {
- List<String> bufferwritePathList = directories.getAllTsFileFolders();
- for (String bufferwritePath : bufferwritePathList) {
- bufferwritePath = standardizeDir(bufferwritePath) + processorName;
- File bufferDir = new File(bufferwritePath);
+ private void cleanDataFiles(String processorName) throws IOException {
+ List<String> seqFilePathList = directories.getAllTsFileFolders();
+ List<String> overflowFilePathList = directories.getAllOverflowFileFolders();
+ cleanDirectories(seqFilePathList, processorName);
+ cleanDirectories(overflowFilePathList, processorName);
+ }
+
+ private void cleanDirectories(List<String> directoryPaths, String processorName) throws IOException {
+ for (String directoryPath : directoryPaths) {
+ directoryPath = standardizeDir(directoryPath) + processorName;
+ File directory = new File(directoryPath);
// free and close the streams under this bufferwrite directory
- if (!bufferDir.exists()) {
+ if (!directory.exists()) {
continue;
}
- File[] bufferFiles = bufferDir.listFiles();
- if (bufferFiles != null) {
- for (File bufferFile : bufferFiles) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(bufferFile.getPath());
+ File[] dataFiles = directory.listFiles();
+ if (dataFiles != null) {
+ for (File dataFile : dataFiles) {
+ FileReaderManager.getInstance().closeFileAndRemoveReader(dataFile.getPath());
}
}
- FileUtils.deleteDirectory(new File(bufferwritePath));
+ FileUtils.deleteDirectory(new File(directoryPath));
}
}
- private void deleteFileNodeBlocked(String processorName) throws StorageGroupManagerException {
- LOGGER.info("Forced to delete the filenode processor {}", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
+ private void deleteStorageGroupBlocked(String processorName) throws StorageGroupManagerException {
+ LOGGER.info("Forced to delete the storage group processor {}", processorName);
+ StorageGroupProcessor processor = processorMap.get(processorName);
while (true) {
if (processor.tryWriteLock()) {
try {
if (processor.canBeClosed()) {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
+ LOGGER.info("Delete the storage group processor {}.", processorName);
+ processor.close();
processorMap.remove(processorName);
break;
} else {
LOGGER.info(
- "Can't delete the filenode processor {}, "
- + "because the filenode processor can't be closed."
+ "Can't delete the storage group processor {}, "
+ + "because the storage group processor can't be closed."
+ " Wait 100ms to retry");
}
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String.
+ format("Delete the storage group processor %s error.", processorName), e);
} finally {
processor.writeUnlock();
}
} else {
LOGGER.info(
- "Can't delete the filenode processor {}, because it can't get the write lock."
+ "Can't delete the storage group processor {}, because it is already locked"
+ " Wait 100ms to retry", processorName);
}
try {
TimeUnit.MILLISECONDS.sleep(100);
} catch (InterruptedException e) {
- LOGGER.error(e.getMessage());
+ LOGGER.error("Unexpected interruption when waiting to delete storage group {}",
+ processorName);
Thread.currentThread().interrupt();
}
}
@@ -744,67 +756,68 @@ public class StorageGroupManager implements IStatistic, IService {
public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor,
Map<String, String> props) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
+ StorageGroupProcessor processor = getProcessor(path.getFullPath(), true);
try {
- fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
+ processor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
}
/**
- * Force to close the filenode processor.
+ * Force to close the storage group processor.
*/
- public void closeOneFileNode(String processorName) throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
+ public void closeStorageGroup(String processorName) throws StorageGroupManagerException {
+ if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
return;
}
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.CLOSE;
try {
- LOGGER.info("Force to close the filenode processor {}.", processorName);
+ LOGGER.info("Force to close the storage group processor {}.", processorName);
while (!closeOneProcessor(processorName)) {
try {
- LOGGER.info("Can't force to close the filenode processor {}, wait 100ms to retry",
+ LOGGER.info("Can't force to close the storage group processor {}, wait 100ms to retry",
processorName);
TimeUnit.MILLISECONDS.sleep(100);
} catch (InterruptedException e) {
// ignore the interrupted exception
- LOGGER.error("Unexpected interruption {}", e);
+ LOGGER.error("Unexpected interruption {} when closing storage group {}",processorName,
+ e);
Thread.currentThread().interrupt();
}
}
} finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
}
}
/**
- * try to close the filenode processor.
+ * try to close the storage group processor.
*/
- private void close(String processorName) throws StorageGroupManagerException {
+ private void closeStorageGroupPrivate(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
+ LOGGER.warn("The processorMap doesn't contain the storage group {}.", processorName);
return;
}
- LOGGER.info("Try to close the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
+ LOGGER.info("Try to close the storage group processor {}.", processorName);
+ StorageGroupProcessor processor = processorMap.get(processorName);
if (!processor.tryWriteLock()) {
- LOGGER.warn("Can't get the write lock of the filenode processor {}.", processorName);
+ LOGGER.warn("Can't get the write lock of the storage group processor {}.", processorName);
return;
}
try {
if (processor.canBeClosed()) {
try {
- LOGGER.info("Close the filenode processor {}.", processorName);
+ LOGGER.info("Close the storage group processor {}.", processorName);
processor.close();
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String
+ .format("Close the storage group processor %s error.", processorName), e);
}
} else {
- LOGGER.warn("The filenode processor {} can't be closed.", processorName);
+ LOGGER.warn("The storage group processor {} can't be closed.", processorName);
}
} finally {
processor.writeUnlock();
@@ -812,27 +825,27 @@ public class StorageGroupManager implements IStatistic, IService {
}
/**
- * delete all filenode.
+ * delete all storage groups.
*/
public synchronized boolean deleteAll() throws StorageGroupManagerException {
- LOGGER.info("Start deleting all filenode");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to delete all filenode processor because of merge operation");
+ LOGGER.info("Start deleting all storage group");
+ if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
+ LOGGER.info("Failed to delete all storage group processor because of merge operation");
return false;
}
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.CLOSE;
try {
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator = processorMap.entrySet()
+ Iterator<Map.Entry<String, StorageGroupProcessor>> processorIterator = processorMap.entrySet()
.iterator();
while (processorIterator.hasNext()) {
- Map.Entry<String, FileNodeProcessor> processorEntry = processorIterator.next();
+ Map.Entry<String, StorageGroupProcessor> processorEntry = processorIterator.next();
deleteProcessor(processorEntry.getKey(), processorIterator);
}
return processorMap.isEmpty();
} finally {
- LOGGER.info("Deleting all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ LOGGER.info("Deleting all StorageGroupProcessors ends");
+ storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
}
}
@@ -840,19 +853,19 @@ public class StorageGroupManager implements IStatistic, IService {
* Try to close All.
*/
public void closeAll() throws StorageGroupManagerException {
- LOGGER.info("Start closing all filenode processor");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to close all filenode processor because of merge operation");
+ LOGGER.info("Start closing all storage group processor");
+ if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
+ LOGGER.info("Failed to close all storage group processor because of merge operation");
return;
}
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
+ storageGroupManagerStatus = StorageGroupManagerStatus.CLOSE;
try {
- for (Map.Entry<String, FileNodeProcessor> processorEntry : processorMap.entrySet()) {
- close(processorEntry.getKey());
+ for (Map.Entry<String, StorageGroupProcessor> processorEntry : processorMap.entrySet()) {
+ closeStorageGroupPrivate(processorEntry.getKey());
}
} finally {
- LOGGER.info("Close all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ LOGGER.info("Close all StorageGroupProcessors ends");
+ storageGroupManagerStatus = StorageGroupManagerStatus.NONE;
}
}
@@ -860,8 +873,6 @@ public class StorageGroupManager implements IStatistic, IService {
* force flush to control memory usage.
*/
public void forceFlush(BasicMemController.UsageLevel level) {
- // you may add some delicate process like below
- // or you could provide multiple methods for different urgency
switch (level) {
// only select the most urgent (most active or biggest in size)
// processors to flush
@@ -870,7 +881,7 @@ public class StorageGroupManager implements IStatistic, IService {
try {
flushTop(0.1f);
} catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
+ LOGGER.error("force flush memory data error in waring level: ", e);
}
break;
// force all processors to flush
@@ -878,34 +889,31 @@ public class StorageGroupManager implements IStatistic, IService {
try {
flushAll();
} catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
+ LOGGER.error("force flush memory data error in dangerous level: ", e);
}
break;
- // if the flush thread pool is not full ( or half full), start a new
- // flush task
case SAFE:
- if (FlushManager.getInstance().getActiveCnt() < 0.5 * FlushManager.getInstance()
- .getThreadCnt()) {
- try {
- flushTop(0.01f);
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: ", e);
- }
- }
break;
default:
+ throw new UnsupportedOperationException("Unreachable");
}
}
private void flushAll() throws IOException {
- for (FileNodeProcessor processor : processorMap.values()) {
+ for (StorageGroupProcessor processor : processorMap.values()) {
if (!processor.tryLock(true)) {
continue;
}
try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
+ try {
+ processor.flush().get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.error("Un expected interruption when flushing storage group {}",
+ processor.getProcessorName(), e);
+ } catch (ExecutionException e) {
+ LOGGER.error("Error occurred when flushing storage group {}",
+ processor.getProcessorName(), e);
}
} finally {
processor.unlock(true);
@@ -914,7 +922,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
private void flushTop(float percentage) throws IOException {
- List<FileNodeProcessor> tempProcessors = new ArrayList<>(processorMap.values());
+ List<StorageGroupProcessor> tempProcessors = new ArrayList<>(processorMap.values());
// sort the tempProcessors as descending order
tempProcessors.sort((o1, o2) -> (int) (o2.memoryUsage() - o1.memoryUsage()));
int flushNum =
@@ -922,16 +930,21 @@ public class StorageGroupManager implements IStatistic, IService {
? (int) (tempProcessors.size() * percentage)
: 1;
for (int i = 0; i < flushNum && i < tempProcessors.size(); i++) {
- FileNodeProcessor processor = tempProcessors.get(i);
- // 64M
+ StorageGroupProcessor processor = tempProcessors.get(i);
if (processor.memoryUsage() <= TSFileConfig.groupSizeInByte / 2) {
continue;
}
processor.writeLock();
try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
+ try {
+ processor.flush().get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.error("Unexpected interruption when flushing storage group {}",
+ processor.getProcessorName(), e);
+ } catch (ExecutionException e) {
+ LOGGER.error("Error occurred when flushing storage group {}",
+ processor.getProcessorName(), e);
}
} finally {
processor.writeUnlock();
@@ -958,46 +971,17 @@ public class StorageGroupManager implements IStatistic, IService {
return ServiceType.FILE_NODE_SERVICE;
}
- /**
- * get restore file path.
- */
- public String getRestoreFilePath(String processorName) {
- FileNodeProcessor fileNodeProcessor = processorMap.get(processorName);
- if (fileNodeProcessor != null) {
- return fileNodeProcessor.getFileNodeRestoreFilePath();
- } else {
- return null;
- }
- }
-
- /**
- * recover filenode.
- */
- public void recoverFileNode(String filenodeName)
- throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(filenodeName, true);
- LOGGER.info("Recover the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- try {
- fileNodeProcessor.fileNodeRecovery();
- } catch (FileNodeProcessorException e) {
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
- private enum FileNodeManagerStatus {
+ private enum StorageGroupManagerStatus {
NONE, MERGE, CLOSE
}
- private static class FileNodeManagerHolder {
+ private static class StorageGroupManagerHolder {
- private FileNodeManagerHolder() {
+ private StorageGroupManagerHolder() {
}
- private static final StorageGroupManager INSTANCE = new StorageGroupManager(
- TsFileDBConf.getFileNodeDir());
+ private static final StorageGroupManager INSTANCE = new StorageGroupManager();
}
}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
index c658b8e..238596c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
@@ -49,7 +49,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.filenode.FileNodeProcessorStatus;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.merge.MergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
@@ -68,6 +67,7 @@ import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
@@ -286,8 +286,8 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
@Override
public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants statConstant :
+ StorageGroupProcessorStatConstants.values()) {
hashMap
.put(statStorageGroupName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
MonitorConstants.DATA_TYPE_INT64);
@@ -298,8 +298,8 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
@Override
public List<String> getAllPathForStatistic() {
List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants statConstant :
+ StorageGroupProcessorStatConstants.values()) {
list.add(
statStorageGroupName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
}
@@ -490,7 +490,7 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
* @param appendFile the appended tsfile information
*/
public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
- throws FileNodeProcessorException {
+ throws TsFileProcessorException {
return tsFileProcessor.getOverlapFiles(appendFile, uuid);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
index d822edb..c729285 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
@@ -923,14 +923,14 @@ public class TsFileProcessor extends Processor {
* @param appendFile the appended tsfile information
*/
public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
- throws FileNodeProcessorException {
+ throws TsFileProcessorException {
List<String> overlapFiles = new ArrayList<>();
try {
for (TsFileResource tsFileResource : tsFileResources) {
getOverlapFile(appendFile, tsFileResource, uuid, overlapFiles);
}
} catch (IOException e) {
- throw new FileNodeProcessorException(String.format("Failed to get tsfiles "
+ throw new TsFileProcessorException(String.format("Failed to get tsfiles "
+ "which overlap with the appendFile: %s.", appendFile.getFilePath()), e);
}
return overlapFiles;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
index a9451a3..ec97a0d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
@@ -60,7 +60,7 @@ public class MonitorConstants {
HashMap<String, AtomicLong> hashMap = new HashMap<>();
switch (constantsType) {
case FILENODE_PROCESSOR_CONST:
- for (FileNodeProcessorStatConstants statConstant : FileNodeProcessorStatConstants
+ for (StorageGroupProcessorStatConstants statConstant : StorageGroupProcessorStatConstants
.values()) {
hashMap.put(statConstant.name(), new AtomicLong(0));
}
@@ -84,7 +84,7 @@ public class MonitorConstants {
TOTAL_POINTS, TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS, TOTAL_POINTS_FAIL
}
- public enum FileNodeProcessorStatConstants {
+ public enum StorageGroupProcessorStatConstants {
TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS, TOTAL_POINTS_FAIL
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index 4283e24..b04e506 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -37,7 +37,7 @@ import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
-import org.apache.iotdb.db.monitor.MonitorConstants.FileNodeProcessorStatConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
import org.apache.iotdb.db.monitor.collector.FileSize;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
@@ -97,7 +97,7 @@ public class StatMonitor implements IService {
for (StorageGroupManagerStatConstants constants : StorageGroupManagerStatConstants.values()) {
temporaryStatList.add(constants.name());
}
- for (FileNodeProcessorStatConstants constants : FileNodeProcessorStatConstants.values()) {
+ for (StorageGroupProcessorStatConstants constants : StorageGroupProcessorStatConstants.values()) {
temporaryStatList.add(constants.name());
}
}
[incubator-iotdb] 05/05: rename querycontext to datasource remove
unused classes
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 57cb4a8f684bdfb6e469b31b262cdbe28bad44e0
Author: 江天 <jt...@163.com>
AuthorDate: Wed May 29 08:56:25 2019 +0800
rename querycontext to datasource
remove unused classes
---
.../org/apache/iotdb/db/engine/DatabaseEngine.java | 3 +-
.../engine/cache/RowGroupBlockMetaDataCache.java | 167 -----
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 98 ---
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 84 ---
.../QueryDataSource.java | 2 +-
.../ReadOnlyMemChunk.java | 8 +-
.../SeriesDataSource.java | 14 +-
.../UnsealedTsFile.java | 2 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 3 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 3 +-
.../db/engine/memtable/MemSeriesLazyMerger.java | 2 +-
.../iotdb/db/engine/overflow/io/OverflowIO.java | 173 -----
.../db/engine/overflow/io/OverflowMemtable.java | 116 ----
.../db/engine/overflow/io/OverflowProcessor.java | 735 ---------------------
.../db/engine/overflow/io/OverflowResource.java | 329 ---------
.../overflow/io/OverflowedTsFileIOWriter.java | 36 -
.../engine/overflow/metadata/OFFileMetadata.java | 107 ---
.../overflow/metadata/OFRowGroupListMetadata.java | 109 ---
.../overflow/metadata/OFSeriesListMetadata.java | 107 ---
.../db/engine/overflow/utils/MergeStatus.java | 29 -
.../db/engine/overflow/utils/OverflowOpType.java | 29 -
.../apache/iotdb/db/engine/pool/FlushManager.java | 56 +-
.../apache/iotdb/db/engine/pool/MergeManager.java | 52 +-
.../engine/querycontext/MergeSeriesDataSource.java | 33 -
.../db/engine/querycontext/OverflowInsertFile.java | 51 --
.../querycontext/OverflowSeriesDataSource.java | 78 ---
.../querycontext/OverflowUpdateDeleteFile.java | 42 --
.../db/engine/sgmanager/StorageGroupManager.java | 2 +-
.../db/engine/sgmanager/StorageGroupProcessor.java | 4 +-
.../db/engine/tsfiledata/TsFileProcessor.java | 6 +-
.../iotdb/db/query/control/JobFileManager.java | 4 +-
.../db/query/control/QueryResourceManager.java | 8 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../EngineExecutorWithoutTimeGenerator.java | 3 +-
.../db/query/executor/FillEngineExecutor.java | 2 +-
.../GroupByWithOnlyTimeFilterDataSetDataSet.java | 2 +-
.../db/query/factory/SeriesReaderFactory.java | 7 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../org/apache/iotdb/db/query/fill/LinearFill.java | 2 +-
.../apache/iotdb/db/query/fill/PreviousFill.java | 2 +-
.../iotdb/db/query/reader/mem/MemChunkReader.java | 2 +-
.../query/reader/sequence/SequenceDataReader.java | 2 +-
.../sequence/SequenceDataReaderByTimestamp.java | 2 +-
.../reader/sequence/UnSealedTsFileReader.java | 2 +-
.../sequence/UnSealedTsFilesReaderByTimestamp.java | 2 +-
.../query/timegenerator/EngineNodeConstructor.java | 2 +-
.../bufferwrite/BufferWriteProcessorNewTest.java | 2 +-
.../bufferwrite/BufferWriteProcessorTest.java | 2 +-
.../engine/modification/DeletionFileNodeTest.java | 2 +-
.../engine/overflow/io/OverflowProcessorTest.java | 4 +-
50 files changed, 46 insertions(+), 2490 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
index 3e4a979..abe90ec 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -32,7 +32,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
/**
* DatabaseEngine is an abstraction of IoTDB storage-level interfaces.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
deleted file mode 100644
index 044fb5d..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/RowGroupBlockMetaDataCache.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.IOException;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is used to cache <code>RowGroupBlockMetaDataCache</code> of tsfile in IoTDB.
- */
-public class RowGroupBlockMetaDataCache {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RowGroupBlockMetaDataCache.class);
-
- private static final int CACHE_SIZE = 100;
- /**
- * key: the file path + deviceId.
- */
- private LinkedHashMap<String, TsDeviceMetadata> lruCache;
-
- private AtomicLong cacheHintNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
-
- private RowGroupBlockMetaDataCache(int cacheSize) {
- lruCache = new LruLinkedHashMap(cacheSize, true);
- }
-
- public static RowGroupBlockMetaDataCache getInstance() {
- return RowGroupBlockMetaDataCacheSingleton.INSTANCE;
- }
-
- /**
- * get {@link TsDeviceMetadata}. THREAD SAFE.
- */
- public TsDeviceMetadata get(String filePath, String deviceId, TsFileMetaData fileMetaData)
- throws IOException {
- // The key(the tsfile path and deviceId) for the lruCache
-
- String jointPath = filePath + deviceId;
- Object jointPathObject = jointPath.intern();
- synchronized (lruCache) {
- cacheRequestNum.incrementAndGet();
- if (lruCache.containsKey(jointPath)) {
- cacheHintNum.incrementAndGet();
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- "Cache hint: the number of requests for cache is {}, "
- + "the number of hints for cache is {}",
- cacheRequestNum.get(), cacheHintNum.get());
- }
- return lruCache.get(jointPath);
- }
- }
- synchronized (jointPathObject) {
- synchronized (lruCache) {
- if (lruCache.containsKey(jointPath)) {
- return lruCache.get(jointPath);
- }
- }
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Cache didn't hint: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
- TsDeviceMetadata blockMetaData = TsFileMetadataUtils
- .getTsRowGroupBlockMetaData(filePath, deviceId,
- fileMetaData);
- synchronized (lruCache) {
- lruCache.put(jointPath, blockMetaData);
- return lruCache.get(jointPath);
- }
- }
- }
-
- /**
- * clear LRUCache.
- */
- public void clear() {
- synchronized (lruCache) {
- lruCache.clear();
- }
- }
-
- /**
- * the default LRU cache size is 100. The singleton pattern.
- */
- private static class RowGroupBlockMetaDataCacheSingleton {
-
- private static final RowGroupBlockMetaDataCache INSTANCE = new
- RowGroupBlockMetaDataCache(CACHE_SIZE);
- }
-
- /**
- * This class is a map used to cache the <code>RowGroupBlockMetaData</code>. The caching strategy
- * is LRU.
- *
- */
- private class LruLinkedHashMap extends LinkedHashMap<String, TsDeviceMetadata> {
-
- private static final long serialVersionUID = 1290160928914532649L;
- private static final float LOAD_FACTOR_MAP = 0.75f;
- private int maxCapacity;
-
- public LruLinkedHashMap(int maxCapacity, boolean isLru) {
- super(maxCapacity, LOAD_FACTOR_MAP, isLru);
- this.maxCapacity = maxCapacity;
- }
-
- @Override
- protected boolean removeEldestEntry(Map.Entry<String, TsDeviceMetadata> eldest) {
- return size() > maxCapacity;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- return super.equals(o);
- }
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- RowGroupBlockMetaDataCache that = (RowGroupBlockMetaDataCache) o;
- return Objects.equals(lruCache, that.lruCache) &&
- Objects.equals(cacheHintNum, that.cacheHintNum) &&
- Objects.equals(cacheRequestNum, that.cacheRequestNum);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(lruCache, cacheHintNum, cacheRequestNum);
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
deleted file mode 100644
index 3cfe180..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class is used to cache <code>TsFileMetaData</code> of tsfile in IoTDB.
- */
-public class TsFileMetaDataCache {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TsFileMetaDataCache.class);
- /**
- * key: The file seriesPath of tsfile.
- */
- private ConcurrentHashMap<String, TsFileMetaData> cache;
- private AtomicLong cacheHintNum = new AtomicLong();
- private AtomicLong cacheRequestNum = new AtomicLong();
-
- private TsFileMetaDataCache() {
- cache = new ConcurrentHashMap<>();
- }
-
- public static TsFileMetaDataCache getInstance() {
- return TsFileMetaDataCacheHolder.INSTANCE;
- }
-
- /**
- * get the TsFileMetaData for the given path.
- *
- * @param path -given path
- */
- public TsFileMetaData get(String path) throws IOException {
-
- Object internPath = path.intern();
- synchronized (internPath) {
- cacheRequestNum.incrementAndGet();
- if (!cache.containsKey(path)) {
- // read value from tsfile
- TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(path);
- cache.put(path, fileMetaData);
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Cache didn't hint: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
- return cache.get(path);
- } else {
- cacheHintNum.incrementAndGet();
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- "Cache hint: the number of requests for cache is {}, the number of hints for cache "
- + "is {}",
- cacheRequestNum.get(), cacheHintNum.get());
- }
- return cache.get(path);
- }
- }
- }
-
- public void remove(String path) {
- cache.remove(path);
- }
-
- public void clear() {
- cache.clear();
- }
-
- /*
- * Singleton pattern
- */
- private static class TsFileMetaDataCacheHolder {
-
- private TsFileMetaDataCacheHolder() {}
-
- private static final TsFileMetaDataCache INSTANCE = new TsFileMetaDataCache();
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
deleted file mode 100644
index 4cd1602..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetadataUtils.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.cache;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-
-/**
- * This class is used to read metadata(<code>TsFileMetaData</code> and
- * <code>TsRowGroupBlockMetaData</code>).
- */
-public class TsFileMetadataUtils {
-
- private TsFileMetadataUtils(){
-
- }
-
- /**
- * get tsfile meta data.
- *
- * @param filePath -given path
- * @return -meta data
- */
- public static TsFileMetaData getTsFileMetaData(String filePath) throws IOException {
- TsFileSequenceReader reader = null;
- try {
- reader = new TsFileSequenceReader(filePath);
- return reader.readFileMetadata();
- } finally {
- if (reader != null) {
- reader.close();
- }
- }
- }
-
- /**
- * get row group block meta data.
- *
- * @param filePath -file path
- * @param deviceId -device id
- * @param fileMetaData -ts file meta data
- * @return -device meta data
- */
- public static TsDeviceMetadata getTsRowGroupBlockMetaData(String filePath, String deviceId,
- TsFileMetaData fileMetaData) throws IOException {
- if (!fileMetaData.getDeviceMap().containsKey(deviceId)) {
- return null;
- } else {
- TsFileSequenceReader reader = null;
- try {
- reader = new TsFileSequenceReader(filePath);
- long offset = fileMetaData.getDeviceMap().get(deviceId).getOffset();
- int size = fileMetaData.getDeviceMap().get(deviceId).getLen();
- ByteBuffer data = ByteBuffer.allocate(size);
- reader.readRaw(offset, size, data);
- data.flip();
- return TsDeviceMetadata.deserializeFrom(data);
- } finally {
- if (reader != null) {
- reader.close();
- }
- }
- }
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/QueryDataSource.java
similarity index 96%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/QueryDataSource.java
index 97f0d1c..3e80d67 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/QueryDataSource.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.querycontext;
+package org.apache.iotdb.db.engine.datasource;
public class QueryDataSource {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/ReadOnlyMemChunk.java
similarity index 96%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/ReadOnlyMemChunk.java
index 5f7aac8..0af6cec 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/ReadOnlyMemChunk.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.querycontext;
+package org.apache.iotdb.db.engine.datasource;
import java.util.Collections;
import java.util.Iterator;
@@ -42,7 +42,7 @@ public class ReadOnlyMemChunk implements TimeValuePairSorter {
private TimeValuePairSorter memSeries;
private List<TimeValuePair> sortedTimeValuePairList;
- Map<String, String> props;
+ private Map<String, String> props;
private int floatPrecision = TSFileConfig.floatPrecision;
/**
@@ -159,4 +159,8 @@ public class ReadOnlyMemChunk implements TimeValuePairSorter {
checkInitialized();
return sortedTimeValuePairList.isEmpty();
}
+
+ public Map<String, String> getProps() {
+ return props;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/SeriesDataSource.java
similarity index 85%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/SeriesDataSource.java
index 9c797ef..43f6972 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/SeriesDataSource.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.querycontext;
+package org.apache.iotdb.db.engine.datasource;
import java.util.List;
import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
@@ -53,10 +53,6 @@ public class SeriesDataSource {
return sealedFiles;
}
- public void setSealedFiles(List<TsFileResource> sealedFiles) {
- this.sealedFiles = sealedFiles;
- }
-
public boolean hasUnsealedFile() {
return unsealedFile != null;
}
@@ -65,10 +61,6 @@ public class SeriesDataSource {
return unsealedFile;
}
- public void setUnsealedFile(UnsealedTsFile unsealedFile) {
- this.unsealedFile = unsealedFile;
- }
-
public boolean hasRawSeriesChunk() {
return readableChunk != null;
}
@@ -77,10 +69,6 @@ public class SeriesDataSource {
return readableChunk;
}
- public void setReadableChunk(ReadOnlyMemChunk readableChunk) {
- this.readableChunk = readableChunk;
- }
-
public Path getSeriesPath() {
return seriesPath;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFile.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/UnsealedTsFile.java
similarity index 96%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFile.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/UnsealedTsFile.java
index 6d140aa..5b6a009 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFile.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/datasource/UnsealedTsFile.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.querycontext;
+package org.apache.iotdb.db.engine.datasource;
import java.util.List;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 097f7f7..8b7d663 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -21,10 +21,9 @@ package org.apache.iotdb.db.engine.memtable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Binary;
public abstract class AbstractMemTable implements IMemTable {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index 53ea2bc..39a4a9c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -19,9 +19,8 @@
package org.apache.iotdb.db.engine.memtable;
import java.util.Map;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Binary;
/**
* IMemTable is designed to store data points which are not flushed into TsFile yet. An instance of
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemSeriesLazyMerger.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemSeriesLazyMerger.java
index d360957..191b94e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemSeriesLazyMerger.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemSeriesLazyMerger.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.engine.memtable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.db.utils.TimeValuePair;
public class MemSeriesLazyMerger implements TimeValuePairSorter {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowIO.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowIO.java
deleted file mode 100644
index 90f0423..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowIO.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.io;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-
-import org.apache.iotdb.tsfile.read.reader.TsFileInput;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.apache.iotdb.tsfile.write.writer.TsFileOutput;
-
-public class OverflowIO extends TsFileIOWriter {
-
- private OverflowReadWriter overflowReadWriter;
-
- public OverflowIO(OverflowReadWriter overflowReadWriter) throws IOException {
- super(overflowReadWriter, new ArrayList<>());
- this.overflowReadWriter = overflowReadWriter;
- toTail();
- }
-
- public void clearRowGroupMetadatas() {
- super.chunkGroupMetaDataList.clear();
- }
-
- @Override
- public long getPos() throws IOException {
- return overflowReadWriter.getPosition();
- }
-
- public void toTail() throws IOException {
- overflowReadWriter.toTail();
- }
-
- public void close() throws IOException {
- overflowReadWriter.close();
- }
-
- public void flush() throws IOException {
- overflowReadWriter.flush();
- }
-
- public TsFileInput getReader() {
- return overflowReadWriter;
- }
-
- public TsFileOutput getWriter() {
- return overflowReadWriter;
- }
-
- public OutputStream getOutputStream() {
- return overflowReadWriter;
- }
-
- public static class OverflowReadWriter extends OutputStream implements TsFileOutput, TsFileInput {
-
- private static final String RW_MODE = "rw";
- private RandomAccessFile raf;
-
- public OverflowReadWriter(String filepath) throws FileNotFoundException {
- this.raf = new RandomAccessFile(filepath, RW_MODE);
- }
-
- public void toTail() throws IOException {
- raf.seek(raf.length());
- }
-
- @Override
- public long size() throws IOException {
- return raf.length();
- }
-
- @Override
- public long position() throws IOException {
- return raf.getFilePointer();
- }
-
- @Override
- public TsFileInput position(long newPosition) throws IOException {
- raf.seek(newPosition);
- return this;
- }
-
- @Override
- public int read(ByteBuffer dst) throws IOException {
- return raf.getChannel().read(dst);
- }
-
- @Override
- public int read(ByteBuffer dst, long position) throws IOException {
- return raf.getChannel().read(dst, position);
- }
-
- @Override
- public FileChannel wrapAsFileChannel() throws IOException {
- return raf.getChannel();
- }
-
- @Override
- public InputStream wrapAsInputStream() throws IOException {
- return Channels.newInputStream(raf.getChannel());
- }
-
- @Override
- public int read() throws IOException {
- return raf.read();
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- raf.readFully(b, off, len);
- return len;
- }
-
- @Override
- public int readInt() throws IOException {
- return raf.readInt();
- }
-
- @Override
- public void write(ByteBuffer b) throws IOException {
- raf.getChannel().write(b);
- }
-
- @Override
- public long getPosition() throws IOException {
- return raf.getFilePointer();
- }
-
- @Override
- public OutputStream wrapAsStream() throws IOException {
- return Channels.newOutputStream(raf.getChannel());
- }
-
- @Override
- public void truncate(long position) throws IOException {
- raf.getChannel().truncate(position);
- }
-
- @Override
- public void write(int b) throws IOException {
- raf.write(b);
- }
-
- @Override
- public void close() throws IOException {
- raf.close();
- }
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
deleted file mode 100644
index 5fadb54..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowMemtable.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.io;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.iotdb.db.engine.memtable.IMemTable;
-import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
-import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
-import org.apache.iotdb.tsfile.read.common.BatchData;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-
-/**
- * This class is used to store and query all overflow data in memory.<br>
- */
-public class OverflowMemtable {
-
- /**
- * store update and delete data
- */
- private Map<String, Map<String, LongStatistics>> indexTrees;
-
- /**
- * store insert data
- */
- private IMemTable memTable;
-
- public OverflowMemtable() {
- indexTrees = new HashMap<>();
- memTable = new PrimitiveMemTable();
- }
-
- public void insert(TSRecord tsRecord) {
- for (DataPoint dataPoint : tsRecord.dataPointList) {
- memTable.write(tsRecord.deviceId, dataPoint.getMeasurementId(), dataPoint.getType(),
- tsRecord.time,
- dataPoint.getValue().toString());
- }
- }
-
- /**
- * @deprecated update time series data
- */
- @Deprecated
- public void update(String deviceId, String measurementId, long startTime, long endTime,
- TSDataType dataType,
- byte[] value) {
- if (!indexTrees.containsKey(deviceId)) {
- indexTrees.put(deviceId, new HashMap<>());
- }
- if (!indexTrees.get(deviceId).containsKey(measurementId)) {
- indexTrees.get(deviceId).put(measurementId, new LongStatistics());
- }
- indexTrees.get(deviceId).get(measurementId).updateStats(startTime, endTime);
- }
-
- public void delete(String deviceId, String measurementId, long timestamp, boolean isFlushing) {
- if (isFlushing) {
- memTable = memTable.copy();
- memTable.delete(deviceId, measurementId, timestamp);
- } else {
- memTable.delete(deviceId, measurementId, timestamp);
- }
- }
-
- public ReadOnlyMemChunk queryOverflowInsertInMemory(String deviceId, String measurementId,
- TSDataType dataType, Map<String, String> props) {
- return memTable.query(deviceId, measurementId, dataType, props);
- }
-
- public boolean isEmptyOfOverflowSeriesMap() {
- return indexTrees.isEmpty();
- }
-
- public Map<String, Map<String, LongStatistics>> getOverflowSeriesMap() {
- return indexTrees;
- }
-
- public boolean isEmptyOfMemTable() {
- return memTable.isEmpty();
- }
-
- public IMemTable getMemTabale() {
- return memTable;
- }
-
- public long getSize() {
- // TODO: calculate the size of this overflow support
- return 0;
- }
-
- public void clear() {
- indexTrees.clear();
- memTable.clear();
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
deleted file mode 100644
index 288382f..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ /dev/null
@@ -1,735 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.io;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.Instant;
-import java.time.ZonedDateTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.Action;
-import org.apache.iotdb.db.engine.EngingeConstants;
-import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
-import org.apache.iotdb.db.engine.memtable.MemSeriesLazyMerger;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.pool.FlushManager;
-import org.apache.iotdb.db.engine.querycontext.MergeSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.OverflowProcessorException;
-import org.apache.iotdb.db.exception.TsFileProcessorException;
-import org.apache.iotdb.db.qp.constant.DatetimeUtils;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.ImmediateFuture;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OverflowProcessor extends Processor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(OverflowProcessor.class);
- private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- private OverflowResource workResource;
- private OverflowResource mergeResource;
-
- private OverflowMemtable workSupport;
- private OverflowMemtable flushSupport;
-
- private volatile Future<Boolean> flushFuture = new ImmediateFuture<>(true);
- private volatile boolean isMerge;
- private int valueCount;
- private String parentPath;
- private long lastFlushTime = -1;
- private AtomicLong dataPathCount = new AtomicLong();
- private ReentrantLock queryFlushLock = new ReentrantLock();
-
- private Action overflowFlushAction;
- private Action filenodeFlushAction;
- private FileSchema fileSchema;
-
- private long memThreshold = TSFileConfig.groupSizeInByte;
- private AtomicLong memSize = new AtomicLong();
-
- private WriteLogNode logNode;
- private VersionController versionController;
-
- private boolean isClosed = true;
- private boolean isFlush = false;
-
- public OverflowProcessor(String processorName, Map<String, Action> parameters,
- FileSchema fileSchema, VersionController versionController)
- throws IOException {
- super(processorName);
- this.fileSchema = fileSchema;
- this.versionController = versionController;
- //for this old version, we only support one Overflow file
- String overflowDirPath = TsFileDBConf.getOverflowDataDirs()[0];
- if (overflowDirPath.length() > 0
- && overflowDirPath.charAt(overflowDirPath.length() - 1) != File.separatorChar) {
- overflowDirPath = overflowDirPath + File.separatorChar;
- }
- this.parentPath = overflowDirPath + processorName;
-
- overflowFlushAction = parameters.get(EngingeConstants.OVERFLOW_FLUSH_ACTION);
- filenodeFlushAction = parameters
- .get(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
-
- reopen();
-
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode = MultiFileLogNodeManager.getInstance().getNode(
- processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX,
- getOverflowRestoreFile());
- }
- }
-
- public void reopen() throws IOException {
- if (!isClosed) {
- return;
- }
- // recover file
- File processorDataDir = new File(parentPath);
- if (!processorDataDir.exists()) {
- processorDataDir.mkdirs();
- }
- recovery(processorDataDir);
-
- // memory
- if (workSupport == null) {
- workSupport = new OverflowMemtable();
- } else {
- workSupport.clear();
- }
- isClosed = false;
- isFlush = false;
- }
- public void checkOpen() throws OverflowProcessorException {
- if (isClosed) {
- throw new OverflowProcessorException("OverflowProcessor already closed");
- }
- }
-
-
- private void recovery(File parentFile) throws IOException {
- String[] subFilePaths = clearFile(parentFile.list());
- if (subFilePaths.length == 0) {
- workResource = new OverflowResource(parentPath,
- String.valueOf(dataPathCount.getAndIncrement()), versionController);
- } else if (subFilePaths.length == 1) {
- long count = Long.parseLong(subFilePaths[0]);
- dataPathCount.addAndGet(count + 1);
- workResource = new OverflowResource(parentPath, String.valueOf(count), versionController);
- LOGGER.info("The overflow processor {} recover from work status.", getProcessorName());
- } else {
- long count1 = Long.parseLong(subFilePaths[0]);
- long count2 = Long.parseLong(subFilePaths[1]);
- if (count1 > count2) {
- long temp = count1;
- count1 = count2;
- count2 = temp;
- }
- dataPathCount.addAndGet(count2 + 1);
- // work dir > merge dir
- workResource = new OverflowResource(parentPath, String.valueOf(count2), versionController);
- mergeResource = new OverflowResource(parentPath, String.valueOf(count1), versionController);
- LOGGER.info("The overflow processor {} recover from merge status.", getProcessorName());
- }
- }
-
- private String[] clearFile(String[] subFilePaths) {
- // just clear the files whose name are number.
- List<String> files = new ArrayList<>();
- for (String file : subFilePaths) {
- try {
- Long.valueOf(file);
- files.add(file);
- } catch (NumberFormatException e) {
- // ignore the exception, if the name of file is not a number.
-
- }
- }
- return files.toArray(new String[files.size()]);
- }
-
- /**
- * insert one time-series record
- */
- public void insert(TSRecord tsRecord) throws IOException {
- try {
- checkOpen();
- } catch (OverflowProcessorException e) {
- throw new IOException(e);
- }
- // memory control
- long memUage = MemUtils.getRecordSize(tsRecord);
- UsageLevel usageLevel = BasicMemController.getInstance().acquireUsage(this, memUage);
- switch (usageLevel) {
- case SAFE:
- // write data
- workSupport.insert(tsRecord);
- valueCount++;
- // check flush
- memUage = memSize.addAndGet(memUage);
- if (memUage > memThreshold) {
- if (LOGGER.isWarnEnabled()) {
- LOGGER.warn("The usage of memory {} in overflow processor {} reaches the threshold {}",
- MemUtils.bytesCntToStr(memUage), getProcessorName(),
- MemUtils.bytesCntToStr(memThreshold));
- }
- flush();
- }
- break;
- case WARNING:
- // write data
- workSupport.insert(tsRecord);
- valueCount++;
- // flush
- memSize.addAndGet(memUage);
- flush();
- break;
- case DANGEROUS:
- throw new IOException("The insertion is rejected because dangerous memory level hit");
- }
-
-
-
- }
-
- /**
- * @deprecated update one time-series data which time range is from startTime from endTime.
- */
- @Deprecated
- public void update(String deviceId, String measurementId, long startTime, long endTime,
- TSDataType type, byte[] value) {
- workSupport.update(deviceId, measurementId, startTime, endTime, type, value);
- valueCount++;
- }
-
- /**
- * @deprecated this function need to be re-implemented.
- */
- @Deprecated
- public void update(String deviceId, String measurementId, long startTime, long endTime,
- TSDataType type, String value) {
- workSupport.update(deviceId, measurementId, startTime, endTime, type,
- convertStringToBytes(type, value));
- valueCount++;
- }
-
- private byte[] convertStringToBytes(TSDataType type, String o) {
- switch (type) {
- case INT32:
- return BytesUtils.intToBytes(Integer.valueOf(o));
- case INT64:
- return BytesUtils.longToBytes(Long.valueOf(o));
- case BOOLEAN:
- return BytesUtils.boolToBytes(Boolean.valueOf(o));
- case FLOAT:
- return BytesUtils.floatToBytes(Float.valueOf(o));
- case DOUBLE:
- return BytesUtils.doubleToBytes(Double.valueOf(o));
- case TEXT:
- return BytesUtils.stringToBytes(o);
- default:
- LOGGER.error("Unsupport data type: {}", type);
- throw new UnsupportedOperationException("Unsupport data type:" + type);
- }
- }
-
- /**
- * Delete data of a timeseries whose time ranges from 0 to timestamp.
- *
- * @param deviceId the deviceId of the timeseries.
- * @param measurementId the measurementId of the timeseries.
- * @param timestamp the upper-bound of deletion time.
- * @param version the version number of this deletion.
- * @param updatedModFiles add successfully updated Modification files to the list, and abort them
- * when exception is raised
- */
- public void delete(String deviceId, String measurementId, long timestamp, long version,
- List<ModificationFile> updatedModFiles) throws IOException {
- try {
- checkOpen();
- } catch (OverflowProcessorException e) {
- throw new IOException(e);
- }
- workResource.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- workSupport.delete(deviceId, measurementId, timestamp, false);
- if (isFlush()) {
- mergeResource.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- flushSupport.delete(deviceId, measurementId, timestamp, true);
- }
- }
-
- /**
- * query all overflow data which contain insert data in memory, insert data in file, update/delete
- * data in memory, update/delete data in file.
- *
- * @return OverflowSeriesDataSource
- */
- public OverflowSeriesDataSource query(String deviceId, String measurementId,
- TSDataType dataType, Map<String, String> props, QueryContext context)
- throws IOException {
- try {
- checkOpen();
- } catch (OverflowProcessorException e) {
- throw new IOException(e);
- }
- queryFlushLock.lock();
- try {
- // query insert data in memory and unseqTsFiles
- // memory
- ReadOnlyMemChunk insertInMem = queryOverflowInsertInMemory(deviceId, measurementId,
- dataType, props);
- List<OverflowInsertFile> overflowInsertFileList = new ArrayList<>();
- // work file
- Pair<String, List<ChunkMetaData>> insertInDiskWork = queryWorkDataInOverflowInsert(deviceId,
- measurementId,
- dataType, context);
- if (insertInDiskWork.left != null) {
- overflowInsertFileList
- .add(0, new OverflowInsertFile(insertInDiskWork.left,
- insertInDiskWork.right));
- }
- // merge file
- Pair<String, List<ChunkMetaData>> insertInDiskMerge = queryMergeDataInOverflowInsert(deviceId,
- measurementId, dataType, context);
- if (insertInDiskMerge.left != null) {
- overflowInsertFileList
- .add(0, new OverflowInsertFile(insertInDiskMerge.left
- , insertInDiskMerge.right));
- }
- // work file
- return new OverflowSeriesDataSource(new Path(deviceId + "." + measurementId), dataType,
- overflowInsertFileList, insertInMem);
- } finally {
- queryFlushLock.unlock();
- }
- }
-
- /**
- * query insert data in memory table. while flushing, merge the work memory table with flush
- * memory table.
- *
- * @return insert data in SeriesChunkInMemTable
- */
- private ReadOnlyMemChunk queryOverflowInsertInMemory(String deviceId, String measurementId,
- TSDataType dataType, Map<String, String> props) {
-
- MemSeriesLazyMerger memSeriesLazyMerger = new MemSeriesLazyMerger();
- queryFlushLock.lock();
- try {
- if (flushSupport != null && isFlush()) {
- memSeriesLazyMerger
- .addMemSeries(
- flushSupport.queryOverflowInsertInMemory(deviceId, measurementId, dataType, props));
- }
- memSeriesLazyMerger
- .addMemSeries(workSupport.queryOverflowInsertInMemory(deviceId, measurementId,
- dataType, props));
- // memSeriesLazyMerger has handled the props,
- // so we do not need to handle it again in the following readOnlyMemChunk
- return new ReadOnlyMemChunk(dataType, memSeriesLazyMerger, Collections.emptyMap());
- } finally {
- queryFlushLock.unlock();
- }
- }
-
- /**
- * Get the insert data which is WORK in unseqTsFile.
- *
- * @param deviceId deviceId of the target time-series
- * @param measurementId measurementId of the target time-series
- * @param dataType data type of the target time-series
- * @return the seriesPath of unseqTsFile, List of TimeSeriesChunkMetaData for the special
- * time-series.
- */
- private Pair<String, List<ChunkMetaData>> queryWorkDataInOverflowInsert(String deviceId,
- String measurementId, TSDataType dataType, QueryContext context) {
- return new Pair<>(
- workResource.getInsertFilePath(),
- workResource.getInsertMetadatas(deviceId, measurementId, dataType, context));
- }
-
- /**
- * Get the all merge data in unseqTsFile and overflowFile.
- *
- * @return MergeSeriesDataSource
- */
- public MergeSeriesDataSource queryMerge(String deviceId, String measurementId,
- TSDataType dataType, QueryContext context) {
- Pair<String, List<ChunkMetaData>> mergeInsert = queryMergeDataInOverflowInsert(deviceId,
- measurementId,
- dataType, context);
- return new MergeSeriesDataSource(new OverflowInsertFile(mergeInsert.left, mergeInsert.right));
- }
-
- public OverflowSeriesDataSource queryMerge(String deviceId, String measurementId,
- TSDataType dataType, boolean isMerge, QueryContext context) {
- Pair<String, List<ChunkMetaData>> mergeInsert = queryMergeDataInOverflowInsert(deviceId,
- measurementId,
- dataType, context);
- OverflowSeriesDataSource overflowSeriesDataSource = new OverflowSeriesDataSource(
- new Path(deviceId + "." + measurementId));
- overflowSeriesDataSource.setReadableMemChunk(null);
- overflowSeriesDataSource
- .setOverflowInsertFileList(
- Arrays.asList(new OverflowInsertFile(mergeInsert.left, mergeInsert.right)));
- return overflowSeriesDataSource;
- }
-
- /**
- * Get the insert data which is MERGE in unseqTsFile
- *
- * @return the seriesPath of unseqTsFile, List of TimeSeriesChunkMetaData for the special
- * time-series.
- **/
- private Pair<String, List<ChunkMetaData>> queryMergeDataInOverflowInsert(String deviceId,
- String measurementId, TSDataType dataType, QueryContext context) {
- if (!isMerge) {
- return new Pair<>(null, null);
- }
- return new Pair<>(
- mergeResource.getInsertFilePath(),
- mergeResource.getInsertMetadatas(deviceId, measurementId, dataType, context));
- }
-
- private void switchWorkToFlush() {
- queryFlushLock.lock();
- try {
- OverflowMemtable temp = flushSupport == null ? new OverflowMemtable() : flushSupport;
- flushSupport = workSupport;
- workSupport = temp;
- isFlush = true;
- } finally {
- queryFlushLock.unlock();
- }
- }
-
- private void switchFlushToWork() {
- queryFlushLock.lock();
- try {
- flushSupport.clear();
- workResource.appendMetadatas();
- isFlush = false;
- } finally {
- queryFlushLock.unlock();
- }
- }
-
- public void switchWorkToMerge() throws IOException {
- if (mergeResource == null) {
- mergeResource = workResource;
- workResource = new OverflowResource(parentPath,
- String.valueOf(dataPathCount.getAndIncrement()), versionController);
- }
- isMerge = true;
- LOGGER.info("The overflow processor {} switch from WORK to MERGE", getProcessorName());
- }
-
- public void switchMergeToWork() throws IOException {
- if (mergeResource != null) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(mergeResource.getInsertFilePath());
- mergeResource.close();
- mergeResource.deleteResource();
- mergeResource = null;
- }
- isMerge = false;
- LOGGER.info("The overflow processor {} switch from MERGE to WORK", getProcessorName());
- }
-
- public boolean isMerge() {
- return isMerge;
- }
-
- public boolean isFlush() {
- return isFlush;
- }
-
- private boolean flushTask(String displayMessage) {
- boolean result;
- long flushStartTime = System.currentTimeMillis();
- try {
- LOGGER.info("The overflow processor {} starts flushing {}.", getProcessorName(),
- displayMessage);
- // flush data
- workResource
- .flush(fileSchema, flushSupport.getMemTabale(),
- getProcessorName());
- filenodeFlushAction.act();
- // write-ahead log
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode.notifyEndFlush(null);
- }
- result = true;
- } catch (IOException e) {
- LOGGER.error("Flush overflow processor {} rowgroup to file error in {}. Thread {} exits.",
- getProcessorName(), displayMessage, Thread.currentThread().getName(), e);
- result = false;
- } catch (Exception e) {
- LOGGER.error("FilenodeFlushAction action failed. Thread {} exits.",
- Thread.currentThread().getName(), e);
- result = false;
- } finally {
- // switch from flush to work.
- switchFlushToWork();
- }
- // log flush time
- if (LOGGER.isInfoEnabled()) {
- LOGGER
- .info("The overflow processor {} ends flushing {}.", getProcessorName(), displayMessage);
- long flushEndTime = System.currentTimeMillis();
- LOGGER.info(
- "The overflow processor {} flush {}, start time is {}, flush end time is {}," +
- " time consumption is {}ms",
- getProcessorName(), displayMessage,
- DatetimeUtils.convertMillsecondToZonedDateTime(flushStartTime),
- DatetimeUtils.convertMillsecondToZonedDateTime(flushEndTime),
- flushEndTime - flushStartTime);
- }
- return result;
- }
-
- @Override
- public synchronized Future<Boolean> flush() throws IOException {
- // statistic information for flush
- if (lastFlushTime > 0 && LOGGER.isInfoEnabled()) {
- long thisFLushTime = System.currentTimeMillis();
- ZonedDateTime lastDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lastFlushTime),
- IoTDBDescriptor.getInstance().getConfig().getZoneID());
- ZonedDateTime thisDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(thisFLushTime),
- IoTDBDescriptor.getInstance().getConfig().getZoneID());
- LOGGER.info(
- "The overflow processor {} last flush time is {}, this flush time is {},"
- + " flush time interval is {}s",
- getProcessorName(), lastDateTime, thisDateTime,
- (thisFLushTime - lastFlushTime) / 1000);
- }
- lastFlushTime = System.currentTimeMillis();
- try {
- flushFuture.get();
- } catch (InterruptedException | ExecutionException e) {
- throw new IOException(e);
- }
- if (valueCount > 0) {
- try {
- // backup newIntervalFile list and emptyIntervalFileNode
- overflowFlushAction.act();
- } catch (Exception e) {
- LOGGER.error("Flush the overflow rowGroup to file faied, when overflowFlushAction act");
- throw new IOException(e);
- }
-
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- try {
- logNode.notifyStartFlush();
- } catch (IOException e) {
- LOGGER.error("Overflow processor {} encountered an error when notifying log node, {}",
- getProcessorName(), e);
- }
- }
- BasicMemController.getInstance().releaseUsage(this, memSize.get());
- memSize.set(0);
- valueCount = 0;
- // switch from work to flush
- switchWorkToFlush();
- flushFuture = FlushManager.getInstance().submit( () ->
- flushTask("asynchronously"));
- } else {
- flushFuture = new ImmediateFuture(true);
- }
- return flushFuture;
-
- }
-
- @Override
- public void close() throws TsFileProcessorException {
- if (isClosed) {
- return;
- }
- LOGGER.info("The overflow processor {} starts close operation.", getProcessorName());
- long closeStartTime = System.currentTimeMillis();
- // flush data
- try {
- flush().get();
- } catch (InterruptedException | ExecutionException e) {
- LOGGER.error("Encounter an interrupt error when waitting for the flushing, "
- + "the bufferwrite processor is {}.",
- getProcessorName(), e);
- Thread.currentThread().interrupt();
- } catch (IOException e) {
- throw new TsFileProcessorException(e);
- }
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("The overflow processor {} ends close operation.", getProcessorName());
- // log close time
- long closeEndTime = System.currentTimeMillis();
- LOGGER.info(
- "The close operation of overflow processor {} starts at {} and ends at {}."
- + " It comsumes {}ms.",
- getProcessorName(), DatetimeUtils.convertMillsecondToZonedDateTime(closeStartTime),
- DatetimeUtils.convertMillsecondToZonedDateTime(closeEndTime),
- closeEndTime - closeStartTime);
- }
- try {
- clear();
- } catch (IOException e) {
- throw new TsFileProcessorException(e);
- }
- isClosed = true;
- }
-
- public void clear() throws IOException {
- if (workResource != null) {
- workResource.close();
- workResource = null;
- }
- if (mergeResource != null) {
- mergeResource.close();
- mergeResource = null;
- }
- }
-
- @Override
- public boolean canBeClosed() {
- // TODO: consider merge
- return !isMerge;
- }
-
- @Override
- public long memoryUsage() {
- return memSize.get();
- }
-
- public String getOverflowRestoreFile() {
- return workResource.getPositionFilePath();
- }
-
- /**
- * @return The sum of all timeseries's metadata size within this file.
- */
- public long getMetaSize() {
- // TODO : [MemControl] implement this
- return 0;
- }
-
- /**
- * @return The size of overflow file corresponding to this processor.
- */
- public long getFileSize() {
- return workResource.getInsertFile().length() + memoryUsage();
- }
-
- /**
- * Check whether current overflow file contains too many metadata or size of current overflow file
- * is too large If true, close current file and open a new one.
- */
- private boolean checkSize() {
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- long metaSize = getMetaSize();
- long fileSize = getFileSize();
- LOGGER.info(
- "The overflow processor {}, the size of metadata reaches {},"
- + " the size of file reaches {}.",
- getProcessorName(), MemUtils.bytesCntToStr(metaSize), MemUtils.bytesCntToStr(fileSize));
- if (metaSize >= config.getOverflowMetaSizeThreshold()
- || fileSize >= config.getOverflowFileSizeThreshold()) {
- LOGGER.info(
- "The overflow processor {}, size({}) of the file {} reaches threshold {},"
- + " size({}) of metadata reaches threshold {}.",
- getProcessorName(), MemUtils.bytesCntToStr(fileSize), workResource.getInsertFilePath(),
- MemUtils.bytesCntToStr(config.getOverflowMetaSizeThreshold()),
- MemUtils.bytesCntToStr(metaSize),
- MemUtils.bytesCntToStr(config.getOverflowMetaSizeThreshold()));
- return true;
- } else {
- return false;
- }
- }
-
- public WriteLogNode getLogNode() {
- return logNode;
- }
-
- public OverflowResource getWorkResource() {
- return workResource;
- }
-
- @Override
- public boolean equals(Object o) {
- return this == o;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode());
- }
-
- /**
- * used for test. We can block to wait for finishing flushing.
- * @return the future of the flush() task.
- */
- public Future<Boolean> getFlushFuture() {
- return flushFuture;
- }
-
- /**
- * used for test. We can know when the flush() is called.
- * @return the last flush() time.
- */
- public long getLastFlushTime() {
- return lastFlushTime;
- }
-
- @Override
- public String toString() {
- return "OverflowProcessor in " + parentPath;
- }
-
- public boolean isClosed() {
- return isClosed;
- }
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
deleted file mode 100644
index 17078f8..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowResource.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.io;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.engine.memtable.IMemTable;
-import org.apache.iotdb.db.engine.memtable.MemTableFlushUtil;
-import org.apache.iotdb.db.engine.modification.Deletion;
-import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.utils.QueryUtils;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.BytesUtils;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OverflowResource {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(OverflowResource.class);
-
- private static final String INSERT_FILE_NAME = "unseqTsFile";
- private static final String POSITION_FILE_NAME = "positionFile";
-
- private static final int FOOTER_LENGTH = 4;
- private static final int POS_LENGTH = 8;
- private String parentPath;
- private String dataPath;
- private String insertFilePath;
- private String positionFilePath;
- private File insertFile;
- private OverflowIO insertIO;
- private Map<String, Map<String, List<ChunkMetaData>>> insertMetadatas;
- private List<ChunkGroupMetaData> appendInsertMetadatas;
- private VersionController versionController;
- private ModificationFile modificationFile;
-
- public OverflowResource(String parentPath, String dataPath, VersionController versionController)
- throws IOException {
- this.insertMetadatas = new HashMap<>();
- this.appendInsertMetadatas = new ArrayList<>();
- this.parentPath = parentPath;
- this.dataPath = dataPath;
- File dataFile = new File(parentPath, dataPath);
- if (!dataFile.exists()) {
- dataFile.mkdirs();
- }
- insertFile = new File(dataFile, INSERT_FILE_NAME);
- insertFilePath = insertFile.getPath();
- positionFilePath = new File(dataFile, POSITION_FILE_NAME).getPath();
-
- Pair<Long, Long> position = readPositionInfo();
- try {
- // insert stream
- OverflowIO.OverflowReadWriter readWriter = new OverflowIO.OverflowReadWriter(insertFilePath);
- // truncate
- readWriter.wrapAsFileChannel().truncate(position.left);
- // reposition
- // seek to zero
- readWriter.wrapAsFileChannel().position(0);
- // seek to tail
- // the tail is at least the len of magic string
- insertIO = new OverflowIO(readWriter);
- readMetadata();
- } catch (FileNotFoundException e){
- LOGGER.debug("Failed to construct the OverflowIO.", e);
- } catch (IOException e) {
- throw e;
- }
- this.versionController = versionController;
- modificationFile = new ModificationFile(insertFilePath + ModificationFile.FILE_SUFFIX);
- }
-
- private Pair<Long, Long> readPositionInfo() throws IOException {
- File positionFile = new File(positionFilePath);
- if (positionFile.exists()) {
- try(FileInputStream inputStream = new FileInputStream(positionFile)) {
- byte[] insertPositionData = new byte[8];
- byte[] updatePositionData = new byte[8];
- int byteRead = inputStream.read(insertPositionData);
- if (byteRead != 8) {
- throw new IOException("Not enough bytes for insertPositionData");
- }
- byteRead = inputStream.read(updatePositionData);
- if (byteRead != 8) {
- throw new IOException("Not enough bytes for updatePositionData");
- }
- long lastInsertPosition = BytesUtils.bytesToLong(insertPositionData);
- long lastUpdatePosition = BytesUtils.bytesToLong(updatePositionData);
- return new Pair<>(lastInsertPosition, lastUpdatePosition);
- }
- } else {
- LOGGER.debug("No position info, returning a default value");
- long left = 0;
- long right = 0;
- File insertTempFile = new File(insertFilePath);
- if (insertTempFile.exists()) {
- left = insertTempFile.length();
- }
- return new Pair<>(left, right);
- }
-
- }
-
- private void writePositionInfo(long lastInsertPosition, long lastUpdatePosition)
- throws IOException {
- try(FileOutputStream outputStream = new FileOutputStream(positionFilePath)) {
- byte[] data = new byte[16];
- BytesUtils.longToBytes(lastInsertPosition, data, 0);
- BytesUtils.longToBytes(lastUpdatePosition, data, 8);
- outputStream.write(data);
- }
- }
-
- private void readMetadata() throws IOException {
- // read insert meta-data
- insertIO.toTail();
- long position = insertIO.getPos();
- while (position != TsFileIOWriter.magicStringBytes.length) {
- insertIO.getReader().position(position - FOOTER_LENGTH);
- int metadataLength = insertIO.getReader().readInt();
- byte[] buf = new byte[metadataLength];
- insertIO.getReader().position(position - FOOTER_LENGTH - metadataLength);
- insertIO.getReader().read(buf, 0, buf.length);
- ByteArrayInputStream inputStream = new ByteArrayInputStream(buf);
- TsDeviceMetadata tsDeviceMetadata = TsDeviceMetadata.deserializeFrom(inputStream);
- byte[] bytesPosition = new byte[8];
- insertIO.getReader().position(position - FOOTER_LENGTH - metadataLength - POS_LENGTH);
- insertIO.getReader().read(bytesPosition, 0, POS_LENGTH);
- position = BytesUtils.bytesToLong(bytesPosition);
- for (ChunkGroupMetaData rowGroupMetaData : tsDeviceMetadata.getChunkGroupMetaDataList()) {
- String deviceId = rowGroupMetaData.getDeviceID();
- if (!insertMetadatas.containsKey(deviceId)) {
- insertMetadatas.put(deviceId, new HashMap<>());
- }
- for (ChunkMetaData chunkMetaData : rowGroupMetaData.getChunkMetaDataList()) {
- chunkMetaData.setVersion(rowGroupMetaData.getVersion());
- String measurementId = chunkMetaData.getMeasurementUid();
- if (!insertMetadatas.get(deviceId).containsKey(measurementId)) {
- insertMetadatas.get(deviceId).put(measurementId, new ArrayList<>());
- }
- insertMetadatas.get(deviceId).get(measurementId).add(0, chunkMetaData);
- }
- }
- }
- }
-
- public List<ChunkMetaData> getInsertMetadatas(String deviceId, String measurementId,
- TSDataType dataType, QueryContext context) {
- List<ChunkMetaData> chunkMetaDatas = new ArrayList<>();
- if (insertMetadatas.containsKey(deviceId) && insertMetadatas.get(deviceId)
- .containsKey(measurementId)) {
- for (ChunkMetaData chunkMetaData : insertMetadatas.get(deviceId).get(measurementId)) {
- // filter
- if (chunkMetaData.getTsDataType().equals(dataType)) {
- chunkMetaDatas.add(chunkMetaData);
- }
- }
- }
- try {
- List<Modification> modifications = context.getPathModifications(modificationFile,
- deviceId + IoTDBConstant.PATH_SEPARATOR + measurementId);
- QueryUtils.modifyChunkMetaData(chunkMetaDatas, modifications);
- } catch (IOException e) {
- LOGGER.error("Cannot access the modification file of Overflow {}, because:", parentPath,
- e);
- }
- return chunkMetaDatas;
- }
-
- public void flush(FileSchema fileSchema, IMemTable memTable, String processorName)
- throws IOException {
- // insert data
- long startPos = insertIO.getPos();
- long startTime = System.currentTimeMillis();
- flush(fileSchema, memTable);
- long timeInterval = System.currentTimeMillis() - startTime;
- timeInterval = timeInterval == 0 ? 1 : timeInterval;
- long insertSize = insertIO.getPos() - startPos;
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "Overflow processor {} flushes overflow insert data, actual:{}, time consumption:{} ms,"
- + " flush rate:{}/s",
- processorName, MemUtils.bytesCntToStr(insertSize), timeInterval,
- MemUtils.bytesCntToStr(insertSize / timeInterval * 1000));
- }
-
- writePositionInfo(insertIO.getPos(), 0);
- }
-
- public void flush(FileSchema fileSchema, IMemTable memTable) throws IOException {
- if (memTable != null && !memTable.isEmpty()) {
- insertIO.toTail();
- long lastPosition = insertIO.getPos();
- MemTableFlushUtil.flushMemTable(fileSchema, insertIO, memTable,
- versionController.nextVersion());
- List<ChunkGroupMetaData> rowGroupMetaDatas = insertIO.getChunkGroupMetaDatas();
- appendInsertMetadatas.addAll(rowGroupMetaDatas);
- if (!rowGroupMetaDatas.isEmpty()) {
- insertIO.getWriter().write(BytesUtils.longToBytes(lastPosition));
- TsDeviceMetadata tsDeviceMetadata = new TsDeviceMetadata();
- tsDeviceMetadata.setChunkGroupMetadataList(rowGroupMetaDatas);
- long start = insertIO.getPos();
- tsDeviceMetadata.serializeTo(insertIO.getOutputStream());
- long end = insertIO.getPos();
- insertIO.getWriter().write(BytesUtils.intToBytes((int) (end - start)));
- // clear the meta-data of insert IO
- insertIO.clearRowGroupMetadatas();
- }
- }
- }
-
- public void appendMetadatas() {
- if (!appendInsertMetadatas.isEmpty()) {
- for (ChunkGroupMetaData rowGroupMetaData : appendInsertMetadatas) {
- for (ChunkMetaData seriesChunkMetaData : rowGroupMetaData.getChunkMetaDataList()) {
- addInsertMetadata(rowGroupMetaData.getDeviceID(), seriesChunkMetaData.getMeasurementUid(),
- seriesChunkMetaData);
- }
- }
- appendInsertMetadatas.clear();
- }
- }
-
- public String getInsertFilePath() {
- return insertFilePath;
- }
-
- public File getInsertFile() {
- return insertFile;
- }
-
- public String getPositionFilePath() {
- return positionFilePath;
- }
-
- public void close() throws IOException {
- insertMetadatas.clear();
- insertIO.close();
- modificationFile.close();
- }
-
- public void deleteResource() throws IOException {
- FileUtils.forceDelete(new File(parentPath, dataPath));
- }
-
- private void cleanDir(String dir) throws IOException {
- File file = new File(dir);
- if (file.exists()) {
- if (file.isDirectory()) {
- File[] files = file.listFiles();
- if (files == null) {
- return;
- }
- for (File subFile : files) {
- cleanDir(subFile.getAbsolutePath());
- }
- }
- if (!file.delete()) {
- throw new IOException(String.format("The file %s can't be deleted", dir));
- }
- }
- }
-
- private void addInsertMetadata(String deviceId, String measurementId,
- ChunkMetaData chunkMetaData) {
- if (!insertMetadatas.containsKey(deviceId)) {
- insertMetadatas.put(deviceId, new HashMap<>());
- }
- if (!insertMetadatas.get(deviceId).containsKey(measurementId)) {
- insertMetadatas.get(deviceId).put(measurementId, new ArrayList<>());
- }
- insertMetadatas.get(deviceId).get(measurementId).add(chunkMetaData);
- }
-
- /**
- * Delete data of a timeseries whose time ranges from 0 to timestamp.
- *
- * @param deviceId the deviceId of the timeseries.
- * @param measurementId the measurementId of the timeseries.
- * @param timestamp the upper-bound of deletion time.
- * @param updatedModFiles add successfully updated modificationFile to this list, so that the
- * deletion can be aborted when exception is thrown.
- */
- public void delete(String deviceId, String measurementId, long timestamp, long version,
- List<ModificationFile> updatedModFiles)
- throws IOException {
- modificationFile.write(new Deletion(deviceId + IoTDBConstant.PATH_SEPARATOR
- + measurementId, version, timestamp));
- updatedModFiles.add(modificationFile);
- }
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowedTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowedTsFileIOWriter.java
deleted file mode 100644
index db86a78..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowedTsFileIOWriter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.overflow.io;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import org.apache.iotdb.tsfile.write.writer.DefaultTsFileOutput;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-
-public class OverflowedTsFileIOWriter extends TsFileIOWriter {
-
- public OverflowedTsFileIOWriter(File file) throws FileNotFoundException {
- super();
- this.out = new DefaultTsFileOutput(file, true);
-
- }
-
-
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFFileMetadata.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFFileMetadata.java
deleted file mode 100644
index 5297aa6..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFFileMetadata.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.metadata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-public class OFFileMetadata {
-
- private long lastFooterOffset;
- private List<OFRowGroupListMetadata> rowGroupLists;
-
- public OFFileMetadata() {
- //allowed to do nothing
- }
-
- public OFFileMetadata(long lastFooterOffset, List<OFRowGroupListMetadata> rowGroupLists) {
- this.lastFooterOffset = lastFooterOffset;
- this.rowGroupLists = rowGroupLists;
- }
-
- /**
- * function for deserializing data from input stream.
- */
- public static OFFileMetadata deserializeFrom(InputStream inputStream) throws IOException {
- long lastFooterOffset = ReadWriteIOUtils.readLong(inputStream);
- int size = ReadWriteIOUtils.readInt(inputStream);
- List<OFRowGroupListMetadata> list = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- list.add(OFRowGroupListMetadata.deserializeFrom(inputStream));
- }
- return new OFFileMetadata(lastFooterOffset, list);
- }
-
- public static OFFileMetadata deserializeFrom(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-
- /**
- * add OFRowGroupListMetadata to list.
- */
- public void addRowGroupListMetaData(OFRowGroupListMetadata rowGroupListMetadata) {
- if (rowGroupLists == null) {
- rowGroupLists = new ArrayList<>();
- }
- rowGroupLists.add(rowGroupListMetadata);
- }
-
- public List<OFRowGroupListMetadata> getRowGroupLists() {
- return rowGroupLists == null ? null : Collections.unmodifiableList(rowGroupLists);
- }
-
- public long getLastFooterOffset() {
- return lastFooterOffset;
- }
-
- public void setLastFooterOffset(long lastFooterOffset) {
- this.lastFooterOffset = lastFooterOffset;
- }
-
- @Override
- public String toString() {
- return String.format("OFFileMetadata{ last offset: %d, RowGroupLists: %s }", lastFooterOffset,
- rowGroupLists.toString());
- }
-
- /**
- * function for serializing data to output stream.
- */
- public int serializeTo(OutputStream outputStream) throws IOException {
- int byteLen = 0;
- byteLen += ReadWriteIOUtils.write(lastFooterOffset, outputStream);
- int size = rowGroupLists.size();
- byteLen += ReadWriteIOUtils.write(size, outputStream);
- for (OFRowGroupListMetadata ofRowGroupListMetadata : rowGroupLists) {
- byteLen += ofRowGroupListMetadata.serializeTo(outputStream);
- }
- return byteLen;
- }
-
- public int serializeTo(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFRowGroupListMetadata.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFRowGroupListMetadata.java
deleted file mode 100644
index 274fb03..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFRowGroupListMetadata.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.metadata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-/**
- * Metadata of overflow RowGroup list.
- */
-public class OFRowGroupListMetadata {
-
- private String deviceId;
- private List<OFSeriesListMetadata> seriesList;
-
- private OFRowGroupListMetadata() {
- }
-
- public OFRowGroupListMetadata(String deviceId) {
- this.deviceId = deviceId;
- seriesList = new ArrayList<>();
- }
-
- /**
- * function for deserializing data from input stream.
- */
- public static OFRowGroupListMetadata deserializeFrom(InputStream inputStream) throws IOException {
- OFRowGroupListMetadata ofRowGroupListMetadata = new OFRowGroupListMetadata();
- ofRowGroupListMetadata.deviceId = ReadWriteIOUtils.readString(inputStream);
- int size = ReadWriteIOUtils.readInt(inputStream);
- List<OFSeriesListMetadata> list = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- list.add(OFSeriesListMetadata.deserializeFrom(inputStream));
- }
- ofRowGroupListMetadata.seriesList = list;
- return ofRowGroupListMetadata;
- }
-
- public static OFRowGroupListMetadata deserializeFrom(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-
- /**
- * add OFSeriesListMetadata metadata to list.
- */
- public void addSeriesListMetaData(OFSeriesListMetadata timeSeries) {
- if (seriesList == null) {
- seriesList = new ArrayList<>();
- }
- seriesList.add(timeSeries);
- }
-
- public List<OFSeriesListMetadata> getSeriesList() {
- return seriesList == null ? null : Collections.unmodifiableList(seriesList);
- }
-
- @Override
- public String toString() {
- return String.format("OFRowGroupListMetadata{ deviceId id: %s, series Lists: %s }", deviceId,
- seriesList.toString());
- }
-
- public String getdeviceId() {
- return deviceId;
- }
-
- /**
- * function for serialing data to output stream.
- */
- public int serializeTo(OutputStream outputStream) throws IOException {
- int byteLen = 0;
- byteLen += ReadWriteIOUtils.write(deviceId, outputStream);
- int size = seriesList.size();
- byteLen += ReadWriteIOUtils.write(size, outputStream);
- for (OFSeriesListMetadata ofSeriesListMetadata : seriesList) {
- byteLen += ofSeriesListMetadata.serializeTo(outputStream);
- }
- return byteLen;
- }
-
- /**
- * function for serializing data to byte buffer.
- */
- public int serializeTo(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFSeriesListMetadata.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFSeriesListMetadata.java
deleted file mode 100644
index 40f7302..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/metadata/OFSeriesListMetadata.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.metadata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-/**
- * Metadata of overflow series list.
- */
-public class OFSeriesListMetadata {
-
- private String measurementId;
- private List<ChunkMetaData> timeSeriesList;
-
- private OFSeriesListMetadata() {
- }
-
- public OFSeriesListMetadata(String measurementId, List<ChunkMetaData> timeSeriesList) {
- this.measurementId = measurementId;
- this.timeSeriesList = timeSeriesList;
- }
-
- /**
- * function for deserializing data from input stream.
- */
- public static OFSeriesListMetadata deserializeFrom(InputStream inputStream) throws IOException {
- OFSeriesListMetadata ofSeriesListMetadata = new OFSeriesListMetadata();
- ofSeriesListMetadata.measurementId = ReadWriteIOUtils.readString(inputStream);
- int size = ReadWriteIOUtils.readInt(inputStream);
- List<ChunkMetaData> list = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- ChunkMetaData chunkMetaData = ChunkMetaData.deserializeFrom(inputStream);
- list.add(chunkMetaData);
- }
- ofSeriesListMetadata.timeSeriesList = list;
- return ofSeriesListMetadata;
- }
-
- public static OFSeriesListMetadata deserializeFrom(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-
- /**
- * add TimeSeriesChunkMetaData to timeSeriesList.
- */
- public void addSeriesMetaData(ChunkMetaData timeSeries) {
- if (timeSeriesList == null) {
- timeSeriesList = new ArrayList<>();
- }
- timeSeriesList.add(timeSeries);
- }
-
- public List<ChunkMetaData> getMetaDatas() {
- return timeSeriesList == null ? null : Collections.unmodifiableList(timeSeriesList);
- }
-
- @Override
- public String toString() {
- return String.format("OFSeriesListMetadata{ measurementId id: %s, series: %s }", measurementId,
- timeSeriesList.toString());
- }
-
- public String getMeasurementId() {
- return measurementId;
- }
-
- /**
- * function for serializing data to output stream.
- */
- public int serializeTo(OutputStream outputStream) throws IOException {
- int byteLen = 0;
- byteLen += ReadWriteIOUtils.write(measurementId, outputStream);
- byteLen += ReadWriteIOUtils.write(timeSeriesList.size(), outputStream);
- for (ChunkMetaData chunkMetaData : timeSeriesList) {
- byteLen += chunkMetaData.serializeTo(outputStream);
- }
- return byteLen;
- }
-
- public int serializeTo(ByteBuffer buffer) throws IOException {
- throw new IOException("The function has not been implemented.");
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/MergeStatus.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/MergeStatus.java
deleted file mode 100644
index 9b9aa85..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/MergeStatus.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.utils;
-
-/**
- * Used for IntervalTreeOperation.queryMemory() and IntervalTreeOperation.queryFileBlock().
- *
- * <p>DONE means that a time pair is not used or this time pair has been merged into a new
- * DynamicOneColumn MERGING means that a time pair is merging into a new DynamicOneColumn
- */
-public enum MergeStatus {
- DONE, MERGING
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/OverflowOpType.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/OverflowOpType.java
deleted file mode 100644
index ba4c4f9..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/utils/OverflowOpType.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.overflow.utils;
-
-/**
- * Include three types: INSERT,UPDATE,DELETE; INSERT is an operation which inserts a time point.<br>
- * UPDATE is an operation which updates a time range.<br> DELETE is an operation which deletes a
- * time range. Note that DELETE operation could only delete a time which is less than given time T.
- * <br>
- */
-public enum OverflowOpType {
- INSERT, UPDATE, DELETE
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/FlushManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/FlushManager.java
index 7c996aa..d6c2a8c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/FlushManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/FlushManager.java
@@ -35,11 +35,10 @@ public class FlushManager {
private static final int EXIT_WAIT_TIME = 60 * 1000;
private ExecutorService pool;
- private int threadCnt;
private FlushManager() {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- this.threadCnt = config.getConcurrentFlushThread();
+ int threadCnt = config.getConcurrentFlushThread();
pool = IoTDBThreadPoolFactory.newFixedThreadPool(threadCnt, ThreadName.FLUSH_SERVICE.getName());
}
@@ -47,47 +46,6 @@ public class FlushManager {
return InstanceHolder.instance;
}
- /**
- * @throws ProcessorException
- * if the pool is not terminated.
- */
- public void reopen() throws ProcessorException {
- if (!pool.isTerminated()) {
- throw new ProcessorException("Flush Pool is not terminated!");
- }
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- pool = Executors.newFixedThreadPool(config.getConcurrentFlushThread());
- }
-
- public FlushManager(ExecutorService pool) {
- this.pool = pool;
- }
-
- /**
- * Refuse new flush submits and exit when all RUNNING THREAD in the pool end.
- *
- * @param block
- * if set to true, this method will wait for timeOut milliseconds.
- * @param timeOut
- * block time out in milliseconds.
- * @throws ProcessorException
- * if timeOut is reached or being interrupted while waiting to exit.
- */
- public void forceClose(boolean block, long timeOut) throws ProcessorException {
- pool.shutdownNow();
- if (block) {
- try {
- if (!pool.awaitTermination(timeOut, TimeUnit.MILLISECONDS)) {
- throw new ProcessorException("Flush thread pool doesn't exit after "
- + EXIT_WAIT_TIME + " ms");
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new ProcessorException("Interrupted while waiting flush thread pool to exit. "
- , e);
- }
- }
- }
/**
* Block new flush submits and exit when all RUNNING THREADS AND TASKS IN THE QUEUE end.
@@ -114,22 +72,10 @@ public class FlushManager {
}
}
- public synchronized Future<?> submit(Runnable task) {
- return pool.submit(task);
- }
-
public synchronized <T>Future<T> submit(Callable<T> task){
return pool.submit(task);
}
- public int getActiveCnt() {
- return ((ThreadPoolExecutor) pool).getActiveCount();
- }
-
- public int getThreadCnt() {
- return threadCnt;
- }
-
private static class InstanceHolder {
private InstanceHolder(){
//allowed to do nothing
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
index 81b0dcb..ff795c6 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
@@ -33,11 +33,10 @@ import org.apache.iotdb.db.exception.ProcessorException;
public class MergeManager {
private ExecutorService pool;
- private int threadCnt;
private MergeManager() {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- this.threadCnt = config.getMergeConcurrentThreads();
+ int threadCnt = config.getMergeConcurrentThreads();
pool = IoTDBThreadPoolFactory.newFixedThreadPool(threadCnt, ThreadName.MERGE_SERVICE.getName());
}
@@ -46,43 +45,6 @@ public class MergeManager {
}
/**
- * reopen function.
- *
- * @throws ProcessorException if the pool is not terminated.
- */
- public void reopen() throws ProcessorException {
- if (!pool.isTerminated()) {
- throw new ProcessorException("Merge pool is not terminated!");
- }
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- pool = Executors.newFixedThreadPool(config.getMergeConcurrentThreads());
- }
-
- /**
- * Refuse new merge submits and exit when all RUNNING THREAD in the pool end.
- *
- * @param block if set block to true, this method will wait for timeOut milliseconds to close the
- * merge pool. false, return directly.
- * @param timeOut block time out in milliseconds.
- * @throws ProcessorException if timeOut reach or interrupted while waiting to exit.
- */
- public void forceClose(boolean block, long timeOut) throws ProcessorException {
- pool.shutdownNow();
- if (block) {
- try {
- if (!pool.awaitTermination(timeOut, TimeUnit.MILLISECONDS)) {
- throw new ProcessorException(
- "Merge thread pool doesn't exit after " + timeOut + " ms");
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new ProcessorException(
- "Interrupted while waiting merge thread pool to exit. ", e);
- }
- }
- }
-
- /**
* Block new merge submits and exit when all RUNNING THREADS AND TASKS IN THE QUEUE end.
*
* @param block if set to true, this method will wait for timeOut milliseconds. false, return
@@ -106,22 +68,10 @@ public class MergeManager {
}
}
- public Future<?> submit(Runnable task) {
- return pool.submit(task);
- }
-
public Future<?> submit(Callable task) {
return pool.submit(task);
}
- public int getActiveCnt() {
- return ((ThreadPoolExecutor) pool).getActiveCount();
- }
-
- public int getThreadCnt() {
- return threadCnt;
- }
-
private static class InstanceHolder {
private InstanceHolder(){
//allowed to do nothing
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/MergeSeriesDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/MergeSeriesDataSource.java
deleted file mode 100644
index 2e42bac..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/MergeSeriesDataSource.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.querycontext;
-
-public class MergeSeriesDataSource {
-
- private OverflowInsertFile insertFile;
-
- public MergeSeriesDataSource(OverflowInsertFile insertFile) {
- this.insertFile = insertFile;
- }
-
- public OverflowInsertFile getInsertFile() {
- return insertFile;
- }
-
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowInsertFile.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowInsertFile.java
deleted file mode 100644
index 0c351df..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowInsertFile.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.querycontext;
-
-import java.util.List;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-
-public class OverflowInsertFile {
-
- private String filePath;
-
- // seriesChunkMetadata of selected series
- private List<ChunkMetaData> timeSeriesChunkMetaData;
-
- public OverflowInsertFile() {
- //allowed to do nothing
- }
-
- public OverflowInsertFile(String path, List<ChunkMetaData> timeSeriesChunkMetaData) {
- this.filePath = path;
- this.timeSeriesChunkMetaData = timeSeriesChunkMetaData;
- }
-
- public String getFilePath() {
- return filePath;
- }
-
- public List<ChunkMetaData> getChunkMetaDataList() {
- return timeSeriesChunkMetaData;
- }
-
- public void setTimeSeriesChunkMetaData(List<ChunkMetaData> timeSeriesChunkMetaData) {
- this.timeSeriesChunkMetaData = timeSeriesChunkMetaData;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowSeriesDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowSeriesDataSource.java
deleted file mode 100644
index 78571f9..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowSeriesDataSource.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.querycontext;
-
-import java.util.List;
-import org.apache.iotdb.db.engine.memtable.TimeValuePairSorter;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.Path;
-
-public class OverflowSeriesDataSource {
-
- private Path seriesPath;
- private TSDataType dataType;
- // overflow tsfile
- private List<OverflowInsertFile> overflowInsertFileList;
- // unSeq mem-table
- private ReadOnlyMemChunk readableMemChunk;
-
- public OverflowSeriesDataSource(Path seriesPath) {
- this.seriesPath = seriesPath;
- }
-
- public OverflowSeriesDataSource(Path seriesPath, TSDataType dataType,
- List<OverflowInsertFile> overflowInsertFileList, ReadOnlyMemChunk readableMemChunk) {
- this.seriesPath = seriesPath;
- this.dataType = dataType;
- this.overflowInsertFileList = overflowInsertFileList;
- this.readableMemChunk = readableMemChunk;
- }
-
- public List<OverflowInsertFile> getOverflowInsertFileList() {
- return overflowInsertFileList;
- }
-
- public void setOverflowInsertFileList(List<OverflowInsertFile> overflowInsertFileList) {
- this.overflowInsertFileList = overflowInsertFileList;
- }
-
- public ReadOnlyMemChunk getReadableMemChunk() {
- return readableMemChunk;
- }
-
- public void setReadableMemChunk(ReadOnlyMemChunk rawChunk) {
- this.readableMemChunk = rawChunk;
- }
-
- public Path getSeriesPath() {
- return seriesPath;
- }
-
- public void setSeriesPath(Path seriesPath) {
- this.seriesPath = seriesPath;
- }
-
- public TSDataType getDataType() {
- return dataType;
- }
-
- public boolean hasRawChunk() {
- return readableMemChunk != null && !readableMemChunk.isEmpty();
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowUpdateDeleteFile.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowUpdateDeleteFile.java
deleted file mode 100644
index 875c98f..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/OverflowUpdateDeleteFile.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.querycontext;
-
-import java.util.List;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-
-public class OverflowUpdateDeleteFile {
-
- private String filePath;
- private List<ChunkMetaData> timeSeriesChunkMetaDataList;
-
- public OverflowUpdateDeleteFile(String filePath,
- List<ChunkMetaData> timeSeriesChunkMetaDataList) {
- this.filePath = filePath;
- this.timeSeriesChunkMetaDataList = timeSeriesChunkMetaDataList;
- }
-
- public String getFilePath() {
- return filePath;
- }
-
- public List<ChunkMetaData> getTimeSeriesChunkMetaDataList() {
- return timeSeriesChunkMetaDataList;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index 45d6faa..91440bd 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -38,7 +38,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.DatabaseEngine;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
index 42923ad..eeac144 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
@@ -54,8 +54,8 @@ import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.overflowdata.OverflowProcessor;
import org.apache.iotdb.db.engine.pool.MergeManager;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.SeriesDataSource;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
import org.apache.iotdb.db.engine.version.VersionController;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
index d787609..6abff9a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
@@ -53,9 +53,9 @@ import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.pool.FlushManager;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.SeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.UnsealedTsFile;
import org.apache.iotdb.db.engine.sgmanager.OperationResult;
import org.apache.iotdb.db.engine.sgmanager.StorageGroupProcessor;
import org.apache.iotdb.db.engine.version.VersionController;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 84ceeb0..46cce5b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -22,8 +22,8 @@ import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
-import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.OverflowInsertFile;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
/**
* <p>
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index 310a850..406eb11 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -18,9 +18,7 @@
*/
package org.apache.iotdb.db.query.control;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -28,10 +26,8 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.engine.DatabaseEngineFactory;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
+import org.apache.iotdb.db.engine.datasource.OverflowSeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.read.common.Path;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
index aa494bb..49cd60c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
index 2819915..cdb286c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
@@ -22,8 +22,7 @@ package org.apache.iotdb.db.query.executor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
index 7dfa31e..d2409a1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
index c2b85eb..ff7a502 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.executor.groupby;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
index c6d34dc..013c0bf 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
@@ -19,16 +19,15 @@
package org.apache.iotdb.db.query.factory;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.SeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.UnsealedTsFile;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/IFill.java b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
index d64b49a..6292a65 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/IFill.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.query.fill;
import java.io.IOException;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
import org.apache.iotdb.db.query.reader.AllDataReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/LinearFill.java b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/LinearFill.java
index dc46082..71d9e03 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/LinearFill.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/LinearFill.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.query.fill;
import java.io.IOException;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.UnSupportedFillTypeException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.IPointReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
index b75fb4f..a893cfc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/fill/PreviousFill.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.query.fill;
import java.io.IOException;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.IPointReader;
import org.apache.iotdb.db.utils.TimeValuePair;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/mem/MemChunkReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/mem/MemChunkReader.java
index 8c3e9cf..01a2d41 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/mem/MemChunkReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/mem/MemChunkReader.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.query.reader.mem;
import java.util.Iterator;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.db.query.reader.IAggregateReader;
import org.apache.iotdb.db.query.reader.IBatchReader;
import org.apache.iotdb.db.query.reader.IPointReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
index 2a9abb4..a84c86b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.SeriesDataSource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.IAggregateReader;
import org.apache.iotdb.db.query.reader.IBatchReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
index e8a3905..e8aafb1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.SeriesDataSource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.mem.MemChunkReaderByTimestamp;
import org.apache.iotdb.db.query.reader.merge.EngineReaderByTimeStamp;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReader.java
index d4e75e6..8948488 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReader.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
+import org.apache.iotdb.db.engine.datasource.UnsealedTsFile;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.IAggregateReader;
import org.apache.iotdb.db.query.reader.IBatchReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFilesReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFilesReaderByTimestamp.java
index 4aebdb8..06b36ba 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFilesReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFilesReaderByTimestamp.java
@@ -20,7 +20,7 @@
package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
+import org.apache.iotdb.db.engine.datasource.UnsealedTsFile;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.merge.EngineReaderByTimeStamp;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
index 86b434f..4e31bff 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
@@ -24,7 +24,7 @@ import static org.apache.iotdb.tsfile.read.expression.ExpressionType.OR;
import static org.apache.iotdb.tsfile.read.expression.ExpressionType.SERIES;
import java.io.IOException;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
index 90a101b..ce7ae70 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
@@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
index 2ad558e..b934c3f 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
@@ -37,7 +37,7 @@ import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.PathUtils;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.datasource.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 2f22670..d08af05 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -33,7 +33,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.datasource.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
index b366c21..7b0a58d 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
@@ -31,8 +31,8 @@ import org.apache.iotdb.db.engine.PathUtils;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
import org.apache.iotdb.db.engine.EngingeConstants;
-import org.apache.iotdb.db.engine.querycontext.MergeSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.MergeSeriesDataSource;
+import org.apache.iotdb.db.engine.datasource.OverflowSeriesDataSource;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.OverflowProcessorException;
import org.apache.iotdb.db.query.context.QueryContext;
[incubator-iotdb] 02/05: abstract StorageGroupManager as
StorageEngine
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 1ea3bfff7b892c696e93d7cbb74e695f07efc96d
Author: 江天 <jt...@163.com>
AuthorDate: Tue May 28 12:43:38 2019 +0800
abstract StorageGroupManager as StorageEngine
---
.../db/engine/memcontrol/FlushPartialPolicy.java | 3 +-
.../db/engine/overflow/io/OverflowProcessor.java | 1 -
.../iotdb/db/engine/sgmanager/StorageEngine.java | 167 +++++++++++++++++++++
.../db/engine/sgmanager/StorageEngineFactory.java | 27 ++++
.../db/engine/sgmanager/StorageGroupManager.java | 136 ++++++-----------
5 files changed, 237 insertions(+), 97 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
index 31a5860..a3604a3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
@@ -20,13 +20,12 @@ package org.apache.iotdb.db.engine.memcontrol;
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.utils.MemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class only gives a hint to FilenodeManager that it may flush some data to avoid rush hour.
+ * This class only gives a hint to StorageGroupManager that it may flush some data to avoid rush hour.
*/
public class FlushPartialPolicy implements Policy {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index f052876..466138f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
import org.apache.iotdb.db.engine.memtable.MemSeriesLazyMerger;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java
new file mode 100644
index 0000000..72c7dcf
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.sgmanager;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
+
+/**
+ * StorageEngine is an abstraction of IoTDB storage-level interfaces.
+ */
+public interface StorageEngine {
+
+ /**
+ * This function is just for unit test.
+ */
+ void reset();
+
+ /**
+ * Execute an insertion.
+ *
+ * @param plan an insert plan
+ * @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
+ * be recorded. if false, the statParamsHashMap will be updated.
+ */
+ void insert(InsertPlan plan, boolean isMonitor) throws StorageGroupManagerException;
+
+ /**
+ * update data.
+ */
+ void update(String deviceId, String measurementId, long startTime, long endTime,
+ TSDataType type, String v)
+ throws StorageGroupManagerException;
+
+ /**
+ * Delete data whose timestamp <= 'timestamp' of time series 'deviceId'.'measurementId'.
+ * @param deviceId
+ * @param measurementId
+ * @param timestamp
+ * @throws StorageGroupManagerException
+ */
+ void deleteData(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException;
+
+ /**
+ * Similar to deleteData(), but only deletes data in sequence files. Only used by WAL recovery.
+ */
+ void deleteInSeqFile(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException;
+
+ /**
+ * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
+ */
+ void deleteInOverflow(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException;
+
+ /**
+ * Get a StorageGroup-level token for this query so that the StorageGroupProcessor may know which
+ * queries are occupying resources.
+ *
+ * @param deviceId queried deviceId
+ * @return a query token for the device.
+ */
+ int beginQuery(String deviceId) throws StorageGroupManagerException;
+
+ /**
+ * Notify the storage group of 'deviceId' that query 'token' has ended and its resource can be
+ * released.
+ */
+ void endQuery(String deviceId, int token) throws StorageGroupManagerException;
+
+ /**
+ * Find sealed files, unsealed file and memtable data in SeqFiles and OverflowFiles that contains the
+ * given series.
+ * @param seriesExpression provides the path of the series.
+ * @param context provides shared modifications across a query.
+ * @return sealed files, unsealed file and memtable data in SeqFiles or OverflowFiles
+ * @throws StorageGroupManagerException
+ */
+ QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
+ throws StorageGroupManagerException;
+
+ /**
+ * Append one specified tsfile to the storage group. <b>This method is only provided for
+ * transmission module</b>
+ *
+ * @param storageGroupName the seriesPath of storage group
+ * @param appendFile the appended tsfile information
+ */
+ boolean appendFileToStorageGroup(String storageGroupName, TsFileResource appendFile,
+ String appendFilePath) throws StorageGroupManagerException;
+
+ /**
+ * get all overlap tsfiles which are conflict with the appendFile.
+ *
+ * @param storageGroupName the seriesPath of storage group
+ * @param appendFile the appended tsfile information
+ */
+ List<String> getOverlapFilesFromStorageGroup(String storageGroupName, TsFileResource appendFile,
+ String uuid) throws StorageGroupManagerException;
+
+ /**
+ * merge all overflowed storage group.
+ *
+ * @throws StorageGroupManagerException StorageGroupManagerException
+ */
+ void mergeAll() throws StorageGroupManagerException;
+
+ /**
+ * delete one storage group.
+ */
+ void deleteOneStorageGroup(String processorName) throws StorageGroupManagerException;
+
+ /**
+ * add time series.
+ */
+ void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
+ CompressionType compressor,
+ Map<String, String> props) throws StorageGroupManagerException;
+
+ /**
+ * Force to close the storage group processor.
+ */
+ void closeStorageGroup(String processorName) throws StorageGroupManagerException;
+
+ /**
+ * delete all storage groups.
+ */
+ boolean deleteAll() throws StorageGroupManagerException;
+
+ /**
+ * delete all storage groups.
+ */
+ void closeAll() throws StorageGroupManagerException;
+
+ /**
+ * force flush to control memory usage.
+ */
+ void forceFlush(BasicMemController.UsageLevel level);
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java
new file mode 100644
index 0000000..85218a7
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.sgmanager;
+
+public class StorageEngineFactory {
+
+ public static StorageEngine getCurrent() {
+ return StorageGroupManager.getInstance();
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index d313c38..22d37b8 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -36,15 +36,11 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.FileNodeProcessor;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.pool.FlushManager;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.StorageGroupManagerException;
-import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.PathErrorException;
-import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
@@ -71,7 +67,7 @@ import org.slf4j.LoggerFactory;
* StorageGroupManager provides top-level interfaces to access IoTDB storage engine. It decides
* which StorageGroup(s) to access in order to complete a query.
*/
-public class StorageGroupManager implements IStatistic, IService {
+public class StorageGroupManager implements IStatistic, IService, StorageEngine {
private static final Logger LOGGER = LoggerFactory.getLogger(StorageGroupManager.class);
private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
@@ -92,9 +88,10 @@ public class StorageGroupManager implements IStatistic, IService {
private HashMap<String, AtomicLong> statParamsHashMap;
- private StorageGroupManager() {
+ private StorageGroupManager() throws StorageGroupManagerException {
processorMap = new ConcurrentHashMap<>();
initStat();
+ recover();
}
private void initStat() {
@@ -112,14 +109,14 @@ public class StorageGroupManager implements IStatistic, IService {
}
public static StorageGroupManager getInstance() {
- return StorageGroupManagerHolder.INSTANCE;
+ return StorageGroupManagerHolder.instance;
}
- private void updateStatHashMapWhenFail(TSRecord tsRecord) {
+ private void updateStatHashMapWhenFail(InsertPlan plan) {
statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_FAIL.name())
.incrementAndGet();
statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS_FAIL.name())
- .addAndGet(tsRecord.dataPointList.size());
+ .addAndGet(plan.getValues().length);
}
/**
@@ -172,7 +169,8 @@ public class StorageGroupManager implements IStatistic, IService {
/**
* This function is just for unit test.
*/
- public synchronized void resetStorageGroupManager() {
+ @Override
+ public synchronized void reset() {
for (String key : statParamsHashMap.keySet()) {
statParamsHashMap.put(key, new AtomicLong());
}
@@ -232,7 +230,7 @@ public class StorageGroupManager implements IStatistic, IService {
/**
* recover the StorageGroupProcessors.
*/
- public void recover() throws StorageGroupManagerException {
+ private void recover() throws StorageGroupManagerException {
List<String> storageGroupNames;
try {
storageGroupNames = MManager.getInstance().getAllStorageGroups();
@@ -254,13 +252,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * insert TsRecord into storage group.
- *
- * @param plan an insert plan
- * @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
- * be recorded. if false, the statParamsHashMap will be updated.
- */
+ @Override
public void insert(InsertPlan plan, boolean isMonitor) throws StorageGroupManagerException {
long timestamp = plan.getTime();
@@ -277,6 +269,7 @@ public class StorageGroupManager implements IStatistic, IService {
result.toString()));
}
} catch (TsFileProcessorException e) {
+ updateStatHashMapWhenFail(plan);
throw new StorageGroupManagerException(String.format("Fail to write in SG %s",
processor.getProcessorName()), e);
} finally {
@@ -313,23 +306,15 @@ public class StorageGroupManager implements IStatistic, IService {
}
- /**
- * update data.
- */
+ @Override
public void update(String deviceId, String measurementId, long startTime, long endTime,
TSDataType type, String v)
throws StorageGroupManagerException {
throw new UnsupportedOperationException("Method unimplemented");
}
- /**
- * Delete data whose timestamp <= 'timestamp' of time series 'deviceId'.'measurementId'.
- * @param deviceId
- * @param measurementId
- * @param timestamp
- * @throws StorageGroupManagerException
- */
- public void deleteProcessor(String deviceId, String measurementId, long timestamp)
+ @Override
+ public void deleteData(String deviceId, String measurementId, long timestamp)
throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(deviceId, true);
@@ -392,9 +377,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Similar to delete(), but only deletes data in sequence files. Only used by WAL recovery.
- */
+ @Override
public void deleteInSeqFile(String deviceId, String measurementId, long timestamp)
throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(deviceId, true);
@@ -405,9 +388,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- */
+ @Override
public void deleteInOverflow(String deviceId, String measurementId, long timestamp)
throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(deviceId, true);
@@ -418,13 +399,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Get a StorageGroup-level token for this query so that the StorageGroupProcessor may know which
- * queries are occupying resources.
- *
- * @param deviceId queried deviceId
- * @return a query token for the device.
- */
+ @Override
public int beginQuery(String deviceId) throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
@@ -438,10 +413,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Notify the storage group of 'deviceId' that query 'token' has ended and its resource can be
- * released.
- */
+ @Override
public void endQuery(String deviceId, int token) throws StorageGroupManagerException {
StorageGroupProcessor processorrocessor = getProcessor(deviceId, true);
@@ -456,14 +428,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Find sealed files, unsealed file and memtable data in SeqFiles and OverflowFiles that contains the
- * given series.
- * @param seriesExpression provides the path of the series.
- * @param context provides shared modifications across a query.
- * @return sealed files, unsealed file and memtable data in SeqFiles or OverflowFiles
- * @throws StorageGroupManagerException
- */
+ @Override
public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
throws StorageGroupManagerException {
String deviceId = seriesExpression.getSeriesPath().getDevice();
@@ -486,13 +451,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Append one specified tsfile to the storage group. <b>This method is only provided for
- * transmission module</b>
- *
- * @param storageGroupName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
+ @Override
public boolean appendFileToStorageGroup(String storageGroupName, TsFileResource appendFile,
String appendFilePath) throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(storageGroupName, true);
@@ -516,13 +475,9 @@ public class StorageGroupManager implements IStatistic, IService {
return true;
}
- /**
- * get all overlap tsfiles which are conflict with the appendFile.
- *
- * @param storageGroupName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
- public List<String> getOverlapFilesFromStorageGroup(String storageGroupName, TsFileResource appendFile,
+ @Override
+ public List<String> getOverlapFilesFromStorageGroup(String storageGroupName,
+ TsFileResource appendFile,
String uuid) throws StorageGroupManagerException {
StorageGroupProcessor processor = getProcessor(storageGroupName, true);
List<String> overlapFiles;
@@ -536,11 +491,8 @@ public class StorageGroupManager implements IStatistic, IService {
return overlapFiles;
}
- /**
- * merge all overflowed storage group.
- *
- * @throws StorageGroupManagerException StorageGroupManagerException
- */
+
+ @Override
public void mergeAll() throws StorageGroupManagerException {
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
LOGGER.warn("Unable to merge all storage groups when the status is {}",
@@ -649,9 +601,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * delete one storage group.
- */
+ @Override
public void deleteOneStorageGroup(String processorName) throws StorageGroupManagerException {
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
return;
@@ -750,9 +700,7 @@ public class StorageGroupManager implements IStatistic, IService {
return res;
}
- /**
- * add time series.
- */
+ @Override
public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor,
Map<String, String> props) throws StorageGroupManagerException {
@@ -764,10 +712,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
-
- /**
- * Force to close the storage group processor.
- */
+ @Override
public void closeStorageGroup(String processorName) throws StorageGroupManagerException {
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
return;
@@ -824,9 +769,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * delete all storage groups.
- */
+ @Override
public synchronized boolean deleteAll() throws StorageGroupManagerException {
LOGGER.info("Start deleting all storage group");
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
@@ -849,9 +792,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * Try to close All.
- */
+ @Override
public void closeAll() throws StorageGroupManagerException {
LOGGER.info("Start closing all storage group processor");
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
@@ -869,9 +810,7 @@ public class StorageGroupManager implements IStatistic, IService {
}
}
- /**
- * force flush to control memory usage.
- */
+ @Override
public void forceFlush(BasicMemController.UsageLevel level) {
switch (level) {
// only select the most urgent (most active or biggest in size)
@@ -981,7 +920,16 @@ public class StorageGroupManager implements IStatistic, IService {
private StorageGroupManagerHolder() {
}
- private static final StorageGroupManager INSTANCE = new StorageGroupManager();
+ private static StorageGroupManager instance;
+
+ static {
+ try {
+ instance = new StorageGroupManager();
+ } catch (StorageGroupManagerException e) {
+ LOGGER.error("Failed to initialize StorageGroupManager due to a FATAL error", e);
+ instance = null;
+ }
+ }
}
}
\ No newline at end of file
[incubator-iotdb] 04/05: migrate usages of FileNodeProcessor
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 02647ac42889806f75a674659437d59fcc0eeefe
Author: 江天 <jt...@163.com>
AuthorDate: Tue May 28 13:40:47 2019 +0800
migrate usages of FileNodeProcessor
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/engine/DatabaseEngine.java | 5 +-
...ileNodeConstants.java => EngingeConstants.java} | 16 +-
.../java/org/apache/iotdb/db/engine/Processor.java | 1 -
.../apache/iotdb/db/engine/bufferwrite/Action.java | 28 -
.../db/engine/bufferwrite/ActionException.java | 30 -
.../engine/bufferwrite/BufferWriteProcessor.java | 560 ------
.../db/engine/filenode/FileNodeFlushFuture.java | 91 -
.../iotdb/db/engine/filenode/FileNodeManager.java | 1217 ------------
.../db/engine/filenode/FileNodeProcessor.java | 2065 --------------------
.../engine/filenode/FileNodeProcessorStatus.java | 51 -
.../db/engine/filenode/FileNodeProcessorStore.java | 165 --
.../db/engine/filenode/OverflowChangeType.java | 57 -
.../db/engine/memcontrol/FlushPartialPolicy.java | 3 +-
.../db/engine/memcontrol/ForceFLushAllPolicy.java | 4 +-
.../db/engine/overflow/io/OverflowProcessor.java | 6 +-
.../db/engine/querycontext/SeriesDataSource.java | 2 +-
.../db/engine/sgmanager/StorageGroupManager.java | 7 +-
.../db/engine/sgmanager/StorageGroupProcessor.java | 7 +-
.../{filenode => sgmanager}/TsFileResource.java | 8 +-
.../RestorableTsFileIOWriter.java | 2 +-
.../db/engine/tsfiledata/TsFileProcessor.java | 14 +-
.../org/apache/iotdb/db/monitor/IStatistic.java | 4 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 79 +-
.../iotdb/db/monitor/collector/FileSize.java | 16 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 61 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 21 +-
.../iotdb/db/query/control/JobFileManager.java | 2 +-
.../db/query/control/QueryResourceManager.java | 60 +-
.../EngineExecutorWithoutTimeGenerator.java | 61 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 17 -
.../db/query/factory/SeriesReaderFactory.java | 16 +-
.../query/reader/sequence/SealedTsFilesReader.java | 2 +-
.../sequence/SealedTsFilesReaderByTimestamp.java | 2 +-
.../apache/iotdb/db/service/CloseMergeService.java | 5 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 7 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 10 +-
.../java/org/apache/iotdb/db/sync/SyncUtils.java | 2 +-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 30 +-
.../org/apache/iotdb/db/utils/LoadDataUtils.java | 11 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 2 +-
.../recover/ExclusiveLogRecoverPerformer.java | 3 +-
.../writelog/recover/FileNodeRecoverPerformer.java | 4 +-
.../db/writelog/replay/ConcreteLogReplayer.java | 30 +-
.../engine/bufferwrite/BufferWriteBenchmark.java | 13 +-
.../bufferwrite/BufferWriteProcessorNewTest.java | 7 +-
.../bufferwrite/BufferWriteProcessorTest.java | 7 +-
.../bufferwrite/RestorableTsFileIOWriterTest.java | 1 +
.../filenodev2/FileNodeManagerBenchmark.java | 6 +-
.../memcontrol/BufferwriteFileSizeControlTest.java | 8 +-
.../memcontrol/BufferwriteMetaSizeControlTest.java | 8 +-
.../memcontrol/OverflowFileSizeControlTest.java | 10 +-
.../memcontrol/OverflowMetaSizeControlTest.java | 10 +-
.../engine/modification/DeletionFileNodeTest.java | 52 +-
.../db/engine/modification/DeletionQueryTest.java | 88 +-
.../overflow/io/OverflowProcessorBenchmark.java | 10 +-
.../engine/overflow/io/OverflowProcessorTest.java | 6 +-
.../integration/QueryDataFromUnclosedTsFileIT.java | 6 +-
.../org/apache/iotdb/db/monitor/MonitorTest.java | 20 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 10 +-
60 files changed, 303 insertions(+), 4745 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 6509812..cb4b799 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -183,7 +183,7 @@ public class IoTDBConfig {
private long memThresholdDangerous = (long) (0.6 * Runtime.getRuntime().maxMemory());
/**
* MemMonitorThread will check every such interval(in ms). If memThresholdWarning is reached,
- * MemMonitorThread will inform FileNodeManager to flush.
+ * MemMonitorThread will inform DataBaseEngine to flush.
*/
private long memMonitorInterval = 1000;
/**
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
index 7dc436e..3e4a979 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.engine;
import java.util.List;
import java.util.Map;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
@@ -32,6 +32,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
/**
* DatabaseEngine is an abstraction of IoTDB storage-level interfaces.
@@ -136,7 +137,7 @@ public interface DatabaseEngine {
/**
* delete one storage group.
*/
- void deleteOneStorageGroup(String processorName) throws StorageGroupManagerException;
+ void deleteStorageGroup(String processorName) throws StorageGroupManagerException;
/**
* add time series.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/FileNodeConstants.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/EngingeConstants.java
similarity index 68%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/FileNodeConstants.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/EngingeConstants.java
index 483f7f1..56cd481 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/FileNodeConstants.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/EngingeConstants.java
@@ -16,29 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.bufferwrite;
+package org.apache.iotdb.db.engine;
/**
* Constants for using in bufferwrite, overflow and filenode.
*
*/
-public class FileNodeConstants {
+public class EngingeConstants {
- private FileNodeConstants(){
+ private EngingeConstants(){
}
- public static final String FILE_NODE_OPERATOR_TYPE = "OPERATOR_TYPE";
- public static final String TIMESTAMP_KEY = "TIMESTAMP";
- public static final String FILE_NODE = "FILE_NODE";
- public static final String CLOSE_ACTION = "CLOSE_ACTION";
-
public static final String OVERFLOW_FLUSH_ACTION = "OVERFLOW_FLUSH_ACTION";
public static final String BUFFERWRITE_FLUSH_ACTION = "BUFFERWRITE_FLUSH_ACTION";
public static final String BUFFERWRITE_CLOSE_ACTION = "BUFFERWRITE_CLOSE_ACTION";
public static final String FILENODE_PROCESSOR_FLUSH_ACTION = "FILENODE_PROCESSOR_FLUSH_ACTION";
- public static final String MREGE_EXTENSION = "merge";
- public static final String ERR_EXTENSION = "err";
- public static final String PATH_SEPARATOR = ".";
- public static final String BUFFERWRITE_FILE_SEPARATOR = "-";
+ public static final String TSFILE_NAME_SEPARATOR = "-";
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/Processor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/Processor.java
index 223523a..363c685 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/Processor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/Processor.java
@@ -23,7 +23,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.filenode.FileNodeProcessor;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/Action.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/Action.java
deleted file mode 100644
index af65a8a..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/Action.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.bufferwrite;
-
-/**
- * Action interface.
- */
-
-public interface Action {
-
- void act() throws ActionException;
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/ActionException.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/ActionException.java
deleted file mode 100644
index 1d76d9f..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/ActionException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.bufferwrite;
-
-public class ActionException extends Exception{
-
- public ActionException(String message) {
- super(message);
- }
-
- public ActionException(Throwable cause) {
- super(cause);
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
deleted file mode 100644
index b99d0a2..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessor.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.bufferwrite;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.memtable.IMemTable;
-import org.apache.iotdb.db.engine.memtable.MemSeriesLazyMerger;
-import org.apache.iotdb.db.engine.memtable.MemTableFlushUtil;
-import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
-import org.apache.iotdb.db.engine.pool.FlushManager;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.TsFileProcessorException;
-import org.apache.iotdb.db.qp.constant.DatetimeUtils;
-import org.apache.iotdb.db.utils.ImmediateFuture;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BufferWriteProcessor extends Processor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(BufferWriteProcessor.class);
- private RestorableTsFileIOWriter writer;
- private FileSchema fileSchema;
- private volatile Future<Boolean> flushFuture = new ImmediateFuture<>(true);
- private ReentrantLock flushQueryLock = new ReentrantLock();
- private AtomicLong memSize = new AtomicLong();
- private long memThreshold = TSFileDescriptor.getInstance().getConfig().groupSizeInByte;
- private IMemTable workMemTable;
- private IMemTable flushMemTable;
- private Action bufferwriteFlushAction;
- private Action bufferwriteCloseAction;
- private Action filenodeFlushAction;
-
- //lastFlushTime time unit: nanosecond
- private long lastFlushTime = -1;
- private long valueCount = 0;
-
- private String baseDir;
- private String insertFilePath;
- private String bufferWriteRelativePath;
-
- private WriteLogNode logNode;
- private VersionController versionController;
-
- private boolean isClosed = true;
- private boolean isFlush = false;
-
- /**
- * constructor of BufferWriteProcessor.
- *
- * @param baseDir base dir
- * @param processorName processor name
- * @param fileName file name
- * @param parameters parameters in Map(String, Action) structure
- * @param fileSchema file schema
- * @throws BufferWriteProcessorException BufferWriteProcessorException
- */
- public BufferWriteProcessor(String baseDir, String processorName, String fileName,
- Map<String, Action> parameters, VersionController versionController,
- FileSchema fileSchema) throws BufferWriteProcessorException {
- super(processorName);
- this.fileSchema = fileSchema;
- this.baseDir = baseDir;
-
- bufferwriteFlushAction = parameters.get(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION);
- bufferwriteCloseAction = parameters.get(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION);
- filenodeFlushAction = parameters.get(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
-
- reopen(fileName);
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- try {
- logNode = MultiFileLogNodeManager.getInstance().getNode(
- processorName + IoTDBConstant.BUFFERWRITE_LOG_NODE_SUFFIX,
- getBufferwriteRestoreFilePath(),
- FileNodeManager.getInstance().getRestoreFilePath(processorName));
- } catch (IOException e) {
- throw new BufferWriteProcessorException(e);
- }
- }
- this.versionController = versionController;
-
- }
-
- public void reopen(String fileName) throws BufferWriteProcessorException {
- if (!isClosed) {
- return;
- }
- new File(baseDir, processorName).mkdirs();
- this.insertFilePath = Paths.get(baseDir, processorName, fileName).toString();
- bufferWriteRelativePath = processorName + File.separatorChar + fileName;
- try {
- writer = new RestorableTsFileIOWriter(processorName, insertFilePath);
- } catch (IOException e) {
- throw new BufferWriteProcessorException(e);
- }
- if (workMemTable == null) {
- workMemTable = new PrimitiveMemTable();
- } else {
- workMemTable.clear();
- }
- isClosed = false;
- isFlush = false;
- }
-
- public void checkOpen() throws BufferWriteProcessorException {
- if (isClosed) {
- throw new BufferWriteProcessorException("BufferWriteProcessor already closed");
- }
- }
-
-
- /**
- * write one data point to the buffer write.
- *
- * @param deviceId device name
- * @param measurementId sensor name
- * @param timestamp timestamp of the data point
- * @param dataType the data type of the value
- * @param value data point value
- * @return true -the size of tsfile or metadata reaches to the threshold. false -otherwise
- * @throws BufferWriteProcessorException if a flushing operation occurs and failed.
- */
- public boolean write(String deviceId, String measurementId, long timestamp, TSDataType dataType,
- String value)
- throws BufferWriteProcessorException {
- checkOpen();
- TSRecord record = new TSRecord(timestamp, deviceId);
- DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementId, value);
- record.addTuple(dataPoint);
- return write(record);
- }
-
- /**
- * wrete a ts record into the memtable. If the memory usage is beyond the memThreshold, an async
- * flushing operation will be called.
- *
- * @param tsRecord data to be written
- * @return FIXME what is the mean about the return value??
- * @throws BufferWriteProcessorException if a flushing operation occurs and failed.
- */
- public boolean write(TSRecord tsRecord) throws BufferWriteProcessorException {
- checkOpen();
- long memUsage = MemUtils.getRecordSize(tsRecord);
- BasicMemController.UsageLevel level = BasicMemController.getInstance()
- .acquireUsage(this, memUsage);
-
- String memory;
- switch (level) {
- case SAFE:
- for (DataPoint dataPoint : tsRecord.dataPointList) {
- workMemTable.write(tsRecord.deviceId, dataPoint.getMeasurementId(), dataPoint.getType(),
- tsRecord.time,
- dataPoint.getValue().toString());
- }
- valueCount++;
- checkMemThreshold4Flush(memUsage);
- return true;
- case WARNING:
- memory = MemUtils.bytesCntToStr(BasicMemController.getInstance().getTotalUsage());
- LOGGER.warn("Memory usage will exceed warning threshold, current : {}.", memory);
- for (DataPoint dataPoint : tsRecord.dataPointList) {
- workMemTable.write(tsRecord.deviceId, dataPoint.getMeasurementId(), dataPoint.getType(),
- tsRecord.time,
- dataPoint.getValue().toString());
- }
- valueCount++;
- try {
- flush();
- } catch (IOException e) {
- throw new BufferWriteProcessorException(e);
- }
- return true;
- case DANGEROUS:
- default:
- memory = MemUtils.bytesCntToStr(BasicMemController.getInstance().getTotalUsage());
- LOGGER.warn("Memory usage will exceed dangerous threshold, current : {}.", memory);
- return false;
- }
- }
-
- private void checkMemThreshold4Flush(long addedMemory) throws BufferWriteProcessorException {
- long newMem = memSize.addAndGet(addedMemory);
- if (newMem > memThreshold) {
- String usageMem = MemUtils.bytesCntToStr(newMem);
- String threshold = MemUtils.bytesCntToStr(memThreshold);
- String processorName = getProcessorName();
- LOGGER.info("The usage of memory {} in bufferwrite processor {} reaches the threshold {}",
- usageMem, processorName, threshold);
- try {
- flush();
- } catch (IOException e) {
- LOGGER.error("Flush bufferwrite error.", e);
- throw new BufferWriteProcessorException(e);
- }
- }
- }
-
- /**
- * get the one (or two) chunk(s) in the memtable ( and the other one in flushing status and then
- * compact them into one TimeValuePairSorter). Then get its (or their) ChunkMetadata(s).
- *
- * @param deviceId device id
- * @param measurementId sensor id
- * @param dataType data type
- * @return corresponding chunk data and chunk metadata in memory
- */
- public Pair<ReadOnlyMemChunk, List<ChunkMetaData>> queryBufferWriteData(String deviceId,
- String measurementId, TSDataType dataType, Map<String, String> props)
- throws BufferWriteProcessorException {
- checkOpen();
- flushQueryLock.lock();
- try {
- MemSeriesLazyMerger memSeriesLazyMerger = new MemSeriesLazyMerger();
- if (flushMemTable != null) {
- memSeriesLazyMerger.addMemSeries(flushMemTable.query(deviceId, measurementId, dataType, props));
- }
- memSeriesLazyMerger.addMemSeries(workMemTable.query(deviceId, measurementId, dataType, props));
- // memSeriesLazyMerger has handled the props,
- // so we do not need to handle it again in the following readOnlyMemChunk
- ReadOnlyMemChunk timeValuePairSorter = new ReadOnlyMemChunk(dataType, memSeriesLazyMerger,
- Collections.emptyMap());
- return new Pair<>(timeValuePairSorter,
- writer.getMetadatas(deviceId, measurementId, dataType));
- } finally {
- flushQueryLock.unlock();
- }
- }
-
- private void switchWorkToFlush() {
- flushQueryLock.lock();
- try {
- IMemTable temp = flushMemTable == null ? new PrimitiveMemTable() : flushMemTable;
- flushMemTable = workMemTable;
- workMemTable = temp;
- isFlush = true;
- } finally {
- flushQueryLock.unlock();
- }
- }
-
- private void switchFlushToWork() {
- flushQueryLock.lock();
- try {
- flushMemTable.clear();
- writer.appendMetadata();
- isFlush = false;
- } finally {
- flushQueryLock.unlock();
- }
- }
-
-
- /**
- * the caller mast guarantee no other concurrent caller entering this function.
- *
- * @param displayMessage message that will appear in system log.
- * @param version the operation version that will tagged on the to be flushed memtable
- * (i.e., ChunkGroup)
- * @return true if successfully.
- */
- private boolean flushTask(String displayMessage, long version) {
- boolean result;
- long flushStartTime = System.currentTimeMillis();
- LOGGER.info("The bufferwrite processor {} starts flushing {}.", getProcessorName(),
- displayMessage);
- try {
- if (flushMemTable != null && !flushMemTable.isEmpty()) {
- // flush data
- MemTableFlushUtil.flushMemTable(fileSchema, writer, flushMemTable,
- version);
- // write restore information
- writer.flush();
- }
-
- filenodeFlushAction.act();
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode.notifyEndFlush(null);
- }
- result = true;
- } catch (Exception e) {
- LOGGER.error(
- "The bufferwrite processor {} failed to flush {}, when calling the filenodeFlushAction.",
- getProcessorName(), displayMessage, e);
- result = false;
- } finally {
- switchFlushToWork();
- LOGGER.info("The bufferwrite processor {} ends flushing {}.", getProcessorName(),
- displayMessage);
- }
- if (LOGGER.isInfoEnabled()) {
- long flushEndTime = System.currentTimeMillis();
- LOGGER.info(
- "The bufferwrite processor {} flush {}, start time is {}, flush end time is {}, "
- + "flush time consumption is {}ms",
- getProcessorName(), displayMessage,
- DatetimeUtils.convertMillsecondToZonedDateTime(flushStartTime),
- DatetimeUtils.convertMillsecondToZonedDateTime(flushEndTime),
- flushEndTime - flushStartTime);
- }
- return result;
- }
-
- // keyword synchronized is added in this method, so that only one flush task can be submitted now.
- @Override
- public synchronized Future<Boolean> flush() throws IOException {
- if (isClosed) {
- throw new IOException("BufferWriteProcessor closed");
- }
- // statistic information for flush
- if (lastFlushTime > 0) {
- if (LOGGER.isInfoEnabled()) {
- long thisFlushTime = System.currentTimeMillis();
- LOGGER.info(
- "The bufferwrite processor {}: last flush time is {}, this flush time is {}, "
- + "flush time interval is {}s", getProcessorName(),
- DatetimeUtils.convertMillsecondToZonedDateTime(lastFlushTime / 1000),
- DatetimeUtils.convertMillsecondToZonedDateTime(thisFlushTime),
- (thisFlushTime - lastFlushTime / 1000) / 1000);
- }
- }
- lastFlushTime = System.nanoTime();
- // check value count
- // waiting for the end of last flush operation.
- try {
- flushFuture.get();
- } catch (InterruptedException | ExecutionException e) {
- throw new IOException(e);
- }
- if (valueCount > 0) {
- // update the lastUpdatetime, prepare for flush
- try {
- bufferwriteFlushAction.act();
- } catch (Exception e) {
- LOGGER.error("Failed to flush bufferwrite row group when calling the action function.");
- throw new IOException(e);
- }
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode.notifyStartFlush();
- }
- valueCount = 0;
- switchWorkToFlush();
- long version = versionController.nextVersion();
- BasicMemController.getInstance().releaseUsage(this, memSize.get());
- memSize.set(0);
- // switch
- flushFuture = FlushManager.getInstance().submit(() -> flushTask("asynchronously",
- version));
- } else {
- flushFuture = new ImmediateFuture<>(true);
- }
- return flushFuture;
- }
-
- @Override
- public boolean canBeClosed() {
- return true;
- }
-
- @Override
- public void close() throws TsFileProcessorException {
- if (isClosed) {
- return;
- }
- try {
- long closeStartTime = System.currentTimeMillis();
- // flush data and wait for finishing flush
- flush().get();
- // end file
- writer.endFile(fileSchema);
- writer = null;
- workMemTable.clear();
-
- // update the IntervalFile for interval list
- bufferwriteCloseAction.act();
- // flush the changed information for filenode
- filenodeFlushAction.act();
- // delete the restore for this bufferwrite processor
- if (LOGGER.isInfoEnabled()) {
- long closeEndTime = System.currentTimeMillis();
- LOGGER.info(
- "Close bufferwrite processor {}, the file name is {}, start time is {}, end time is {}, "
- + "time consumption is {}ms",
- getProcessorName(), insertFilePath,
- DatetimeUtils.convertMillsecondToZonedDateTime(closeStartTime),
- DatetimeUtils.convertMillsecondToZonedDateTime(closeEndTime),
- closeEndTime - closeStartTime);
- }
- isClosed = true;
- } catch (IOException e) {
- LOGGER.error("Close the bufferwrite processor error, the bufferwrite is {}.",
- getProcessorName(), e);
- throw new TsFileProcessorException(e);
- } catch (Exception e) {
- LOGGER
- .error("Failed to close the bufferwrite processor when calling the action function.", e);
- throw new TsFileProcessorException(e);
- }
- }
-
- @Override
- public long memoryUsage() {
- return memSize.get();
- }
-
- /**
- * check if is flushing.
- *
- * @return True if flushing
- */
- public boolean isFlush() {
- return isFlush;
- }
-
- /**
- * get metadata size.
- *
- * @return The sum of all timeseries's metadata size within this file.
- */
- public long getMetaSize() {
- // TODO : [MemControl] implement this
- return 0;
- }
-
- /**
- * get file size.
- *
- * @return The file size of the TsFile corresponding to this processor.
- */
- public long getFileSize() {
- // TODO : save this variable to avoid object creation?
- File file = new File(insertFilePath);
- return file.length() + memoryUsage();
- }
-
- public String getBaseDir() {
- return baseDir;
- }
-
-
- public String getFileRelativePath() {
- return bufferWriteRelativePath;
- }
-
- private String getBufferwriteRestoreFilePath() {
- return writer.getRestoreFilePath();
- }
-
- public boolean isNewProcessor() {
- return writer.isNewResource();
- }
-
- public void setNewProcessor(boolean isNewProcessor) {
- writer.setNewResource(isNewProcessor);
- }
-
- public WriteLogNode getLogNode() {
- return logNode;
- }
-
- /**
- * used for test. We can know when the flush() is called.
- * @return the last flush() time. Time unit: nanosecond.
- */
- public long getLastFlushTime() {
- return lastFlushTime;
- }
-
- /**
- * used for test. We can block to wait for finishing flushing.
- * @return the future of the flush() task.
- */
- public Future<Boolean> getFlushFuture() {
- return flushFuture;
- }
-
- /**
- * Delete data whose timestamp <= 'timestamp' and belonging to timeseries deviceId.measurementId.
- * Delete data in both working MemTable and flushing MemTable.
- *
- * @param deviceId the deviceId of the timeseries to be deleted.
- * @param measurementId the measurementId of the timeseries to be deleted.
- * @param timestamp the upper-bound of deletion time.
- */
- public void delete(String deviceId, String measurementId, long timestamp)
- throws BufferWriteProcessorException {
- checkOpen();
- workMemTable.delete(deviceId, measurementId, timestamp);
- if (isFlush()) {
- // flushing MemTable cannot be directly modified since another thread is reading it
- flushMemTable = flushMemTable.copy();
- flushMemTable.delete(deviceId, measurementId, timestamp);
- }
- }
-
- @Override
- public boolean equals(Object o) {
- return this == o;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode());
- }
-
- @Override
- public String toString() {
- return "BufferWriteProcessor in " + insertFilePath;
- }
-
- public String getInsertFilePath() {
- return insertFilePath;
- }
-
- public boolean isClosed() {
- return isClosed;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeFlushFuture.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeFlushFuture.java
deleted file mode 100644
index 85c5bb4..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeFlushFuture.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.filenode;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.iotdb.db.utils.ImmediateFuture;
-
-public class FileNodeFlushFuture implements Future<Boolean> {
- Future<Boolean> bufferWriteFlushFuture;
- Future<Boolean> overflowFlushFuture;
- boolean hasOverflowFlushTask;
-
- public FileNodeFlushFuture(Future<Boolean> bufferWriteFlushFuture, Future<Boolean> overflowFlushFuture){
- if(bufferWriteFlushFuture != null) {
- this.bufferWriteFlushFuture = bufferWriteFlushFuture;
- } else {
- this.bufferWriteFlushFuture = new ImmediateFuture<>(true);
- }
- if(overflowFlushFuture !=null) {
- this.overflowFlushFuture = overflowFlushFuture;
- hasOverflowFlushTask = true;
- } else {
- this.overflowFlushFuture = new ImmediateFuture<>(true);
- hasOverflowFlushTask = false;
- }
- }
-
- /**
- * @param mayInterruptIfRunning true if the thread executing this task should be interrupted;
- * otherwise, in-progress tasks are allowed to complete
- * @return true if both of the two sub-future are canceled successfully.
- * @see Future#cancel(boolean) The difference is that this Future consists of two sub-Futures. If
- * the first sub-future is canceled successfully but the second sub-future fails, the result is
- * false.
- */
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- boolean bwResult = bufferWriteFlushFuture.cancel(mayInterruptIfRunning);
- boolean ofResult = overflowFlushFuture.cancel(mayInterruptIfRunning);
- return bwResult && ofResult;
- }
-
- @Override
- public boolean isCancelled() {
- return bufferWriteFlushFuture.isCancelled() && overflowFlushFuture.isCancelled();
- }
-
- @Override
- public boolean isDone() {
- return bufferWriteFlushFuture.isDone() && overflowFlushFuture.isDone();
- }
-
- @Override
- public Boolean get() throws InterruptedException, ExecutionException {
- boolean bwResult = bufferWriteFlushFuture.get();
- boolean ofResult = overflowFlushFuture.get();
- return bwResult && ofResult;
- }
-
- @Override
- public Boolean get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- boolean result = bufferWriteFlushFuture.get(timeout, unit);
- result = result && overflowFlushFuture.get(timeout, unit);
- return result;
- }
-
- public boolean isHasOverflowFlushTask() {
- return hasOverflowFlushTask;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
deleted file mode 100644
index 0404a5c..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ /dev/null
@@ -1,1217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.filenode;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
-import org.apache.iotdb.db.engine.pool.FlushManager;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.StorageGroupManagerException;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
-import org.apache.iotdb.db.exception.PathErrorException;
-import org.apache.iotdb.db.exception.ProcessorException;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.monitor.IStatistic;
-import org.apache.iotdb.db.monitor.MonitorConstants;
-import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
-import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
-import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.service.IService;
-import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileNodeManager implements IStatistic, IService {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeManager.class);
- private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- private static final Directories directories = Directories.getInstance();
- /**
- * a folder that persist FileNodeProcessorStore classes. Each stroage group will have a subfolder.
- * by default, it is system/info
- */
- private final String baseDir;
-
- /**
- * This map is used to manage all filenode processor,<br> the key is filenode name which is
- * storage group seriesPath.
- */
- private ConcurrentHashMap<String, FileNodeProcessor> processorMap;
- /**
- * This set is used to store overflowed filenode name.<br> The overflowed filenode will be merge.
- */
- private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // There is no need to add concurrently
- private HashMap<String, AtomicLong> statParamsHashMap;
-
- private FileNodeManager(String baseDir) {
- processorMap = new ConcurrentHashMap<>();
- statParamsHashMap = new HashMap<>();
- //label: A
- for (StorageGroupManagerStatConstants fileNodeManagerStatConstant :
- StorageGroupManagerStatConstants.values()) {
- statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
- }
-
- String normalizedBaseDir = baseDir;
- if (normalizedBaseDir.charAt(normalizedBaseDir.length() - 1) != File.separatorChar) {
- normalizedBaseDir += Character.toString(File.separatorChar);
- }
- this.baseDir = normalizedBaseDir;
- File dir = new File(normalizedBaseDir);
- if (dir.mkdirs()) {
- LOGGER.info("{} dir home doesn't exist, create it", dir.getPath());
- }
- //TODO merge this with label A
- if (TsFileDBConf.isEnableStatMonitor()) {
- StatMonitor statMonitor = StatMonitor.getInstance();
- registerStatMetadata();
- statMonitor.registerStatistics(MonitorConstants.STAT_STORAGE_DELTA_NAME, this);
- }
- }
-
- public static FileNodeManager getInstance() {
- return FileNodeManagerHolder.INSTANCE;
- }
-
- private void updateStatHashMapWhenFail(TSRecord tsRecord) {
- statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_FAIL.name())
- .incrementAndGet();
- statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS_FAIL.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
-
- /**
- * get stats parameter hash map.
- *
- * @return the key represents the params' name, values is AtomicLong type
- */
- @Override
- public Map<String, AtomicLong> getStatParamsHashMap() {
- return statParamsHashMap;
- }
-
- @Override
- public List<String> getAllPathForStatistic() {
- List<String> list = new ArrayList<>();
- for (StorageGroupManagerStatConstants statConstant :
- StorageGroupManagerStatConstants.values()) {
- list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- + statConstant.name());
- }
- return list;
- }
-
- @Override
- public Map<String, TSRecord> getAllStatisticsValue() {
- long curTime = System.currentTimeMillis();
- TSRecord tsRecord = StatMonitor
- .convertToTSRecord(getStatParamsHashMap(), MonitorConstants.STAT_STORAGE_DELTA_NAME,
- curTime);
- HashMap<String, TSRecord> ret = new HashMap<>();
- ret.put(MonitorConstants.STAT_STORAGE_DELTA_NAME, tsRecord);
- return ret;
- }
-
- /**
- * Init Stat MetaDta.
- */
- @Override
- public void registerStatMetadata() {
- Map<String, String> hashMap = new HashMap<>();
- for (StorageGroupManagerStatConstants statConstant :
- StorageGroupManagerStatConstants.values()) {
- hashMap
- .put(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- + statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
- }
- StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- }
-
- /**
- * This function is just for unit test.
- */
- public synchronized void resetFileNodeManager() {
- for (String key : statParamsHashMap.keySet()) {
- statParamsHashMap.put(key, new AtomicLong());
- }
- processorMap.clear();
- }
-
- /**
- * @param filenodeName storage name, e.g., root.a.b
- */
- private FileNodeProcessor constructNewProcessor(String filenodeName)
- throws StorageGroupManagerException {
- try {
- return new FileNodeProcessor(baseDir, filenodeName);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Can't construct the FileNodeProcessor, the filenode is {}", filenodeName, e);
- throw new StorageGroupManagerException(e);
- }
- }
-
- private FileNodeProcessor getProcessor(String path, boolean isWriteLock)
- throws StorageGroupManagerException {
- String filenodeName;
- try {
- // return the stroage name
- filenodeName = MManager.getInstance().getStorageGroupByPath(path);
- } catch (PathErrorException e) {
- LOGGER.error("MManager get filenode name error, seriesPath is {}", path);
- throw new StorageGroupManagerException(e);
- }
- FileNodeProcessor processor;
- processor = processorMap.get(filenodeName);
- if (processor != null) {
- processor.lock(isWriteLock);
- } else {
- filenodeName = filenodeName.intern();
- // calculate the value with same key synchronously
- synchronized (filenodeName) {
- processor = processorMap.get(filenodeName);
- if (processor != null) {
- processor.lock(isWriteLock);
- } else {
- // calculate the value with the key monitor
- LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
- filenodeName, Thread.currentThread().getId());
- processor = constructNewProcessor(filenodeName);
- processor.lock(isWriteLock);
- processorMap.put(filenodeName, processor);
- }
- }
- }
- return processor;
- }
-
- /**
- * recovery the filenode processor.
- */
- public void recovery() {
- List<String> filenodeNames = null;
- try {
- filenodeNames = MManager.getInstance().getAllStorageGroups();
- } catch (PathErrorException e) {
- LOGGER.error("Restoring all FileNodes failed.", e);
- return;
- }
- for (String filenodeName : filenodeNames) {
- FileNodeProcessor fileNodeProcessor = null;
- try {
- fileNodeProcessor = getProcessor(filenodeName, true);
- if (fileNodeProcessor.shouldRecovery()) {
- LOGGER.info("Recovery the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- fileNodeProcessor.fileNodeRecovery();
- }
- } catch (StorageGroupManagerException | FileNodeProcessorException e) {
- LOGGER.error("Restoring fileNode {} failed.", filenodeName, e);
- } finally {
- if (fileNodeProcessor != null) {
- fileNodeProcessor.writeUnlock();
- }
- }
- // add index check sum
- }
- }
-
- /**
- * insert TsRecord into storage group.
- *
- * @param tsRecord input Data
- * @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
- * be recorded. if false, the statParamsHashMap will be updated.
- */
- public void insert(TSRecord tsRecord, boolean isMonitor) throws StorageGroupManagerException {
- long timestamp = tsRecord.time;
-
- String deviceId = tsRecord.deviceId;
- checkTimestamp(tsRecord);
- updateStat(isMonitor, tsRecord);
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
-
- try {
- long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
- if (timestamp < lastUpdateTime) {
- insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
- } else {
- insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
- }
- } catch (FileNodeProcessorException e) {
- LOGGER.error(String.format("Encounter an error when closing the buffer write processor %s.",
- fileNodeProcessor.getProcessorName()), e);
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // Modify the insert
- if (!isMonitor) {
- fileNodeProcessor.getStatParamsHashMap()
- .get(StorageGroupProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
- fileNodeProcessor.getStatParamsHashMap()
- .get(StorageGroupProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
- .incrementAndGet();
- statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_SUCCESS.name())
- .incrementAndGet();
- statParamsHashMap
- .get(StorageGroupManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
- }
-
- private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
- throws StorageGroupManagerException {
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- String[] measurementList = new String[tsRecord.dataPointList.size()];
- String[] insertValues = new String[tsRecord.dataPointList.size()];
- int i=0;
- for (DataPoint dp : tsRecord.dataPointList) {
- measurementList[i] = dp.getMeasurementId();
- insertValues[i] = dp.getValue().toString();
- i++;
- }
- logNode.write(new InsertPlan(2, tsRecord.deviceId, tsRecord.time, measurementList,
- insertValues));
- }
- } catch (IOException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
- }
-
- private void checkTimestamp(TSRecord tsRecord) throws StorageGroupManagerException {
- if (tsRecord.time < 0) {
- LOGGER.error("The insert time lt 0, {}.", tsRecord);
- throw new StorageGroupManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
- }
- }
-
- private void updateStat(boolean isMonitor, TSRecord tsRecord) {
- if (!isMonitor) {
- statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
- }
-
- private void insertOverflow(FileNodeProcessor fileNodeProcessor, long timestamp,
- TSRecord tsRecord, boolean isMonitor, String deviceId)
- throws StorageGroupManagerException {
- // get overflow processor
- OverflowProcessor overflowProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
- // write wal
- writeLog(tsRecord, isMonitor, overflowProcessor.getLogNode());
- // write overflow data
- try {
- overflowProcessor.insert(tsRecord);
- fileNodeProcessor.changeTypeToChanged(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- } catch (IOException e) {
- LOGGER.error("Insert into overflow error, the reason is {}", e);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
- }
-
- private void insertBufferWrite(FileNodeProcessor fileNodeProcessor, long timestamp,
- boolean isMonitor, TSRecord tsRecord, String deviceId)
- throws StorageGroupManagerException, FileNodeProcessorException {
- // get bufferwrite processor
- BufferWriteProcessor bufferWriteProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName, timestamp);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Get the bufferwrite processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
- // Add a new interval file to newfilelist
- if (bufferWriteProcessor.isNewProcessor()) {
- bufferWriteProcessor.setNewProcessor(false);
- String bufferwriteBaseDir = bufferWriteProcessor.getBaseDir();
- String bufferwriteRelativePath = bufferWriteProcessor.getFileRelativePath();
- try {
- fileNodeProcessor.addIntervalFileNode(new File(new File(bufferwriteBaseDir), bufferwriteRelativePath));
- } catch (Exception e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
- }
- // write wal
- writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
- // Write data
- long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
- long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
-
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId);
- fileNodeProcessor.setLastUpdateTime(deviceId, timestamp);
- try {
- if (!bufferWriteProcessor.write(tsRecord)) {
- // undo time update
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId, prevStartTime);
- fileNodeProcessor.setLastUpdateTime(deviceId, prevUpdateTime);
- }
- } catch (BufferWriteProcessorException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new StorageGroupManagerException(e);
- }
-
- if (bufferWriteProcessor
- .getFileSize() > IoTDBDescriptor.getInstance()
- .getConfig().getBufferwriteFileSizeThreshold()) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "The filenode processor {} will close the bufferwrite processor, "
- + "because the size[{}] of tsfile {} reaches the threshold {}",
- filenodeName, MemUtils.bytesCntToStr(bufferWriteProcessor.getFileSize()),
- bufferWriteProcessor.getInsertFilePath(), MemUtils.bytesCntToStr(
- IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold()));
- }
-
- fileNodeProcessor.closeBufferWrite();
- }
- }
-
- /**
- * update data.
- */
- public void update(String deviceId, String measurementId, long startTime, long endTime,
- TSDataType type, String v)
- throws StorageGroupManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
-
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- if (startTime > lastUpdateTime) {
- LOGGER.warn("The update range is error, startTime {} is great than lastUpdateTime {}",
- startTime,
- lastUpdateTime);
- return;
- }
- long finalEndTime = endTime > lastUpdateTime ? lastUpdateTime : endTime;
-
- String filenodeName = fileNodeProcessor.getProcessorName();
- // get overflow processor
- OverflowProcessor overflowProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error(
- "Get the overflow processor failed, the filenode is {}, "
- + "insert time range is from {} to {}",
- filenodeName, startTime, finalEndTime);
- throw new StorageGroupManagerException(e);
- }
- overflowProcessor.update(deviceId, measurementId, startTime, finalEndTime, type, v);
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChanged(deviceId, startTime, finalEndTime);
- fileNodeProcessor.setOverflowed(true);
-
- // write wal
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- overflowProcessor.getLogNode().write(
- new UpdatePlan(startTime, finalEndTime, v, new Path(deviceId
- + "." + measurementId)));
- }
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * delete data.
- */
- public void delete(String deviceId, String measurementId, long timestamp)
- throws StorageGroupManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- // no tsfile data, the delete operation is invalid
- if (lastUpdateTime == -1) {
- LOGGER.warn("The last update time is -1, delete overflow is invalid, "
- + "the filenode processor is {}",
- fileNodeProcessor.getProcessorName());
- } else {
- // write wal
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // get processors for wal
- String filenodeName = fileNodeProcessor.getProcessorName();
- OverflowProcessor overflowProcessor;
- BufferWriteProcessor bufferWriteProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // in case that no BufferWriteProcessor is available, a new BufferWriteProcessor is
- // needed to access LogNode.
- // TODO this may make the time range of the next TsFile a little wider
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName,
- lastUpdateTime + 1);
- } catch (IOException | FileNodeProcessorException e) {
- LOGGER.error("Getting the processor failed, the filenode is {}, delete time is {}.",
- filenodeName, timestamp);
- throw new StorageGroupManagerException(e);
- }
- try {
- overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
- }
-
- try {
- fileNodeProcessor.delete(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
-
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- private void delete(String processorName,
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- throws StorageGroupManagerException {
- if (!processorMap.containsKey(processorName)) {
- //TODO do we need to call processorIterator.remove() ?
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- return;
- }
- LOGGER.info("Try to delete the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- if (!processor.tryWriteLock()) {
- throw new StorageGroupManagerException(String
- .format("Can't delete the filenode processor %s because Can't get the write lock.",
- processorName));
- }
-
- try {
- if (!processor.canBeClosed()) {
- LOGGER.warn("The filenode processor {} can't be deleted.", processorName);
- return;
- }
-
- try {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
- processorIterator.remove();
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} by iterator error.", processorName, e);
- throw new StorageGroupManagerException(e);
- }
- } finally {
- processor.writeUnlock();
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- */
- public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- fileNodeProcessor.deleteBufferWrite(deviceId, measurementId, timestamp);
- } catch (BufferWriteProcessorException | IOException e) {
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- }
-
- /**
- * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- */
- public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- fileNodeProcessor.deleteOverflow(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- }
-
- /**
- * begin query.
- *
- * @param deviceId queried deviceId
- * @return a query token for the device.
- */
- public int beginQuery(String deviceId) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, begin query.",
- fileNodeProcessor.getProcessorName());
- return fileNodeProcessor.addMultiPassCount();
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * query data.
- */
- public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
- throws StorageGroupManagerException {
- String deviceId = seriesExpression.getSeriesPath().getDevice();
- String measurementId = seriesExpression.getSeriesPath().getMeasurement();
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, false);
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
- fileNodeProcessor.getProcessorName());
- try {
- QueryDataSource queryDataSource;
- // query operation must have overflow processor
- if (!fileNodeProcessor.hasOverflowProcessor()) {
- try {
- fileNodeProcessor.getOverflowProcessor(fileNodeProcessor.getProcessorName());
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, query is {},{}",
- fileNodeProcessor.getProcessorName(), deviceId, measurementId);
- throw new StorageGroupManagerException(e);
- }
- }
- try {
- queryDataSource = fileNodeProcessor.query(deviceId, measurementId, context);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Query error: the deviceId {}, the measurementId {}", deviceId, measurementId,
- e);
- throw new StorageGroupManagerException(e);
- }
- // return query structure
- return queryDataSource;
- } finally {
- fileNodeProcessor.readUnlock();
- }
- }
-
- /**
- * end query.
- */
- public void endQuery(String deviceId, int token) throws StorageGroupManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- LOGGER.debug("Get the FileNodeProcessor: {} end query.",
- fileNodeProcessor.getProcessorName());
- fileNodeProcessor.decreaseMultiPassCount(token);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Failed to end query: the deviceId {}, token {}.", deviceId, token, e);
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * Append one specified tsfile to the storage group. <b>This method is only provided for
- * transmission module</b>
- *
- * @param fileNodeName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
- public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
- String appendFilePath) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- try {
- // check append file
- for (Map.Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- if (fileNodeProcessor.getLastUpdateTime(entry.getKey()) >= entry.getValue()) {
- return false;
- }
- }
- // close bufferwrite file
- fileNodeProcessor.closeBufferWrite();
- // append file to storage group.
- fileNodeProcessor.appendFile(appendFile, appendFilePath);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- return true;
- }
-
- /**
- * get all overlap tsfiles which are conflict with the appendFile.
- *
- * @param fileNodeName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
- public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
- String uuid) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- List<String> overlapFiles;
- try {
- overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
- } catch (FileNodeProcessorException e) {
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- return overlapFiles;
- }
-
- /**
- * merge all overflowed filenode.
- *
- * @throws StorageGroupManagerException StorageGroupManagerException
- */
- public void mergeAll() throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
- fileNodeManagerStatus);
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.MERGE;
- LOGGER.info("Start to merge all overflowed filenode");
- List<String> allFileNodeNames;
- try {
- allFileNodeNames = MManager.getInstance().getAllStorageGroups();
- } catch (PathErrorException e) {
- LOGGER.error("Get all storage group seriesPath error,", e);
- throw new StorageGroupManagerException(e);
- }
- List<Future<?>> futureTasks = new ArrayList<>();
- for (String fileNodeName : allFileNodeNames) {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- try {
- Future<?> task = fileNodeProcessor.submitToMerge();
- if (task != null) {
- LOGGER.info("Submit the filenode {} to the merge pool", fileNodeName);
- futureTasks.add(task);
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
- long totalTime = 0;
- // loop waiting for merge to end, the longest waiting time is
- // 60s.
- int time = 2;
- List<Exception> mergeException = new ArrayList<>();
- for (Future<?> task : futureTasks) {
- while (!task.isDone()) {
- try {
- LOGGER.info(
- "Waiting for the end of merge, already waiting for {}s, "
- + "continue to wait anothor {}s",
- totalTime, time);
- TimeUnit.SECONDS.sleep(time);
- totalTime += time;
- time = updateWaitTime(time);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
- Thread.currentThread().interrupt();
- }
- }
- try {
- task.get();
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
- } catch (ExecutionException e) {
- mergeException.add(e);
- LOGGER.error("The exception for merge: {}", e);
- }
- }
- if (!mergeException.isEmpty()) {
- // just throw the first exception
- throw new StorageGroupManagerException(mergeException.get(0));
- }
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- LOGGER.info("End to merge all overflowed filenode");
- }
-
- private int updateWaitTime(int time) {
- return time < 32 ? time * 2 : 60;
- }
-
- /**
- * try to close the filenode processor. The name of filenode processor is processorName
- */
- private boolean closeOneProcessor(String processorName) throws StorageGroupManagerException {
- if (!processorMap.containsKey(processorName)) {
- return true;
- }
-
- Processor processor = processorMap.get(processorName);
- if (processor.tryWriteLock()) {
- try {
- if (processor.canBeClosed()) {
- processor.close();
- return true;
- } else {
- return false;
- }
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
- } finally {
- processor.writeUnlock();
- }
- } else {
- return false;
- }
- }
-
- /**
- * delete one filenode.
- */
- public void deleteOneFileNode(String processorName) throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- if (processorMap.containsKey(processorName)) {
- deleteFileNodeBlocked(processorName);
- }
- String fileNodePath = TsFileDBConf.getFileNodeDir();
- fileNodePath = standardizeDir(fileNodePath) + processorName;
- FileUtils.deleteDirectory(new File(fileNodePath));
-
- cleanBufferWrite(processorName);
-
- MultiFileLogNodeManager.getInstance()
- .deleteNode(processorName + IoTDBConstant.BUFFERWRITE_LOG_NODE_SUFFIX);
- MultiFileLogNodeManager.getInstance()
- .deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
- } catch (IOException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- private void cleanBufferWrite(String processorName) throws IOException {
- List<String> bufferwritePathList = directories.getAllTsFileFolders();
- for (String bufferwritePath : bufferwritePathList) {
- bufferwritePath = standardizeDir(bufferwritePath) + processorName;
- File bufferDir = new File(bufferwritePath);
- // free and close the streams under this bufferwrite directory
- if (!bufferDir.exists()) {
- continue;
- }
- File[] bufferFiles = bufferDir.listFiles();
- if (bufferFiles != null) {
- for (File bufferFile : bufferFiles) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(bufferFile.getPath());
- }
- }
- FileUtils.deleteDirectory(new File(bufferwritePath));
- }
- }
-
- private void deleteFileNodeBlocked(String processorName) throws StorageGroupManagerException {
- LOGGER.info("Forced to delete the filenode processor {}", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- while (true) {
- if (processor.tryWriteLock()) {
- try {
- if (processor.canBeClosed()) {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
- processorMap.remove(processorName);
- break;
- } else {
- LOGGER.info(
- "Can't delete the filenode processor {}, "
- + "because the filenode processor can't be closed."
- + " Wait 100ms to retry");
- }
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
- } finally {
- processor.writeUnlock();
- }
- } else {
- LOGGER.info(
- "Can't delete the filenode processor {}, because it can't get the write lock."
- + " Wait 100ms to retry", processorName);
- }
- try {
- TimeUnit.MILLISECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error(e.getMessage());
- Thread.currentThread().interrupt();
- }
- }
- }
-
- private String standardizeDir(String originalPath) {
- String res = originalPath;
- if ((originalPath.length() > 0
- && originalPath.charAt(originalPath.length() - 1) != File.separatorChar)
- || originalPath.length() == 0) {
- res = originalPath + File.separatorChar;
- }
- return res;
- }
-
- /**
- * add time series.
- */
- public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
- CompressionType compressor,
- Map<String, String> props) throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
- try {
- fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
-
- /**
- * Force to close the filenode processor.
- */
- public void closeOneFileNode(String processorName) throws StorageGroupManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- LOGGER.info("Force to close the filenode processor {}.", processorName);
- while (!closeOneProcessor(processorName)) {
- try {
- LOGGER.info("Can't force to close the filenode processor {}, wait 100ms to retry",
- processorName);
- TimeUnit.MILLISECONDS.sleep(100);
- } catch (InterruptedException e) {
- // ignore the interrupted exception
- LOGGER.error("Unexpected interruption {}", e);
- Thread.currentThread().interrupt();
- }
- }
- } finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * try to close the filenode processor.
- */
- private void close(String processorName) throws StorageGroupManagerException {
- if (!processorMap.containsKey(processorName)) {
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- return;
- }
- LOGGER.info("Try to close the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- if (!processor.tryWriteLock()) {
- LOGGER.warn("Can't get the write lock of the filenode processor {}.", processorName);
- return;
- }
- try {
- if (processor.canBeClosed()) {
- try {
- LOGGER.info("Close the filenode processor {}.", processorName);
- processor.close();
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new StorageGroupManagerException(e);
- }
- } else {
- LOGGER.warn("The filenode processor {} can't be closed.", processorName);
- }
- } finally {
- processor.writeUnlock();
- }
- }
-
- /**
- * delete all filenode.
- */
- public synchronized boolean deleteAll() throws StorageGroupManagerException {
- LOGGER.info("Start deleting all filenode");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to delete all filenode processor because of merge operation");
- return false;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator = processorMap.entrySet()
- .iterator();
- while (processorIterator.hasNext()) {
- Map.Entry<String, FileNodeProcessor> processorEntry = processorIterator.next();
- delete(processorEntry.getKey(), processorIterator);
- }
- return processorMap.isEmpty();
- } finally {
- LOGGER.info("Deleting all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * Try to close All.
- */
- public void closeAll() throws StorageGroupManagerException {
- LOGGER.info("Start closing all filenode processor");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to close all filenode processor because of merge operation");
- return;
- }
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- for (Map.Entry<String, FileNodeProcessor> processorEntry : processorMap.entrySet()) {
- close(processorEntry.getKey());
- }
- } finally {
- LOGGER.info("Close all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * force flush to control memory usage.
- */
- public void forceFlush(BasicMemController.UsageLevel level) {
- // you may add some delicate process like below
- // or you could provide multiple methods for different urgency
- switch (level) {
- // only select the most urgent (most active or biggest in size)
- // processors to flush
- // only select top 10% active memory user to flush
- case WARNING:
- try {
- flushTop(0.1f);
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
- }
- break;
- // force all processors to flush
- case DANGEROUS:
- try {
- flushAll();
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
- }
- break;
- // if the flush thread pool is not full ( or half full), start a new
- // flush task
- case SAFE:
- if (FlushManager.getInstance().getActiveCnt() < 0.5 * FlushManager.getInstance()
- .getThreadCnt()) {
- try {
- flushTop(0.01f);
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: ", e);
- }
- }
- break;
- default:
- }
- }
-
- private void flushAll() throws IOException {
- for (FileNodeProcessor processor : processorMap.values()) {
- if (!processor.tryLock(true)) {
- continue;
- }
- try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
- }
- } finally {
- processor.unlock(true);
- }
- }
- }
-
- private void flushTop(float percentage) throws IOException {
- List<FileNodeProcessor> tempProcessors = new ArrayList<>(processorMap.values());
- // sort the tempProcessors as descending order
- tempProcessors.sort((o1, o2) -> (int) (o2.memoryUsage() - o1.memoryUsage()));
- int flushNum =
- (int) (tempProcessors.size() * percentage) > 1
- ? (int) (tempProcessors.size() * percentage)
- : 1;
- for (int i = 0; i < flushNum && i < tempProcessors.size(); i++) {
- FileNodeProcessor processor = tempProcessors.get(i);
- // 64M
- if (processor.memoryUsage() <= TSFileConfig.groupSizeInByte / 2) {
- continue;
- }
- processor.writeLock();
- try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
- }
- } finally {
- processor.writeUnlock();
- }
- }
- }
-
- @Override
- public void start() {
- // do no thing
- }
-
- @Override
- public void stop() {
- try {
- closeAll();
- } catch (StorageGroupManagerException e) {
- LOGGER.error("Failed to close file node manager because .", e);
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.FILE_NODE_SERVICE;
- }
-
- /**
- * get restore file path.
- */
- public String getRestoreFilePath(String processorName) {
- FileNodeProcessor fileNodeProcessor = processorMap.get(processorName);
- if (fileNodeProcessor != null) {
- return fileNodeProcessor.getFileNodeRestoreFilePath();
- } else {
- return null;
- }
- }
-
- /**
- * recover filenode.
- */
- public void recoverFileNode(String filenodeName)
- throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(filenodeName, true);
- LOGGER.info("Recover the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- try {
- fileNodeProcessor.fileNodeRecovery();
- } catch (FileNodeProcessorException e) {
- throw new StorageGroupManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- private enum FileNodeManagerStatus {
- NONE, MERGE, CLOSE
- }
-
- private static class FileNodeManagerHolder {
-
- private FileNodeManagerHolder() {
- }
-
- private static final FileNodeManager INSTANCE = new FileNodeManager(
- TsFileDBConf.getFileNodeDir());
- }
-
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
deleted file mode 100644
index a66116e..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++ /dev/null
@@ -1,2065 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.filenode;
-
-import static java.time.ZonedDateTime.ofInstant;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.Action;
-import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.modification.Deletion;
-import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
-import org.apache.iotdb.db.engine.pool.MergeManager;
-import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
-import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
-import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.ErrorDebugException;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
-import org.apache.iotdb.db.exception.OverflowProcessorException;
-import org.apache.iotdb.db.exception.PathErrorException;
-import org.apache.iotdb.db.exception.ProcessorException;
-import org.apache.iotdb.db.exception.TsFileProcessorException;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.monitor.IStatistic;
-import org.apache.iotdb.db.monitor.MonitorConstants;
-import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
-import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
-import org.apache.iotdb.db.query.reader.IReader;
-import org.apache.iotdb.db.sync.conf.Constans;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.utils.QueryUtils;
-import org.apache.iotdb.db.utils.TimeValuePair;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.read.filter.TimeFilter;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
-import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileNodeProcessor extends Processor implements IStatistic {
-
- private static final String WARN_NO_SUCH_OVERFLOWED_FILE = "Can not find any tsfile which"
- + " will be overflowed in the filenode processor {}, ";
- private static final String RESTORE_FILE_SUFFIX = ".restore";
- private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeProcessor.class);
- private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- private static final MManager mManager = MManager.getInstance();
- private static final Directories directories = Directories.getInstance();
- private final String statStorageDeltaName;
- private final HashMap<String, AtomicLong> statParamsHashMap = new HashMap<>();
- /**
- * Used to keep the oldest timestamp for each deviceId. The key is deviceId.
- */
- private volatile boolean isOverflowed;
- private Map<String, Long> lastUpdateTimeMap;
- private Map<String, Long> flushLastUpdateTimeMap;
- private Map<String, List<TsFileResource>> invertedIndexOfFiles;
- private TsFileResource emptyTsFileResource;
- private TsFileResource currentTsFileResource;
- private List<TsFileResource> newFileNodes;
- private FileNodeProcessorStatus isMerging;
-
- /**
- * this is used when work->merge operation
- */
- private int numOfMergeFile;
- private FileNodeProcessorStore fileNodeProcessorStore;
- private String fileNodeRestoreFilePath;
- private final Object fileNodeRestoreLock = new Object();
-
- /**
- * last merge time
- */
- private long lastMergeTime = -1;
- private BufferWriteProcessor bufferWriteProcessor = null;
- private OverflowProcessor overflowProcessor1 = null;
- private Set<Integer> oldMultiPassTokenSet = null;
- private Set<Integer> newMultiPassTokenSet = new HashSet<>();
-
- /**
- * Represent the number of old queries that have not ended.
- * This parameter only decreases but not increase.
- */
- private CountDownLatch oldMultiPassCount = null;
-
- /**
- * Represent the number of new queries that have not ended.
- */
- private AtomicInteger newMultiPassCount = new AtomicInteger(0);
- /**
- * system recovery
- */
- private boolean shouldRecovery = false;
- /**
- * statistic monitor parameters
- */
- private Map<String, Action> parameters;
- private FileSchema fileSchema;
- private Action flushFileNodeProcessorAction = () -> {
- synchronized (fileNodeProcessorStore) {
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- throw new ActionException(e);
- }
- }
- };
- private Action bufferwriteFlushAction = () -> {
- // update the lastUpdateTime Notice: Thread safe
- synchronized (fileNodeProcessorStore) {
- // deep copy
- Map<String, Long> tempLastUpdateMap = new HashMap<>(lastUpdateTimeMap);
- // update flushLastUpdateTimeMap
- for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- flushLastUpdateTimeMap.put(entry.getKey(), entry.getValue() + 1);
- }
- fileNodeProcessorStore.setLastUpdateTimeMap(tempLastUpdateMap);
- }
- };
-
- private Action bufferwriteCloseAction = new Action() {
-
- @Override
- public void act() {
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setLastUpdateTimeMap(lastUpdateTimeMap);
- addLastTimeToIntervalFile();
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- }
- }
-
- private void addLastTimeToIntervalFile() {
-
- if (!newFileNodes.isEmpty()) {
- // end time with one start time
- Map<String, Long> endTimeMap = new HashMap<>();
- for (Entry<String, Long> startTime : currentTsFileResource.getStartTimeMap().entrySet()) {
- String deviceId = startTime.getKey();
- endTimeMap.put(deviceId, lastUpdateTimeMap.get(deviceId));
- }
- currentTsFileResource.setEndTimeMap(endTimeMap);
- }
- }
- };
- private Action overflowFlushAction = () -> {
-
- // update the new TsFileResource List and emptyIntervalFile.
- // Notice: thread safe
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setOverflowed(isOverflowed);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- }
- };
- // Token for query which used to
- private int multiPassLockToken = 0;
- private VersionController versionController;
- private ReentrantLock mergeDeleteLock = new ReentrantLock();
-
- /**
- * This is the modification file of the result of the current merge.
- */
- private ModificationFile mergingModification;
-
- private TsFileIOWriter mergeFileWriter = null;
- private String mergeOutputPath = null;
- private String mergeBaseDir = null;
- private String mergeFileName = null;
- private boolean mergeIsChunkGroupHasData = false;
- private long mergeStartPos;
-
- /**
- * constructor of FileNodeProcessor.
- */
- FileNodeProcessor(String fileNodeDirPath, String processorName)
- throws FileNodeProcessorException {
- super(processorName);
- for (StorageGroupProcessorStatConstants statConstant :
- StorageGroupProcessorStatConstants.values()) {
- statParamsHashMap.put(statConstant.name(), new AtomicLong(0));
- }
- statStorageDeltaName =
- MonitorConstants.STAT_STORAGE_GROUP_PREFIX + MonitorConstants.MONITOR_PATH_SEPARATOR
- + MonitorConstants.FILE_NODE_PATH + MonitorConstants.MONITOR_PATH_SEPARATOR
- + processorName.replaceAll("\\.", "_");
-
- this.parameters = new HashMap<>();
- String dirPath = fileNodeDirPath;
- if (dirPath.length() > 0
- && dirPath.charAt(dirPath.length() - 1) != File.separatorChar) {
- dirPath = dirPath + File.separatorChar;
- }
-
- File restoreFolder = new File(dirPath + processorName);
- if (!restoreFolder.exists()) {
- restoreFolder.mkdirs();
- LOGGER.info(
- "The restore directory of the filenode processor {} doesn't exist. Create new " +
- "directory {}",
- getProcessorName(), restoreFolder.getAbsolutePath());
- }
- fileNodeRestoreFilePath = new File(restoreFolder, processorName + RESTORE_FILE_SUFFIX)
- .getPath();
- try {
- fileNodeProcessorStore = readStoreFromDisk();
- } catch (FileNodeProcessorException e) {
- LOGGER.error(
- "The fileNode processor {} encountered an error when recoverying restore " +
- "information.", processorName);
- throw new FileNodeProcessorException(e);
- }
- // TODO deep clone the lastupdate time
- lastUpdateTimeMap = fileNodeProcessorStore.getLastUpdateTimeMap();
- emptyTsFileResource = fileNodeProcessorStore.getEmptyTsFileResource();
- newFileNodes = fileNodeProcessorStore.getNewFileNodes();
- isMerging = fileNodeProcessorStore.getFileNodeProcessorStatus();
- numOfMergeFile = fileNodeProcessorStore.getNumOfMergeFile();
- invertedIndexOfFiles = new HashMap<>();
- // deep clone
- flushLastUpdateTimeMap = new HashMap<>();
- for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- flushLastUpdateTimeMap.put(entry.getKey(), entry.getValue() + 1);
- }
- // construct the fileschema
- try {
- this.fileSchema = constructFileSchema(processorName);
- } catch (WriteProcessException e) {
- throw new FileNodeProcessorException(e);
- }
- // status is not NONE, or the last intervalFile is not closed
- if (isMerging != FileNodeProcessorStatus.NONE
- || (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed())) {
- shouldRecovery = true;
- } else {
- // add file into the index of file
- addAllFileIntoIndex(newFileNodes);
- }
- // RegistStatService
- if (TsFileDBConf.isEnableStatMonitor()) {
- StatMonitor statMonitor = StatMonitor.getInstance();
- registerStatMetadata();
- statMonitor.registerStatistics(statStorageDeltaName, this);
- }
- try {
- versionController = new SimpleFileVersionController(restoreFolder.getPath());
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- @Override
- public Map<String, AtomicLong> getStatParamsHashMap() {
- return statParamsHashMap;
- }
-
- @Override
- public void registerStatMetadata() {
- Map<String, String> hashMap = new HashMap<>();
- for (StorageGroupProcessorStatConstants statConstant :
- StorageGroupProcessorStatConstants.values()) {
- hashMap
- .put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
- MonitorConstants.DATA_TYPE_INT64);
- }
- StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- }
-
- @Override
- public List<String> getAllPathForStatistic() {
- List<String> list = new ArrayList<>();
- for (StorageGroupProcessorStatConstants statConstant :
- StorageGroupProcessorStatConstants.values()) {
- list.add(
- statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
- }
- return list;
- }
-
- @Override
- public Map<String, TSRecord> getAllStatisticsValue() {
- Long curTime = System.currentTimeMillis();
- HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
- TSRecord tsRecord = new TSRecord(curTime, statStorageDeltaName);
-
- Map<String, AtomicLong> hashMap = getStatParamsHashMap();
- tsRecord.dataPointList = new ArrayList<>();
- for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
- tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(), entry.getValue().get()));
- }
-
- tsRecordHashMap.put(statStorageDeltaName, tsRecord);
- return tsRecordHashMap;
- }
-
- /**
- * add interval FileNode.
- */
- void addIntervalFileNode(File file) throws ActionException, IOException {
-
- TsFileResource tsFileResource = new TsFileResource(file, false);
- this.currentTsFileResource = tsFileResource;
- newFileNodes.add(tsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- flushFileNodeProcessorAction.act();
- }
-
- /**
- * set interval filenode start time.
- *
- * @param deviceId device ID
- */
- void setIntervalFileNodeStartTime(String deviceId) {
- if (currentTsFileResource.getStartTime(deviceId) == -1) {
- currentTsFileResource.setStartTime(deviceId, flushLastUpdateTimeMap.get(deviceId));
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- }
- invertedIndexOfFiles.get(deviceId).add(currentTsFileResource);
- }
- }
-
- void setIntervalFileNodeStartTime(String deviceId, long time) {
- if (time != -1) {
- currentTsFileResource.setStartTime(deviceId, time);
- } else {
- currentTsFileResource.removeTime(deviceId);
- invertedIndexOfFiles.get(deviceId).remove(currentTsFileResource);
- }
- }
-
- long getIntervalFileNodeStartTime(String deviceId) {
- return currentTsFileResource.getStartTime(deviceId);
- }
-
- private void addAllFileIntoIndex(List<TsFileResource> fileList) {
- // clear map
- invertedIndexOfFiles.clear();
- // add all file to index
- for (TsFileResource fileNode : fileList) {
- if (fileNode.getStartTimeMap().isEmpty()) {
- continue;
- }
- for (String deviceId : fileNode.getStartTimeMap().keySet()) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- }
- invertedIndexOfFiles.get(deviceId).add(fileNode);
- }
- }
- }
-
- public boolean shouldRecovery() {
- return shouldRecovery;
- }
-
- public boolean isOverflowed() {
- return isOverflowed;
- }
-
- /**
- * if overflow insert, update and delete write into this filenode processor, set
- * <code>isOverflowed</code> to true.
- */
- public void setOverflowed(boolean isOverflowed) {
- if (this.isOverflowed != isOverflowed) {
- this.isOverflowed = isOverflowed;
- }
- }
-
- public FileNodeProcessorStatus getFileNodeProcessorStatus() {
- return isMerging;
- }
-
- /**
- * execute filenode recovery.
- */
- public void fileNodeRecovery() throws FileNodeProcessorException {
- // restore bufferwrite
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
- //
- // add the current file
- //
- currentTsFileResource = newFileNodes.get(newFileNodes.size() - 1);
-
- // this bufferwrite file is not close by normal operation
- String damagedFilePath = newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath();
- String[] fileNames = damagedFilePath.split("\\" + File.separator);
- // all information to recovery the damaged file.
- // contains file seriesPath, action parameters and processorName
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- parameters
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- String baseDir = directories
- .getTsFileFolder(newFileNodes.get(newFileNodes.size() - 1).getBaseDirIndex());
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "The filenode processor {} will recovery the bufferwrite processor, "
- + "the bufferwrite file is {}",
- getProcessorName(), fileNames[fileNames.length - 1]);
- }
-
- try {
- bufferWriteProcessor = new BufferWriteProcessor(baseDir, getProcessorName(),
- fileNames[fileNames.length - 1], parameters, versionController, fileSchema);
- } catch (BufferWriteProcessorException e) {
- LOGGER.error(
- "The filenode processor {} failed to recovery the bufferwrite processor, "
- + "the last bufferwrite file is {}.",
- getProcessorName(), fileNames[fileNames.length - 1]);
- throw new FileNodeProcessorException(e);
- }
- }
- // restore the overflow processor
- LOGGER.info("The filenode processor {} will recovery the overflow processor.",
- getProcessorName());
- try {
- overflowProcessor1 = new OverflowProcessor(getProcessorName(), overflowFlushAction,
- flushFileNodeProcessorAction, ()->{}, versionController, fileSchema);
- } catch (TsFileProcessorException | IOException e) {
- LOGGER.error("The filenode processor {} failed to recovery the overflow processor.",
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
-
- shouldRecovery = false;
-
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- // re-merge all file
- // if bufferwrite processor is not null, and close
- LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- getProcessorName(), isMerging);
- merge();
- } else if (isMerging == FileNodeProcessorStatus.WAITING) {
- // unlock
- LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- getProcessorName(), isMerging);
- //writeUnlock();
- switchWaitingToWorking();
- } else {
- //writeUnlock();
- }
- // add file into index of file
- addAllFileIntoIndex(newFileNodes);
- }
-
- /**
- * get buffer write processor by processor name and insert time.
- */
- public BufferWriteProcessor getBufferWriteProcessor(String processorName, long insertTime)
- throws FileNodeProcessorException {
- if (bufferWriteProcessor == null) {
- Map<String, Action> params = new HashMap<>();
- params.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- params.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- params
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- String baseDir = directories.getNextFolderForTsfile();
- LOGGER.info("Allocate folder {} for the new bufferwrite processor.", baseDir);
- // construct processor or restore
- try {
- bufferWriteProcessor = new BufferWriteProcessor(baseDir, processorName,
- insertTime + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
- + System.currentTimeMillis(),
- params, versionController, fileSchema);
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(String
- .format("The filenode processor %s failed to get the bufferwrite processor.",
- processorName), e);
- }
- } else if (bufferWriteProcessor.isClosed()) {
- try {
- bufferWriteProcessor.reopen(insertTime + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
- + System.currentTimeMillis());
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException("Cannot reopen BufferWriteProcessor", e);
- }
- }
- return bufferWriteProcessor;
- }
-
- /**
- * get overflow processor by processor name.
- */
- public OverflowProcessor getOverflowProcessor(String processorName)
- throws IOException, TsFileProcessorException {
- if (overflowProcessor1 == null) {
- Map<String, Action> params = new HashMap<>();
- // construct processor or restore
- params.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
- params
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- overflowProcessor1 = new OverflowProcessor(getProcessorName(), overflowFlushAction,
- flushFileNodeProcessorAction, ()->{}, versionController, fileSchema);
- } else if (overflowProcessor1.isClosed()) {
- overflowProcessor1.reopen();
- }
- return overflowProcessor1;
- }
-
- /**
- * get overflow processor.
- */
- public OverflowProcessor getOverflowProcessor() {
- if (overflowProcessor == null || overflowProcessor.isClosed()) {
- LOGGER.error("The overflow processor is null when getting the overflowProcessor");
- }
- return overflowProcessor;
- }
-
- public boolean hasOverflowProcessor() {
- return overflowProcessor != null && !overflowProcessor.isClosed();
- }
-
- public void setBufferwriteProcessroToClosed() {
-
- bufferWriteProcessor = null;
- }
-
- public boolean hasBufferwriteProcessor() {
-
- return bufferWriteProcessor != null;
- }
-
- /**
- * set last update time.
- */
- public void setLastUpdateTime(String deviceId, long timestamp) {
- if (!lastUpdateTimeMap.containsKey(deviceId) || lastUpdateTimeMap.get(deviceId) < timestamp) {
- lastUpdateTimeMap.put(deviceId, timestamp);
- }
- if (timestamp == -1) {
- lastUpdateTimeMap.remove(deviceId);
- }
- }
-
- /**
- * get last update time.
- */
- public long getLastUpdateTime(String deviceId) {
-
- if (lastUpdateTimeMap.containsKey(deviceId)) {
- return lastUpdateTimeMap.get(deviceId);
- } else {
- return -1;
- }
- }
-
- /**
- * get flush last update time.
- */
- public long getFlushLastUpdateTime(String deviceId) {
- if (!flushLastUpdateTimeMap.containsKey(deviceId)) {
- flushLastUpdateTimeMap.put(deviceId, 0L);
- }
- return flushLastUpdateTimeMap.get(deviceId);
- }
-
- public Map<String, Long> getLastUpdateTimeMap() {
- return lastUpdateTimeMap;
- }
-
- /**
- * For insert overflow.
- */
- public void changeTypeToChanged(String deviceId, long timestamp) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{},time:{}]",
- getProcessorName(), deviceId, timestamp);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- changeTypeToChanged(temp.get(index), deviceId);
- }
- }
-
- private void changeTypeToChanged(TsFileResource fileNode, String deviceId) {
- fileNode.changeTypeToChanged(isMerging);
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- fileNode.addMergeChanged(deviceId);
- }
- }
-
- /**
- * For update overflow.
- */
- public void changeTypeToChanged(String deviceId, long startTime, long endTime) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{}, start time:{}, end time:{}]",
- getProcessorName(), deviceId, startTime, endTime);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int left = searchIndexNodeByTimestamp(deviceId, startTime, temp);
- int right = searchIndexNodeByTimestamp(deviceId, endTime, temp);
- for (int i = left; i <= right; i++) {
- changeTypeToChanged(temp.get(i), deviceId);
- }
- }
- }
-
- /**
- * For delete overflow.
- */
- public void changeTypeToChangedForDelete(String deviceId, long timestamp) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{}, delete time:{}]",
- getProcessorName(), deviceId, timestamp);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- for (int i = 0; i <= index; i++) {
- temp.get(i).changeTypeToChanged(isMerging);
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- temp.get(i).addMergeChanged(deviceId);
- }
- }
- }
- }
-
- /**
- * Search the index of the interval by the timestamp.
- *
- * @return index of interval
- */
- private int searchIndexNodeByTimestamp(String deviceId, long timestamp,
- List<TsFileResource> fileList) {
- int index = 1;
- while (index < fileList.size()) {
- if (timestamp < fileList.get(index).getStartTime(deviceId)) {
- break;
- } else {
- index++;
- }
- }
- return index - 1;
- }
-
- /**
- * add multiple pass lock.
- */
- public int addMultiPassCount() {
- LOGGER.debug("Add MultiPassCount: read lock newMultiPassCount.");
- newMultiPassCount.incrementAndGet();
- while (newMultiPassTokenSet.contains(multiPassLockToken)) {
- multiPassLockToken++;
- }
- newMultiPassTokenSet.add(multiPassLockToken);
- LOGGER.debug("Add multi token:{}, nsPath:{}.", multiPassLockToken, getProcessorName());
- return multiPassLockToken;
- }
-
- /**
- * decrease multiple pass count. TODO: use the return value or remove it.
- */
- public boolean decreaseMultiPassCount(int token) throws FileNodeProcessorException {
- if (newMultiPassTokenSet.contains(token)) {
- int newMultiPassCountValue = newMultiPassCount.decrementAndGet();
- if (newMultiPassCountValue < 0) {
- throw new FileNodeProcessorException(String
- .format("Remove MultiPassCount error, newMultiPassCount:%d", newMultiPassCountValue));
- }
- newMultiPassTokenSet.remove(token);
- LOGGER.debug("Remove multi token:{}, nspath:{}, new set:{}, count:{}", token,
- getProcessorName(),
- newMultiPassTokenSet, newMultiPassCount);
- return true;
- } else if (oldMultiPassTokenSet != null && oldMultiPassTokenSet.contains(token)) {
- // remove token first, then unlock
- oldMultiPassTokenSet.remove(token);
- oldMultiPassCount.countDown();
- long oldMultiPassCountValue = oldMultiPassCount.getCount();
- if (oldMultiPassCountValue < 0) {
- throw new FileNodeProcessorException(String
- .format("Remove MultiPassCount error, oldMultiPassCount:%d", oldMultiPassCountValue));
- }
- LOGGER.debug("Remove multi token:{}, old set:{}, count:{}", token, oldMultiPassTokenSet,
- oldMultiPassCount.getCount());
- return true;
- } else {
- LOGGER.error("remove token error:{},new set:{}, old set:{}", token, newMultiPassTokenSet,
- oldMultiPassTokenSet);
- // should add throw exception
- return false;
- }
- }
-
- /**
- * query data.
- */
- public <T extends Comparable<T>> QueryDataSource query(String deviceId, String measurementId,
- QueryContext context) throws FileNodeProcessorException {
- // query overflow data
- MeasurementSchema mSchema;
- TSDataType dataType;
-
- //mSchema = mManager.getSchemaForOnePath(deviceId + "." + measurementId);
- mSchema = fileSchema.getMeasurementSchema(measurementId);
- dataType = mSchema.getType();
-
- OverflowSeriesDataSource overflowSeriesDataSource;
- try {
- overflowSeriesDataSource = overflowProcessor.query(deviceId, measurementId, dataType,
- mSchema.getProps(), context);
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- // tsfile dataØØ
- List<TsFileResource> bufferwriteDataInFiles = new ArrayList<>();
- for (TsFileResource tsFileResource : newFileNodes) {
- // add the same tsFileResource, but not the same reference
- if (tsFileResource.isClosed()) {
- bufferwriteDataInFiles.add(tsFileResource.backUp());
- }
- }
- Pair<ReadOnlyMemChunk, List<ChunkMetaData>> bufferwritedata = new Pair<>(null, null);
- // bufferwrite data
- UnsealedTsFile unsealedTsFile = null;
-
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()
- && !newFileNodes.get(newFileNodes.size() - 1).getStartTimeMap().isEmpty()) {
- unsealedTsFile = new UnsealedTsFile();
- unsealedTsFile.setFilePath(newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath());
- if (bufferWriteProcessor == null) {
- throw new FileNodeProcessorException(String.format(
- "The last of tsfile %s in filenode processor %s is not closed, "
- + "but the bufferwrite processor is null.",
- newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath(), getProcessorName()));
- }
- try {
- bufferwritedata = bufferWriteProcessor
- .queryBufferWriteData(deviceId, measurementId, dataType, mSchema.getProps());
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
-
- try {
- List<Modification> pathModifications = context.getPathModifications(
- currentTsFileResource.getModFile(), deviceId
- + IoTDBConstant.PATH_SEPARATOR + measurementId
- );
- if (!pathModifications.isEmpty()) {
- QueryUtils.modifyChunkMetaData(bufferwritedata.right, pathModifications);
- }
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
-
- unsealedTsFile.setTimeSeriesChunkMetaDatas(bufferwritedata.right);
- }
- SeriesDataSource seriesDataSource = new SeriesDataSource(
- new Path(deviceId + "." + measurementId), bufferwriteDataInFiles, unsealedTsFile,
- bufferwritedata.left);
- return new QueryDataSource(seriesDataSource, overflowSeriesDataSource);
-
- }
-
- /**
- * append one specified tsfile to this filenode processor.
- *
- * @param appendFile the appended tsfile information
- * @param appendFilePath the seriesPath of appended file
- */
- public void appendFile(TsFileResource appendFile, String appendFilePath)
- throws FileNodeProcessorException {
- try {
- if (!appendFile.getFile().getParentFile().exists()) {
- appendFile.getFile().getParentFile().mkdirs();
- }
- // move file
- File originFile = new File(appendFilePath);
- File targetFile = appendFile.getFile();
- if (!originFile.exists()) {
- throw new FileNodeProcessorException(
- String.format("The appended file %s does not exist.", appendFilePath));
- }
- if (targetFile.exists()) {
- throw new FileNodeProcessorException(
- String.format("The appended target file %s already exists.",
- appendFile.getFile().getAbsolutePath()));
- }
- if (!originFile.renameTo(targetFile)) {
- LOGGER.warn("File renaming failed when appending new file. Origin: {}, Target: {}",
- originFile.getPath(), targetFile.getPath());
- }
- // append the new tsfile
- this.newFileNodes.add(appendFile);
- // update the lastUpdateTime
- for (Entry<String, Long> entry : appendFile.getEndTimeMap().entrySet()) {
- lastUpdateTimeMap.put(entry.getKey(), entry.getValue());
- }
- bufferwriteFlushAction.act();
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // reconstruct the inverted index of the newFileNodes
- flushFileNodeProcessorAction.act();
- addAllFileIntoIndex(newFileNodes);
- } catch (Exception e) {
- LOGGER.error("Failed to append the tsfile {} to filenode processor {}.", appendFile,
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
- }
-
- /**
- * get overlap tsfiles which are conflict with the appendFile.
- *
- * @param appendFile the appended tsfile information
- */
- public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
- throws FileNodeProcessorException {
- List<String> overlapFiles = new ArrayList<>();
- try {
- for (TsFileResource tsFileResource : newFileNodes) {
- getOverlapFiles(appendFile, tsFileResource, uuid, overlapFiles);
- }
- } catch (IOException e) {
- LOGGER.error("Failed to get overlap tsfiles which conflict with the appendFile.");
- throw new FileNodeProcessorException(e);
- }
- return overlapFiles;
- }
-
- private void getOverlapFiles(TsFileResource appendFile, TsFileResource tsFileResource,
- String uuid, List<String> overlapFiles) throws IOException {
- for (Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- if (tsFileResource.getStartTimeMap().containsKey(entry.getKey()) &&
- tsFileResource.getEndTime(entry.getKey()) >= entry.getValue()
- && tsFileResource.getStartTime(entry.getKey()) <= appendFile
- .getEndTime(entry.getKey())) {
- String relativeFilePath =
- Constans.SYNC_SERVER + File.separatorChar + uuid + File.separatorChar
- + Constans.BACK_UP_DIRECTORY_NAME
- + File.separatorChar + tsFileResource.getRelativePath();
- File newFile = new File(
- Directories.getInstance().getTsFileFolder(tsFileResource.getBaseDirIndex()),
- relativeFilePath);
- if (!newFile.getParentFile().exists()) {
- newFile.getParentFile().mkdirs();
- }
- java.nio.file.Path link = FileSystems.getDefault().getPath(newFile.getPath());
- java.nio.file.Path target = FileSystems.getDefault()
- .getPath(tsFileResource.getFile().getAbsolutePath());
- Files.createLink(link, target);
- overlapFiles.add(newFile.getPath());
- break;
- }
- }
- }
-
- /**
- * add time series.
- */
- public void addTimeSeries(String measurementId, TSDataType dataType, TSEncoding encoding,
- CompressionType compressor, Map<String, String> props) {
- fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
- compressor, props));
- }
-
- /**
- * submit the merge task to the <code>MergePool</code>.
- *
- * @return null -can't submit the merge task, because this filenode is not overflowed or it is
- * merging now. Future - submit the merge task successfully.
- */
- Future submitToMerge() {
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- if (lastMergeTime > 0) {
- long thisMergeTime = System.currentTimeMillis();
- long mergeTimeInterval = thisMergeTime - lastMergeTime;
- ZonedDateTime lastDateTime = ofInstant(Instant.ofEpochMilli(lastMergeTime),
- zoneId);
- ZonedDateTime thisDateTime = ofInstant(Instant.ofEpochMilli(thisMergeTime),
- zoneId);
- LOGGER.info(
- "The filenode {} last merge time is {}, this merge time is {}, "
- + "merge time interval is {}s",
- getProcessorName(), lastDateTime, thisDateTime, mergeTimeInterval / 1000);
- }
- lastMergeTime = System.currentTimeMillis();
-
- if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- if (overflowProcessor.getFileSize() < IoTDBDescriptor.getInstance()
- .getConfig().getOverflowFileSizeThreshold()) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "Skip this merge taks submission, because the size{} of overflow processor {} "
- + "does not reaches the threshold {}.",
- MemUtils.bytesCntToStr(overflowProcessor.getFileSize()), getProcessorName(),
- MemUtils.bytesCntToStr(
- IoTDBDescriptor.getInstance().getConfig().getOverflowFileSizeThreshold()));
- }
- return null;
- }
- } else {
- LOGGER.info(
- "Skip this merge taks submission, because the filenode processor {} "
- + "has no overflow processor.",
- getProcessorName());
- return null;
- }
- if (isOverflowed && isMerging == FileNodeProcessorStatus.NONE) {
- Runnable mergeThread;
- mergeThread = new MergeRunnale();
- LOGGER.info("Submit the merge task, the merge filenode is {}", getProcessorName());
- return MergeManager.getInstance().submit(mergeThread);
- } else {
- if (!isOverflowed) {
- LOGGER.info(
- "Skip this merge taks submission, because the filenode processor {} is not " +
- "overflowed.",
- getProcessorName());
- } else {
- LOGGER.warn(
- "Skip this merge task submission, because last merge task is not over yet, "
- + "the merge filenode processor is {}",
- getProcessorName());
- }
- }
- return null;
- }
-
- /**
- * Prepare for merge, close the bufferwrite and overflow.
- */
- private void prepareForMerge() {
- try {
- LOGGER.info("The filenode processor {} prepares for merge, closes the bufferwrite processor",
- getProcessorName());
- closeBufferWrite();
- // try to get overflow processor
- getOverflowProcessor(getProcessorName());
- // must close the overflow processor
- while (!getOverflowProcessor().canBeClosed()) {
- waitForClosing();
- }
- LOGGER.info("The filenode processor {} prepares for merge, closes the overflow processor",
- getProcessorName());
- getOverflowProcessor().close();
- } catch (FileNodeProcessorException | OverflowProcessorException | IOException e) {
- LOGGER.error("The filenode processor {} prepares for merge error.", getProcessorName());
- writeUnlock();
- throw new ErrorDebugException(e);
- }
- }
-
- private void waitForClosing() {
- try {
- LOGGER.info(
- "The filenode processor {} prepares for merge, the overflow {} can't be closed, "
- + "wait 100ms,",
- getProcessorName(), getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Merge this storage group, merge the tsfile data with overflow data.
- */
- public void merge() throws FileNodeProcessorException {
- // close bufferwrite and overflow, prepare for merge
- LOGGER.info("The filenode processor {} begins to merge.", getProcessorName());
- prepareForMerge();
- // change status from overflowed to no overflowed
- isOverflowed = false;
- // change status from work to merge
- isMerging = FileNodeProcessorStatus.MERGING_WRITE;
- // check the empty file
- Map<String, Long> startTimeMap = emptyTsFileResource.getStartTimeMap();
- mergeCheckEmptyFile(startTimeMap);
-
- for (TsFileResource tsFileResource : newFileNodes) {
- if (tsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- tsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
-
- addAllFileIntoIndex(newFileNodes);
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setOverflowed(isOverflowed);
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // flush this filenode information
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("The filenode processor {} writes restore information error when merging.",
- getProcessorName(), e);
- writeUnlock();
- throw new FileNodeProcessorException(e);
- }
- }
- // add numOfMergeFile to control the number of the merge file
- List<TsFileResource> backupIntervalFiles;
-
- backupIntervalFiles = switchFileNodeToMerge();
- //
- // clear empty file
- //
- boolean needEmtpy = false;
- if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- needEmtpy = true;
- }
- emptyTsFileResource.clear();
- // attention
- try {
- if (overflowProcessor.isClosed()) {
- overflowProcessor.reopen();
- }
- overflowProcessor.switchWorkToMerge();
- } catch (IOException e) {
- LOGGER.error("The filenode processor {} can't switch overflow processor from work to merge.",
- getProcessorName(), e);
- writeUnlock();
- throw new FileNodeProcessorException(e);
- }
- LOGGER.info("The filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.NONE, FileNodeProcessorStatus.MERGING_WRITE);
- writeUnlock();
-
- // query tsfile data and overflow data, and merge them
- int numOfMergeFiles = 0;
- int allNeedMergeFiles = backupIntervalFiles.size();
- for (TsFileResource backupIntervalFile : backupIntervalFiles) {
- numOfMergeFiles++;
- if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.CHANGED) {
- // query data and merge
- String filePathBeforeMerge = backupIntervalFile.getRelativePath();
- try {
- LOGGER.info(
- "The filenode processor {} begins merging the {}/{} tsfile[{}] with "
- + "overflow file, the process is {}%",
- getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- (int) (((numOfMergeFiles - 1) / (float) allNeedMergeFiles) * 100));
- long startTime = System.currentTimeMillis();
- String newFile = queryAndWriteDataForMerge(backupIntervalFile);
- long endTime = System.currentTimeMillis();
- long timeConsume = endTime - startTime;
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- LOGGER.info(
- "The fileNode processor {} has merged the {}/{} tsfile[{}->{}] over, "
- + "start time of merge is {}, end time of merge is {}, "
- + "time consumption is {}ms,"
- + " the process is {}%",
- getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- newFile, ofInstant(Instant.ofEpochMilli(startTime),
- zoneId), ofInstant(Instant.ofEpochMilli(endTime), zoneId), timeConsume,
- numOfMergeFiles / (float) allNeedMergeFiles * 100);
- } catch (IOException | PathErrorException e) {
- LOGGER.error("Merge: query and write data error.", e);
- throw new FileNodeProcessorException(e);
- }
- } else if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- LOGGER.error("The overflowChangeType of backupIntervalFile must not be {}",
- OverflowChangeType.MERGING_CHANGE);
- // handle this error, throw one runtime exception
- throw new FileNodeProcessorException(
- "The overflowChangeType of backupIntervalFile must not be "
- + OverflowChangeType.MERGING_CHANGE);
- } else {
- LOGGER.debug(
- "The filenode processor {} is merging, the interval file {} doesn't "
- + "need to be merged.",
- getProcessorName(), backupIntervalFile.getRelativePath());
- }
- }
-
- // change status from merge to wait
- switchMergeToWaiting(backupIntervalFiles, needEmtpy);
-
- // change status from wait to work
- switchWaitingToWorking();
- }
-
- private void mergeCheckEmptyFile(Map<String, Long> startTimeMap) {
- if (emptyTsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- return;
- }
- Iterator<Entry<String, Long>> iterator = emptyTsFileResource.getEndTimeMap().entrySet()
- .iterator();
- while (iterator.hasNext()) {
- Entry<String, Long> entry = iterator.next();
- String deviceId = entry.getKey();
- if (invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.get(deviceId).get(0).setOverflowChangeType(OverflowChangeType.CHANGED);
- startTimeMap.remove(deviceId);
- iterator.remove();
- }
- }
- if (emptyTsFileResource.checkEmpty()) {
- emptyTsFileResource.clear();
- } else {
- if (!newFileNodes.isEmpty()) {
- TsFileResource first = newFileNodes.get(0);
- for (String deviceId : emptyTsFileResource.getStartTimeMap().keySet()) {
- first.setStartTime(deviceId, emptyTsFileResource.getStartTime(deviceId));
- first.setEndTime(deviceId, emptyTsFileResource.getEndTime(deviceId));
- first.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- emptyTsFileResource.clear();
- } else {
- emptyTsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
- }
-
- private List<TsFileResource> switchFileNodeToMerge() throws FileNodeProcessorException {
- List<TsFileResource> result = new ArrayList<>();
- if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // add empty
- result.add(emptyTsFileResource.backUp());
- if (!newFileNodes.isEmpty()) {
- throw new FileNodeProcessorException(
- String.format("The status of empty file is %s, but the new file list is not empty",
- emptyTsFileResource.getOverflowChangeType()));
- }
- return result;
- }
- if (newFileNodes.isEmpty()) {
- LOGGER.error("No file was changed when merging, the filenode is {}", getProcessorName());
- throw new FileNodeProcessorException(
- "No file was changed when merging, the filenode is " + getProcessorName());
- }
- for (TsFileResource tsFileResource : newFileNodes) {
- updateFileNode(tsFileResource, result);
- }
- return result;
- }
-
- private void updateFileNode(TsFileResource tsFileResource, List<TsFileResource> result) {
- if (tsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- result.add(tsFileResource.backUp());
- } else {
- Map<String, Long> startTimeMap = new HashMap<>();
- Map<String, Long> endTimeMap = new HashMap<>();
- for (String deviceId : tsFileResource.getEndTimeMap().keySet()) {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = temp.indexOf(tsFileResource);
- int size = temp.size();
- // start time
- if (index == 0) {
- startTimeMap.put(deviceId, 0L);
- } else {
- startTimeMap.put(deviceId, tsFileResource.getStartTime(deviceId));
- }
- // end time
- if (index < size - 1) {
- endTimeMap.put(deviceId, temp.get(index + 1).getStartTime(deviceId) - 1);
- } else {
- endTimeMap.put(deviceId, tsFileResource.getEndTime(deviceId));
- }
- }
- TsFileResource node = new TsFileResource(startTimeMap, endTimeMap, tsFileResource.getFile());
- result.add(node);
- }
- }
-
- private void switchMergeToWaiting(List<TsFileResource> backupIntervalFiles, boolean needEmpty)
- throws FileNodeProcessorException {
- LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.MERGING_WRITE, FileNodeProcessorStatus.WAITING);
- writeLock();
- try {
- oldMultiPassTokenSet = newMultiPassTokenSet;
- oldMultiPassCount = new CountDownLatch(newMultiPassCount.get());
- newMultiPassTokenSet = new HashSet<>();
- newMultiPassCount = new AtomicInteger(0);
- List<TsFileResource> result = new ArrayList<>();
- int beginIndex = 0;
- if (needEmpty) {
- TsFileResource empty = backupIntervalFiles.get(0);
- if (!empty.checkEmpty()) {
- updateEmpty(empty, result);
- beginIndex++;
- }
- }
- // reconstruct the file index
- addAllFileIntoIndex(backupIntervalFiles);
- // check the merge changed file
- for (int i = beginIndex; i < backupIntervalFiles.size(); i++) {
- TsFileResource newFile = newFileNodes.get(i - beginIndex);
- TsFileResource temp = backupIntervalFiles.get(i);
- if (newFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- updateMergeChanged(newFile, temp);
- }
- if (!temp.checkEmpty()) {
- result.add(temp);
- }
- }
- // add new file when merge
- for (int i = backupIntervalFiles.size() - beginIndex; i < newFileNodes.size(); i++) {
- TsFileResource fileNode = newFileNodes.get(i);
- if (fileNode.isClosed()) {
- result.add(fileNode.backUp());
- } else {
- result.add(fileNode);
- }
- }
-
- isMerging = FileNodeProcessorStatus.WAITING;
- newFileNodes = result;
- // reconstruct the index
- addAllFileIntoIndex(newFileNodes);
- // clear merge changed
- for (TsFileResource fileNode : newFileNodes) {
- fileNode.clearMergeChanged();
- }
-
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- LOGGER.error(
- "Merge: failed to write filenode information to revocery file, the filenode is " +
- "{}.",
- getProcessorName(), e);
- throw new FileNodeProcessorException(
- "Merge: write filenode information to revocery file failed, the filenode is "
- + getProcessorName());
- }
- }
- } finally {
- writeUnlock();
- }
- }
-
- private void updateEmpty(TsFileResource empty, List<TsFileResource> result) {
- for (String deviceId : empty.getStartTimeMap().keySet()) {
- if (invertedIndexOfFiles.containsKey(deviceId)) {
- TsFileResource temp = invertedIndexOfFiles.get(deviceId).get(0);
- if (temp.getMergeChanged().contains(deviceId)) {
- empty.setOverflowChangeType(OverflowChangeType.CHANGED);
- break;
- }
- }
- }
- empty.clearMergeChanged();
- result.add(empty.backUp());
- }
-
- private void updateMergeChanged(TsFileResource newFile, TsFileResource temp) {
- for (String deviceId : newFile.getMergeChanged()) {
- if (temp.getStartTimeMap().containsKey(deviceId)) {
- temp.setOverflowChangeType(OverflowChangeType.CHANGED);
- } else {
- changeTypeToChanged(deviceId, newFile.getStartTime(deviceId),
- newFile.getEndTime(deviceId));
- }
- }
- }
-
-
- private void switchWaitingToWorking()
- throws FileNodeProcessorException {
-
- LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.WAITING, FileNodeProcessorStatus.NONE);
-
- if (oldMultiPassCount != null) {
- LOGGER.info("The old Multiple Pass Token set is {}, the old Multiple Pass Count is {}",
- oldMultiPassTokenSet,
- oldMultiPassCount);
- try {
- oldMultiPassCount.await();
- } catch (InterruptedException e) {
- LOGGER.info(
- "The filenode processor {} encountered an error when it waits for all old queries over.",
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
- }
-
- try {
- writeLock();
- try {
- // delete the all files which are in the newFileNodes
- // notice: the last restore file of the interval file
-
- List<String> bufferwriteDirPathList = directories.getAllTsFileFolders();
- List<File> bufferwriteDirList = new ArrayList<>();
- collectBufferWriteDirs(bufferwriteDirPathList, bufferwriteDirList);
-
- Set<String> bufferFiles = new HashSet<>();
- collectBufferWriteFiles(bufferFiles);
-
- // add the restore file, if the last file is not closed
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
- String bufferFileRestorePath =
- newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath() + RESTORE_FILE_SUFFIX;
- bufferFiles.add(bufferFileRestorePath);
- }
-
- deleteBufferWriteFiles(bufferwriteDirList, bufferFiles);
-
- // merge switch
- changeFileNodes();
-
- // overflow switch from merge to work
- overflowProcessor.switchMergeToWork();
-
- // write status to file
- isMerging = FileNodeProcessorStatus.NONE;
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- writeStoreToDisk(fileNodeProcessorStore);
- }
- } catch (IOException e) {
- LOGGER.info(
- "The filenode processor {} encountered an error when its "
- + "status switched from {} to {}.",
- getProcessorName(), FileNodeProcessorStatus.NONE,
- FileNodeProcessorStatus.MERGING_WRITE);
- throw new FileNodeProcessorException(e);
- } finally {
- writeUnlock();
- }
- } finally {
- oldMultiPassTokenSet = null;
- oldMultiPassCount = null;
- }
-
- }
-
- private void collectBufferWriteDirs(List<String> bufferwriteDirPathList,
- List<File> bufferwriteDirList) {
- for (String bufferwriteDirPath : bufferwriteDirPathList) {
- if (bufferwriteDirPath.length() > 0
- && bufferwriteDirPath.charAt(bufferwriteDirPath.length() - 1)
- != File.separatorChar) {
- bufferwriteDirPath = bufferwriteDirPath + File.separatorChar;
- }
- bufferwriteDirPath = bufferwriteDirPath + getProcessorName();
- File bufferwriteDir = new File(bufferwriteDirPath);
- bufferwriteDirList.add(bufferwriteDir);
- if (!bufferwriteDir.exists()) {
- bufferwriteDir.mkdirs();
- }
- }
- }
-
- private void collectBufferWriteFiles(Set<String> bufferFiles) {
- for (TsFileResource bufferFileNode : newFileNodes) {
- String bufferFilePath = bufferFileNode.getFile().getAbsolutePath();
- if (bufferFilePath != null) {
- bufferFiles.add(bufferFilePath);
- }
- }
- }
-
- private void deleteBufferWriteFiles(List<File> bufferwriteDirList, Set<String> bufferFiles)
- throws IOException {
- for (File bufferwriteDir : bufferwriteDirList) {
- File[] files = bufferwriteDir.listFiles();
- if (files == null) {
- continue;
- }
- for (File file : files) {
- if (!bufferFiles.contains(file.getPath())) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(file.getPath());
- if (!file.delete()) {
- LOGGER.warn("Cannot delete BufferWrite file {}", file.getPath());
- }
- }
- }
- }
- }
-
- private void changeFileNodes() {
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- fileNode.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
- }
-
- private String queryAndWriteDataForMerge(TsFileResource backupIntervalFile)
- throws IOException, FileNodeProcessorException, PathErrorException {
- Map<String, Long> startTimeMap = new HashMap<>();
- Map<String, Long> endTimeMap = new HashMap<>();
-
- mergeFileWriter = null;
- mergeOutputPath = null;
- mergeBaseDir = null;
- mergeFileName = null;
- // modifications are blocked before mergeModification is created to avoid
- // losing some modification.
- mergeDeleteLock.lock();
- QueryContext context = new QueryContext();
- try {
- FileReaderManager.getInstance().increaseFileReaderReference(backupIntervalFile.getFilePath(),
- true);
- for (String deviceId : backupIntervalFile.getStartTimeMap().keySet()) {
- // query one deviceId
- List<Path> pathList = new ArrayList<>();
- mergeIsChunkGroupHasData = false;
- mergeStartPos = -1;
- ChunkGroupFooter footer;
- int numOfChunk = 0;
- try {
- List<String> pathStrings = mManager.getLeafNodePathInNextLevel(deviceId);
- for (String string : pathStrings) {
- pathList.add(new Path(string));
- }
- } catch (PathErrorException e) {
- LOGGER.error("Can't get all the paths from MManager, the deviceId is {}", deviceId);
- throw new FileNodeProcessorException(e);
- }
- if (pathList.isEmpty()) {
- continue;
- }
- for (Path path : pathList) {
- // query one measurement in the special deviceId
- String measurementId = path.getMeasurement();
- TSDataType dataType = mManager.getSeriesType(path.getFullPath());
- OverflowSeriesDataSource overflowSeriesDataSource = overflowProcessor.queryMerge(deviceId,
- measurementId, dataType, true, context);
- Filter timeFilter = FilterFactory
- .and(TimeFilter.gtEq(backupIntervalFile.getStartTime(deviceId)),
- TimeFilter.ltEq(backupIntervalFile.getEndTime(deviceId)));
- SingleSeriesExpression seriesFilter = new SingleSeriesExpression(path, timeFilter);
-
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
- FileReaderManager.getInstance()
- .increaseFileReaderReference(overflowInsertFile.getFilePath(),
- false);
- }
-
- IReader seriesReader = SeriesReaderFactory.getInstance()
- .createSeriesReaderForMerge(backupIntervalFile,
- overflowSeriesDataSource, seriesFilter, context);
- numOfChunk += queryAndWriteSeries(seriesReader, path, seriesFilter, dataType,
- startTimeMap, endTimeMap, overflowSeriesDataSource);
- }
- if (mergeIsChunkGroupHasData) {
- // end the new rowGroupMetadata
- long size = mergeFileWriter.getPos() - mergeStartPos;
- footer = new ChunkGroupFooter(deviceId, size, numOfChunk);
- mergeFileWriter.endChunkGroup(footer, 0);
- }
- }
- } finally {
- FileReaderManager.getInstance().decreaseFileReaderReference(backupIntervalFile.getFilePath(),
- true);
-
- if (mergeDeleteLock.isLocked()) {
- mergeDeleteLock.unlock();
- }
- }
-
- if (mergeFileWriter != null) {
- mergeFileWriter.endFile(fileSchema);
- }
- backupIntervalFile.setFile(new File(mergeBaseDir + File.separator + mergeFileName));
- backupIntervalFile.setOverflowChangeType(OverflowChangeType.NO_CHANGE);
- backupIntervalFile.setStartTimeMap(startTimeMap);
- backupIntervalFile.setEndTimeMap(endTimeMap);
- backupIntervalFile.setModFile(mergingModification);
- mergingModification = null;
- return mergeFileName;
- }
-
- private int queryAndWriteSeries(IReader seriesReader, Path path,
- SingleSeriesExpression seriesFilter, TSDataType dataType,
- Map<String, Long> startTimeMap, Map<String, Long> endTimeMap,
- OverflowSeriesDataSource overflowSeriesDataSource)
- throws IOException {
- int numOfChunk = 0;
- try {
- if (!seriesReader.hasNext()) {
- LOGGER.debug(
- "The time-series {} has no data with the filter {} in the filenode processor {}",
- path, seriesFilter, getProcessorName());
- } else {
- numOfChunk++;
- TimeValuePair timeValuePair = seriesReader.next();
- if (mergeFileWriter == null) {
- mergeBaseDir = directories.getNextFolderForTsfile();
- mergeFileName = timeValuePair.getTimestamp()
- + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR + System.currentTimeMillis();
- mergeOutputPath = constructOutputFilePath(mergeBaseDir, getProcessorName(),
- mergeFileName);
- mergeFileName = getProcessorName() + File.separatorChar + mergeFileName;
- mergeFileWriter = new TsFileIOWriter(new File(mergeOutputPath));
- mergingModification = new ModificationFile(mergeOutputPath
- + ModificationFile.FILE_SUFFIX);
- mergeDeleteLock.unlock();
- }
- if (!mergeIsChunkGroupHasData) {
- // start a new rowGroupMetadata
- mergeIsChunkGroupHasData = true;
- // the datasize and numOfChunk is fake
- // the accurate datasize and numOfChunk will get after write all this device data.
- mergeFileWriter.startFlushChunkGroup(path.getDevice());// TODO please check me.
- mergeStartPos = mergeFileWriter.getPos();
- }
- // init the serieswWriteImpl
- MeasurementSchema measurementSchema = fileSchema
- .getMeasurementSchema(path.getMeasurement());
- ChunkBuffer pageWriter = new ChunkBuffer(measurementSchema);
- int pageSizeThreshold = TSFileConfig.pageSizeInByte;
- ChunkWriterImpl seriesWriterImpl = new ChunkWriterImpl(measurementSchema, pageWriter,
- pageSizeThreshold);
- // write the series data
- writeOneSeries(path.getDevice(), seriesWriterImpl, dataType,
- seriesReader,
- startTimeMap, endTimeMap, timeValuePair);
- // flush the series data
- seriesWriterImpl.writeToFileWriter(mergeFileWriter);
- }
- } finally {
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
- FileReaderManager.getInstance()
- .decreaseFileReaderReference(overflowInsertFile.getFilePath(),
- false);
- }
- }
- return numOfChunk;
- }
-
-
- private void writeOneSeries(String deviceId, ChunkWriterImpl seriesWriterImpl,
- TSDataType dataType, IReader seriesReader, Map<String, Long> startTimeMap,
- Map<String, Long> endTimeMap, TimeValuePair firstTVPair) throws IOException {
- long startTime;
- long endTime;
- TimeValuePair localTV = firstTVPair;
- writeTVPair(seriesWriterImpl, dataType, localTV);
- startTime = endTime = localTV.getTimestamp();
- if (!startTimeMap.containsKey(deviceId) || startTimeMap.get(deviceId) > startTime) {
- startTimeMap.put(deviceId, startTime);
- }
- if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- endTimeMap.put(deviceId, endTime);
- }
- while (seriesReader.hasNext()) {
- localTV = seriesReader.next();
- endTime = localTV.getTimestamp();
- writeTVPair(seriesWriterImpl, dataType, localTV);
- }
- if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- endTimeMap.put(deviceId, endTime);
- }
- }
-
- private void writeTVPair(ChunkWriterImpl seriesWriterImpl, TSDataType dataType,
- TimeValuePair timeValuePair) throws IOException {
- switch (dataType) {
- case BOOLEAN:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
- break;
- case INT32:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
- break;
- case INT64:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
- break;
- case FLOAT:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- break;
- case DOUBLE:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
- break;
- case TEXT:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
- break;
- default:
- LOGGER.error("Not support data type: {}", dataType);
- break;
- }
- }
-
-
- private String constructOutputFilePath(String baseDir, String processorName, String fileName) {
-
- String localBaseDir = baseDir;
- if (localBaseDir.charAt(localBaseDir.length() - 1) != File.separatorChar) {
- localBaseDir = localBaseDir + File.separatorChar + processorName;
- }
- File dataDir = new File(localBaseDir);
- if (!dataDir.exists()) {
- LOGGER.warn("The bufferwrite processor data dir doesn't exists, create new directory {}",
- localBaseDir);
- dataDir.mkdirs();
- }
- File outputFile = new File(dataDir, fileName);
- return outputFile.getPath();
- }
-
- private FileSchema constructFileSchema(String processorName) throws WriteProcessException {
-
- List<MeasurementSchema> columnSchemaList;
- columnSchemaList = mManager.getSchemaForFileName(processorName);
-
- FileSchema schema = new FileSchema();
- for (MeasurementSchema measurementSchema : columnSchemaList) {
- schema.registerMeasurement(measurementSchema);
- }
- return schema;
-
- }
-
- @Override
- public boolean canBeClosed() {
- if (isMerging != FileNodeProcessorStatus.NONE) {
- LOGGER.info("The filenode {} can't be closed, because the filenode status is {}",
- getProcessorName(),
- isMerging);
- return false;
- }
- if (newMultiPassCount.get() != 0) {
- LOGGER.warn("The filenode {} can't be closed, because newMultiPassCount is {}. The newMultiPassTokenSet is {}",
- getProcessorName(), newMultiPassCount, newMultiPassTokenSet);
- return false;
- }
-
- if (oldMultiPassCount == null) {
- return true;
- }
- if (oldMultiPassCount.getCount() == 0) {
- return true;
- } else {
- LOGGER.info("The filenode {} can't be closed, because oldMultiPassCount is {}",
- getProcessorName(), oldMultiPassCount.getCount());
- return false;
- }
- }
-
- @Override
- public FileNodeFlushFuture flush() throws IOException {
- Future<Boolean> bufferWriteFlushFuture = null;
- Future<Boolean> overflowFlushFuture = null;
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- bufferWriteFlushFuture = bufferWriteProcessor.flush();
- }
- if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- overflowFlushFuture = overflowProcessor.flush();
- }
- return new FileNodeFlushFuture(bufferWriteFlushFuture, overflowFlushFuture);
- }
-
- /**
- * Close the bufferwrite processor.
- */
- public void closeBufferWrite() throws FileNodeProcessorException {
- if (bufferWriteProcessor == null || bufferWriteProcessor.isClosed()) {
- return;
- }
- try {
- while (!bufferWriteProcessor.canBeClosed()) {
- waitForBufferWriteClose();
- }
- bufferWriteProcessor.close();
- bufferWriteProcessor = null;
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- private void waitForBufferWriteClose() {
- try {
- LOGGER.info("The bufferwrite {} can't be closed, wait 100ms",
- bufferWriteProcessor.getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption", e);
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Close the overflow processor.
- */
- public void closeOverflow() throws FileNodeProcessorException {
- if (overflowProcessor == null || overflowProcessor.isClosed()) {
- return;
- }
- try {
- while (!overflowProcessor.canBeClosed()) {
- waitForOverflowClose();
- }
- overflowProcessor.close();
- } catch (OverflowProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- private void waitForOverflowClose() {
- try {
- LOGGER.info("The overflow {} can't be closed, wait 100ms",
- overflowProcessor.getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption", e);
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void close() throws FileNodeProcessorException {
- closeBufferWrite();
- closeOverflow();
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getModFile() != null) {
- try {
- fileNode.getModFile().close();
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
- }
-
- /**
- * deregister the filenode processor.
- */
- public void delete() throws ProcessorException {
- if (TsFileDBConf.isEnableStatMonitor()) {
- // remove the monitor
- LOGGER.info("Deregister the filenode processor: {} from monitor.", getProcessorName());
- StatMonitor.getInstance().deregisterStatistics(statStorageDeltaName);
- }
- closeBufferWrite();
- closeOverflow();
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getModFile() != null) {
- try {
- fileNode.getModFile().close();
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
- }
-
- @Override
- public long memoryUsage() {
- long memSize = 0;
- if (bufferWriteProcessor != null) {
- memSize += bufferWriteProcessor.memoryUsage();
- }
- if (overflowProcessor != null) {
- memSize += overflowProcessor.memoryUsage();
- }
- return memSize;
- }
-
- private void writeStoreToDisk(FileNodeProcessorStore fileNodeProcessorStore)
- throws FileNodeProcessorException {
-
- synchronized (fileNodeRestoreLock) {
- try (FileOutputStream fileOutputStream = new FileOutputStream(fileNodeRestoreFilePath)) {
- fileNodeProcessorStore.serialize(fileOutputStream);
- LOGGER.debug("The filenode processor {} writes restore information to the restore file",
- getProcessorName());
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
-
- private FileNodeProcessorStore readStoreFromDisk() throws FileNodeProcessorException {
-
- synchronized (fileNodeRestoreLock) {
- File restoreFile = new File(fileNodeRestoreFilePath);
- if (!restoreFile.exists() || restoreFile.length() == 0) {
- try {
- return new FileNodeProcessorStore(false, new HashMap<>(),
- new TsFileResource(null, false),
- new ArrayList<>(), FileNodeProcessorStatus.NONE, 0);
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- try (FileInputStream inputStream = new FileInputStream(fileNodeRestoreFilePath)) {
- return FileNodeProcessorStore.deSerialize(inputStream);
- } catch (IOException e) {
- LOGGER
- .error("Failed to deserialize the FileNodeRestoreFile {}, {}", fileNodeRestoreFilePath,
- e);
- throw new FileNodeProcessorException(e);
- }
- }
- }
-
- String getFileNodeRestoreFilePath() {
- return fileNodeRestoreFilePath;
- }
-
- /**
- * Delete data whose timestamp <= 'timestamp' and belong to timeseries deviceId.measurementId.
- *
- * @param deviceId the deviceId of the timeseries to be deleted.
- * @param measurementId the measurementId of the timeseries to be deleted.
- * @param timestamp the delete range is (0, timestamp].
- */
- public void delete(String deviceId, String measurementId, long timestamp) throws IOException {
- // TODO: how to avoid partial deletion?
- mergeDeleteLock.lock();
- long version = versionController.nextVersion();
-
- // record what files are updated so we can roll back them in case of exception
- List<ModificationFile> updatedModFiles = new ArrayList<>();
-
- try {
- String fullPath = deviceId +
- IoTDBConstant.PATH_SEPARATOR + measurementId;
- Deletion deletion = new Deletion(fullPath, version, timestamp);
- if (mergingModification != null) {
- mergingModification.write(deletion);
- updatedModFiles.add(mergingModification);
- }
- deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- // delete data in memory
- OverflowProcessor ofProcessor = getOverflowProcessor(getProcessorName());
- ofProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- }
- } catch (Exception e) {
- // roll back
- for (ModificationFile modFile : updatedModFiles) {
- modFile.abort();
- }
- throw new IOException(e);
- } finally {
- mergeDeleteLock.unlock();
- }
- }
-
- private void deleteBufferWriteFiles(String deviceId, Deletion deletion,
- List<ModificationFile> updatedModFiles) throws IOException {
- if (currentTsFileResource != null && currentTsFileResource.containsDevice(deviceId)) {
- currentTsFileResource.getModFile().write(deletion);
- updatedModFiles.add(currentTsFileResource.getModFile());
- }
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode != currentTsFileResource && fileNode.containsDevice(deviceId)
- && fileNode.getStartTime(deviceId) <= deletion.getTimestamp()) {
- fileNode.getModFile().write(deletion);
- updatedModFiles.add(fileNode.getModFile());
- }
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- */
- public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws IOException, BufferWriteProcessorException {
- String fullPath = deviceId +
- IoTDBConstant.PATH_SEPARATOR + measurementId;
- long version = versionController.nextVersion();
- Deletion deletion = new Deletion(fullPath, version, timestamp);
-
- List<ModificationFile> updatedModFiles = new ArrayList<>();
- try {
- deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- } catch (IOException e) {
- for (ModificationFile modificationFile : updatedModFiles) {
- modificationFile.abort();
- }
- throw e;
- }
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- try {
- bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- } catch (BufferWriteProcessorException e) {
- throw new IOException(e);
- }
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- */
- public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws IOException {
- long version = versionController.nextVersion();
-
- OverflowProcessor overflowProcessor = getOverflowProcessor(getProcessorName());
- List<ModificationFile> updatedModFiles = new ArrayList<>();
- try {
- overflowProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- } catch (IOException e) {
- for (ModificationFile modificationFile : updatedModFiles) {
- modificationFile.abort();
- }
- throw e;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- FileNodeProcessor that = (FileNodeProcessor) o;
- return isOverflowed == that.isOverflowed &&
- numOfMergeFile == that.numOfMergeFile &&
- lastMergeTime == that.lastMergeTime &&
- shouldRecovery == that.shouldRecovery &&
- multiPassLockToken == that.multiPassLockToken &&
- Objects.equals(statStorageDeltaName, that.statStorageDeltaName) &&
- Objects.equals(statParamsHashMap, that.statParamsHashMap) &&
- Objects.equals(lastUpdateTimeMap, that.lastUpdateTimeMap) &&
- Objects.equals(flushLastUpdateTimeMap, that.flushLastUpdateTimeMap) &&
- Objects.equals(invertedIndexOfFiles, that.invertedIndexOfFiles) &&
- Objects.equals(emptyTsFileResource, that.emptyTsFileResource) &&
- Objects.equals(currentTsFileResource, that.currentTsFileResource) &&
- Objects.equals(newFileNodes, that.newFileNodes) &&
- isMerging == that.isMerging &&
- Objects.equals(fileNodeProcessorStore, that.fileNodeProcessorStore) &&
- Objects.equals(fileNodeRestoreFilePath, that.fileNodeRestoreFilePath) &&
- Objects.equals(bufferWriteProcessor, that.bufferWriteProcessor) &&
- Objects.equals(overflowProcessor, that.overflowProcessor) &&
- Objects.equals(oldMultiPassTokenSet, that.oldMultiPassTokenSet) &&
- Objects.equals(newMultiPassTokenSet, that.newMultiPassTokenSet) &&
- Objects.equals(oldMultiPassCount, that.oldMultiPassCount) &&
- Objects.equals(newMultiPassCount, that.newMultiPassCount) &&
- Objects.equals(parameters, that.parameters) &&
- Objects.equals(fileSchema, that.fileSchema) &&
- Objects.equals(flushFileNodeProcessorAction, that.flushFileNodeProcessorAction) &&
- Objects.equals(bufferwriteFlushAction, that.bufferwriteFlushAction) &&
- Objects.equals(bufferwriteCloseAction, that.bufferwriteCloseAction) &&
- Objects.equals(overflowFlushAction, that.overflowFlushAction);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), statStorageDeltaName, statParamsHashMap, isOverflowed,
- lastUpdateTimeMap, flushLastUpdateTimeMap, invertedIndexOfFiles,
- emptyTsFileResource, currentTsFileResource, newFileNodes, isMerging,
- numOfMergeFile, fileNodeProcessorStore, fileNodeRestoreFilePath,
- lastMergeTime, bufferWriteProcessor, overflowProcessor, oldMultiPassTokenSet,
- newMultiPassTokenSet, oldMultiPassCount, newMultiPassCount, shouldRecovery, parameters,
- fileSchema, flushFileNodeProcessorAction, bufferwriteFlushAction,
- bufferwriteCloseAction, overflowFlushAction, multiPassLockToken);
- }
-
- public class MergeRunnale implements Runnable {
-
- @Override
- public void run() {
- try {
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- long mergeStartTime = System.currentTimeMillis();
- writeLock();
- merge();
- long mergeEndTime = System.currentTimeMillis();
- long intervalTime = mergeEndTime - mergeStartTime;
- LOGGER.info(
- "The filenode processor {} merge start time is {}, "
- + "merge end time is {}, merge consumes {}ms.",
- getProcessorName(), ofInstant(Instant.ofEpochMilli(mergeStartTime),
- zoneId), ofInstant(Instant.ofEpochMilli(mergeEndTime),
- zoneId), intervalTime);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("The filenode processor {} encountered an error when merging.",
- getProcessorName(), e);
- throw new ErrorDebugException(e);
- }
- }
- }
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStatus.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStatus.java
deleted file mode 100644
index b3ce1e5..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStatus.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.filenode;
-
-public enum FileNodeProcessorStatus {
- NONE, MERGING_WRITE, WAITING;
-
- public static FileNodeProcessorStatus deserialize(short i) {
- switch (i) {
- case 0:
- return NONE;
- case 1:
- return MERGING_WRITE;
- case 2:
- return WAITING;
- default:
- throw new IllegalArgumentException(
- String.format("Invalid input %d for FileNodeProcessorStatus", i));
- }
- }
-
- public short serialize() {
- switch (this) {
- case NONE:
- return 0;
- case MERGING_WRITE:
- return 1;
- case WAITING:
- return 2;
- default:
- throw new IllegalStateException("Unsupported type");
- }
-
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStore.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStore.java
deleted file mode 100644
index 7c1eca4..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessorStore.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.filenode;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-/**
- * FileNodeProcessorStore is used to store information about FileNodeProcessor's status.
- * lastUpdateTime is changed and stored by BufferWrite flush or BufferWrite close.
- * emptyTsFileResource and newFileNodes are changed and stored by Overflow flush and
- * Overflow close. fileNodeProcessorState is changed and stored by the change of FileNodeProcessor's
- * status such as "work->merge merge->wait wait->work". numOfMergeFile is changed
- * and stored when FileNodeProcessor's status changes from work to merge.
- */
-public class FileNodeProcessorStore implements Serializable {
-
- private static final long serialVersionUID = -54525372941897565L;
-
- private boolean isOverflowed;
- private Map<String, Long> lastUpdateTimeMap;
- private TsFileResource emptyTsFileResource;
- private List<TsFileResource> newFileNodes;
- private int numOfMergeFile;
- private FileNodeProcessorStatus fileNodeProcessorStatus;
-
- /**
- * Constructor of FileNodeProcessorStore.
- *
- * @param isOverflowed whether this FileNode contains unmerged Overflow operations.
- * @param lastUpdateTimeMap the timestamp of last data point of each device in this FileNode.
- * @param emptyTsFileResource a place holder when the FileNode contains no TsFile.
- * @param newFileNodes TsFiles in the FileNode.
- * @param fileNodeProcessorStatus the status of the FileNode.
- * @param numOfMergeFile the number of files already merged in one merge operation.
- */
- public FileNodeProcessorStore(boolean isOverflowed, Map<String, Long> lastUpdateTimeMap,
- TsFileResource emptyTsFileResource,
- List<TsFileResource> newFileNodes,
- FileNodeProcessorStatus fileNodeProcessorStatus,
- int numOfMergeFile) {
- this.isOverflowed = isOverflowed;
- this.lastUpdateTimeMap = lastUpdateTimeMap;
- this.emptyTsFileResource = emptyTsFileResource;
- this.newFileNodes = newFileNodes;
- this.fileNodeProcessorStatus = fileNodeProcessorStatus;
- this.numOfMergeFile = numOfMergeFile;
- }
-
- public void serialize(OutputStream outputStream) throws IOException {
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- ReadWriteIOUtils.write(this.isOverflowed, byteArrayOutputStream);
- // lastUpdateTimeMap
- ReadWriteIOUtils.write(lastUpdateTimeMap.size(), byteArrayOutputStream);
- for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- ReadWriteIOUtils.write(entry.getKey(), byteArrayOutputStream);
- ReadWriteIOUtils.write(entry.getValue(), byteArrayOutputStream);
- }
- this.emptyTsFileResource.serialize(byteArrayOutputStream);
- ReadWriteIOUtils.write(this.newFileNodes.size(), byteArrayOutputStream);
- for (TsFileResource tsFileResource : this.newFileNodes) {
- tsFileResource.serialize(byteArrayOutputStream);
- }
- ReadWriteIOUtils.write(this.numOfMergeFile, byteArrayOutputStream);
- ReadWriteIOUtils.write(this.fileNodeProcessorStatus.serialize(), byteArrayOutputStream);
- // buffer array to outputstream
- byteArrayOutputStream.writeTo(outputStream);
- }
-
- public static FileNodeProcessorStore deSerialize(InputStream inputStream) throws IOException {
- boolean isOverflowed = ReadWriteIOUtils.readBool(inputStream);
- Map<String, Long> lastUpdateTimeMap = new HashMap<>();
- int size = ReadWriteIOUtils.readInt(inputStream);
- for (int i = 0; i < size; i++) {
- String path = ReadWriteIOUtils.readString(inputStream);
- long time = ReadWriteIOUtils.readLong(inputStream);
- lastUpdateTimeMap.put(path, time);
- }
- TsFileResource emptyTsFileResource = TsFileResource.deSerialize(inputStream);
- size = ReadWriteIOUtils.readInt(inputStream);
- List<TsFileResource> newFileNodes = new ArrayList<>();
- for (int i = 0; i < size; i++) {
- newFileNodes.add(TsFileResource.deSerialize(inputStream));
- }
- int numOfMergeFile = ReadWriteIOUtils.readInt(inputStream);
- FileNodeProcessorStatus fileNodeProcessorStatus = FileNodeProcessorStatus
- .deserialize(ReadWriteIOUtils.readShort(inputStream));
-
- return new FileNodeProcessorStore(isOverflowed, lastUpdateTimeMap, emptyTsFileResource,
- newFileNodes, fileNodeProcessorStatus, numOfMergeFile);
- }
-
- public boolean isOverflowed() {
- return isOverflowed;
- }
-
- public void setOverflowed(boolean isOverflowed) {
- this.isOverflowed = isOverflowed;
- }
-
- public FileNodeProcessorStatus getFileNodeProcessorStatus() {
- return fileNodeProcessorStatus;
- }
-
- public void setFileNodeProcessorStatus(FileNodeProcessorStatus fileNodeProcessorStatus) {
- this.fileNodeProcessorStatus = fileNodeProcessorStatus;
- }
-
- public Map<String, Long> getLastUpdateTimeMap() {
- return new HashMap<>(lastUpdateTimeMap);
- }
-
- public void setLastUpdateTimeMap(Map<String, Long> lastUpdateTimeMap) {
- this.lastUpdateTimeMap = lastUpdateTimeMap;
- }
-
- public TsFileResource getEmptyTsFileResource() {
- return emptyTsFileResource;
- }
-
- public void setEmptyTsFileResource(TsFileResource emptyTsFileResource) {
- this.emptyTsFileResource = emptyTsFileResource;
- }
-
- public List<TsFileResource> getNewFileNodes() {
- return newFileNodes;
- }
-
- public void setNewFileNodes(List<TsFileResource> newFileNodes) {
- this.newFileNodes = newFileNodes;
- }
-
- public int getNumOfMergeFile() {
- return numOfMergeFile;
- }
-
- public void setNumOfMergeFile(int numOfMergeFile) {
- this.numOfMergeFile = numOfMergeFile;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/OverflowChangeType.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/OverflowChangeType.java
deleted file mode 100644
index 48c8eee..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/OverflowChangeType.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.filenode;
-
-/**
- * if a file is not changed by overflow, it's in NO_CHANGE;<br>
- * if it's changed and in NO_CHANGE previously, NO_CHANGE-->CHANGED, update file<br>
- * If it's changed and in CHANGED previously, and in merging, CHANGED-->MERGING_CHANGE, update file<br>
- * If it's changed and in CHANGED previously, and not in merging, do nothing<br>
- * After merging, if it's MERGING_CHANGE, MERGING_CHANGE-->CHANGED, otherwise in NO_CHANGE, MERGING_CHANGE-->NO_CHANGE
- */
-public enum OverflowChangeType {
- NO_CHANGE, CHANGED, MERGING_CHANGE;
-
- public short serialize() {
- switch (this) {
- case NO_CHANGE:
- return 0;
- case CHANGED:
- return 1;
- case MERGING_CHANGE:
- return 2;
- default:
- throw new IllegalStateException("Unsupported type");
- }
- }
-
- public static OverflowChangeType deserialize(short i) {
- switch (i) {
- case 0:
- return NO_CHANGE;
- case 1:
- return CHANGED;
- case 2:
- return MERGING_CHANGE;
- default:
- throw new IllegalArgumentException(
- String.format("Invalid input %d for OverflowChangeType", i));
- }
- }
-}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
index a3604a3..606a7bd 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/FlushPartialPolicy.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.memcontrol;
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.utils.MemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +60,7 @@ public class FlushPartialPolicy implements Policy {
private Thread createWorkerThread() {
return new Thread(() -> {
- FileNodeManager.getInstance().forceFlush(BasicMemController.UsageLevel.SAFE);
+ DatabaseEngineFactory.getCurrent().forceFlush(BasicMemController.UsageLevel.SAFE);
try {
Thread.sleep(sleepInterval);
} catch (InterruptedException e) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
index 53533d9..3b50810 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memcontrol/ForceFLushAllPolicy.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.engine.memcontrol;
import org.apache.iotdb.db.concurrent.ThreadName;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.utils.MemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,7 +55,7 @@ public class ForceFLushAllPolicy implements Policy {
private Thread createWorkerThread() {
return new Thread(() ->
- FileNodeManager.getInstance().forceFlush(BasicMemController.UsageLevel.DANGEROUS),
+ DatabaseEngineFactory.getCurrent().forceFlush(BasicMemController.UsageLevel.DANGEROUS),
ThreadName.FORCE_FLUSH_ALL_POLICY.getName());
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
index 466138f..288382f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessor.java
@@ -37,7 +37,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.Processor;
import org.apache.iotdb.db.engine.bufferwrite.Action;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
import org.apache.iotdb.db.engine.memtable.MemSeriesLazyMerger;
@@ -113,9 +113,9 @@ public class OverflowProcessor extends Processor {
}
this.parentPath = overflowDirPath + processorName;
- overflowFlushAction = parameters.get(FileNodeConstants.OVERFLOW_FLUSH_ACTION);
+ overflowFlushAction = parameters.get(EngingeConstants.OVERFLOW_FLUSH_ACTION);
filenodeFlushAction = parameters
- .get(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
+ .get(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
reopen();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
index fd7280f..9c797ef 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.engine.querycontext;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.tsfile.read.common.Path;
public class SeriesDataSource {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index b1488cc..45d6faa 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -37,7 +37,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.DatabaseEngine;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -142,10 +141,10 @@ public class StorageGroupManager implements IStatistic, IService, DatabaseEngine
}
@Override
- public Map<String, TSRecord> getAllStatisticsValue() {
+ public Map<String, InsertPlan> getAllStatisticsValue() {
long curTime = System.currentTimeMillis();
TSRecord tsRecord = StatMonitor
- .convertToTSRecord(getStatParamsHashMap(), MonitorConstants.STAT_STORAGE_DELTA_NAME,
+ .convertToInsertPlan(getStatParamsHashMap(), MonitorConstants.STAT_STORAGE_DELTA_NAME,
curTime);
HashMap<String, TSRecord> ret = new HashMap<>();
ret.put(MonitorConstants.STAT_STORAGE_DELTA_NAME, tsRecord);
@@ -603,7 +602,7 @@ public class StorageGroupManager implements IStatistic, IService, DatabaseEngine
}
@Override
- public void deleteOneStorageGroup(String processorName) throws StorageGroupManagerException {
+ public void deleteStorageGroup(String processorName) throws StorageGroupManagerException {
if (storageGroupManagerStatus != StorageGroupManagerStatus.NONE) {
return;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
index 238596c..42923ad 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
@@ -48,8 +48,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.merge.MergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.ModificationFile;
@@ -268,7 +267,7 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
@Override
- public Map<String, TSRecord> getAllStatisticsValue() {
+ public Map<String, InsertPlan> getAllStatisticsValue() {
Long curTime = System.currentTimeMillis();
HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
TSRecord tsRecord = new TSRecord(curTime, statStorageGroupName);
@@ -781,7 +780,7 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
mergeBaseDir = Directories.getInstance().getNextFolderForTsfile();
mergeFileName = minimumTime
- + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR + System.currentTimeMillis()
+ + EngingeConstants.TSFILE_NAME_SEPARATOR + System.currentTimeMillis()
+ MERGE_TEMP_SUFFIX;
mergeOutputPath = constructOutputFilePath(mergeBaseDir, getProcessorName(),
mergeFileName);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/TsFileResource.java
similarity index 97%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/TsFileResource.java
index d8ba95f..4927ba9 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/TsFileResource.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.filenode;
+package org.apache.iotdb.db.engine.sgmanager;
import java.io.File;
import java.io.IOException;
@@ -28,11 +28,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.bufferwrite.RestorableTsFileIOWriter;
+import org.apache.iotdb.db.engine.tsfiledata.RestorableTsFileIOWriter;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
@@ -53,7 +51,7 @@ public class TsFileResource {
private Map<String, Long> endTimeMap;
private Set<String> mergeChanged = new HashSet<>();
- private transient ModificationFile modFile;
+ private ModificationFile modFile;
/**
* @param autoRead whether read the file to initialize startTimeMap and endTimeMap
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/RestorableTsFileIOWriter.java
similarity index 99%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/RestorableTsFileIOWriter.java
index a7542b5..7fa9190 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/RestorableTsFileIOWriter.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.db.engine.bufferwrite;
+package org.apache.iotdb.db.engine.tsfiledata;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
index c729285..d787609 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
@@ -42,9 +42,8 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.bufferwrite.RestorableTsFileIOWriter;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.EngingeConstants;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
import org.apache.iotdb.db.engine.memtable.IMemTable;
@@ -60,7 +59,6 @@ import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
import org.apache.iotdb.db.engine.sgmanager.OperationResult;
import org.apache.iotdb.db.engine.sgmanager.StorageGroupProcessor;
import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -236,12 +234,12 @@ public class TsFileProcessor extends Processor {
throws TsFileProcessorException {
File[] tsFiles = dataFolder
.listFiles(x -> !x.getName().contains(RestorableTsFileIOWriter.RESTORE_SUFFIX)
- && x.getName().split(FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR).length == 2);
+ && x.getName().split(EngingeConstants.TSFILE_NAME_SEPARATOR).length == 2);
if (tsFiles == null || tsFiles.length == 0) {
return;
}
Arrays.sort(tsFiles, Comparator.comparingLong(x -> Long
- .parseLong(x.getName().split(FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR)[0])));
+ .parseLong(x.getName().split(EngingeConstants.TSFILE_NAME_SEPARATOR)[0])));
for (File tsfile : tsFiles) {
if (tsfile.getName().endsWith(StorageGroupProcessor.MERGE_TEMP_SUFFIX)) {
@@ -259,7 +257,7 @@ public class TsFileProcessor extends Processor {
// add one TsFiles to tsFileResources and update device inserted time map
private void addResource(File tsfile) throws TsFileProcessorException {
//TODO we'd better define a file suffix for TsFile, e.g., .ts
- String[] names = tsfile.getName().split(FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR);
+ String[] names = tsfile.getName().split(EngingeConstants.TSFILE_NAME_SEPARATOR);
long time = Long.parseLong(names[0]);
if (fileNamePrefix < time) {
fileNamePrefix = time;
@@ -299,7 +297,7 @@ public class TsFileProcessor extends Processor {
LOGGER.debug("The bufferwrite processor data dir doesn't exists, create new directory {}.",
dataFolder.getAbsolutePath());
}
- String fileName = (fileNamePrefix + 1) + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
+ String fileName = (fileNamePrefix + 1) + EngingeConstants.TSFILE_NAME_SEPARATOR
+ System.currentTimeMillis();
return new File(dataFolder, fileName);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
index 36c83da..a25d595 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/IStatistic.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
public interface IStatistic {
@@ -32,7 +32,7 @@ public interface IStatistic {
* @return A HashMap that contains the module seriesPath like: root.stats.write.global,
* and its value is TSRecord format contains all statistics measurement
*/
- Map<String, TSRecord> getAllStatisticsValue();
+ Map<String, InsertPlan> getAllStatisticsValue();
/**
* registerStatMetadata registers statistics info to the manager.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index b04e506..2622bcd 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -31,7 +32,8 @@ import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngine;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
@@ -39,6 +41,7 @@ import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupProcessorStatConstants;
import org.apache.iotdb.db.monitor.collector.FileSize;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -107,22 +110,25 @@ public class StatMonitor implements IService {
}
/**
- * generate TSRecord.
+ * generate InsertPlan.
*
* @param hashMap key is statParams name, values is AtomicLong type
* @param statGroupDeltaName is the deviceId seriesPath of this module
* @param curTime current time stamp
- * @return TSRecord contains the DataPoints of a statGroupDeltaName
+ * @return InsertPlan contains the DataPoints of a statGroupDeltaName
*/
- public static TSRecord convertToTSRecord(Map<String, AtomicLong> hashMap,
+ public static InsertPlan convertToInsertPlan(Map<String, AtomicLong> hashMap,
String statGroupDeltaName, long curTime) {
- TSRecord tsRecord = new TSRecord(curTime, statGroupDeltaName);
- tsRecord.dataPointList = new ArrayList<>();
+
+ List<String> measurements = new ArrayList<>();
+ List<String> values = new ArrayList<>();
for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
AtomicLong value = entry.getValue();
- tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(), value.get()));
+ measurements.add(entry.getKey());
+ values.add(String.valueOf(value.get()));
}
- return tsRecord;
+ return new InsertPlan(statGroupDeltaName, curTime,
+ measurements.toArray(new String[0]), values.toArray(new String[0]));
}
public long getNumPointsInsert() {
@@ -218,53 +224,24 @@ public class StatMonitor implements IService {
}
/**
- * This function is not used and need to complete the query key concept.
- *
- * @return TSRecord, query statistics params
- */
- public Map<String, TSRecord> getOneStatisticsValue(String key) {
- // queryPath like fileNode seriesPath: root.stats.car1,
- // or FileNodeManager seriesPath:FileNodeManager
- String queryPath;
- if (key.contains("\\.")) {
- queryPath =
- MonitorConstants.STAT_STORAGE_GROUP_PREFIX + MonitorConstants.MONITOR_PATH_SEPARATOR
- + key.replaceAll("\\.", "_");
- } else {
- queryPath = key;
- }
- if (statisticMap.containsKey(queryPath)) {
- return statisticMap.get(queryPath).getAllStatisticsValue();
- } else {
- long currentTimeMillis = System.currentTimeMillis();
- HashMap<String, TSRecord> hashMap = new HashMap<>();
- TSRecord tsRecord = convertToTSRecord(
- MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST), queryPath,
- currentTimeMillis);
- hashMap.put(queryPath, tsRecord);
- return hashMap;
- }
- }
-
- /**
* get all statistics.
*/
- public Map<String, TSRecord> gatherStatistics() {
+ public Map<String, InsertPlan> gatherStatistics() {
synchronized (statisticMap) {
long currentTimeMillis = System.currentTimeMillis();
- HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
+ HashMap<String, InsertPlan> tsRecordHashMap = new HashMap<>();
for (Map.Entry<String, IStatistic> entry : statisticMap.entrySet()) {
if (entry.getValue() == null) {
switch (entry.getKey()) {
case MonitorConstants.STAT_STORAGE_DELTA_NAME:
tsRecordHashMap.put(entry.getKey(),
- convertToTSRecord(
+ convertToInsertPlan(
MonitorConstants.initValues(MonitorConstants.FILENODE_PROCESSOR_CONST),
entry.getKey(), currentTimeMillis));
break;
case MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME:
tsRecordHashMap.put(entry.getKey(),
- convertToTSRecord(
+ convertToInsertPlan(
MonitorConstants.initValues(MonitorConstants.FILE_SIZE_CONST),
entry.getKey(), currentTimeMillis));
break;
@@ -274,8 +251,8 @@ public class StatMonitor implements IService {
tsRecordHashMap.putAll(entry.getValue().getAllStatisticsValue());
}
}
- for (TSRecord value : tsRecordHashMap.values()) {
- value.time = currentTimeMillis;
+ for (InsertPlan value : tsRecordHashMap.values()) {
+ value.setTime(currentTimeMillis);
}
return tsRecordHashMap;
}
@@ -348,12 +325,12 @@ public class StatMonitor implements IService {
if (seconds >= statMonitorDetectFreqSec) {
runningTimeMillis = currentTimeMillis;
// delete time-series data
- FileNodeManager fManager = FileNodeManager.getInstance();
+ DatabaseEngine dbEngine = DatabaseEngineFactory.getCurrent();
try {
for (Map.Entry<String, IStatistic> entry : statisticMap.entrySet()) {
for (String statParamName : entry.getValue().getStatParamsHashMap().keySet()) {
if (temporaryStatList.contains(statParamName)) {
- fManager.delete(entry.getKey(), statParamName,
+ dbEngine.deleteData(entry.getKey(), statParamName,
currentTimeMillis - statMonitorRetainIntervalSec * 1000);
}
}
@@ -364,7 +341,7 @@ public class StatMonitor implements IService {
e);
}
}
- Map<String, TSRecord> tsRecordHashMap = gatherStatistics();
+ Map<String, InsertPlan> tsRecordHashMap = gatherStatistics();
insert(tsRecordHashMap);
numBackLoop.incrementAndGet();
} catch (Exception e) {
@@ -372,14 +349,14 @@ public class StatMonitor implements IService {
}
}
- public void insert(Map<String, TSRecord> tsRecordHashMap) {
- FileNodeManager fManager = FileNodeManager.getInstance();
+ public void insert(Map<String, InsertPlan> tsRecordHashMap) {
+ DatabaseEngine dbEngine = DatabaseEngineFactory.getCurrent();
int pointNum;
- for (Map.Entry<String, TSRecord> entry : tsRecordHashMap.entrySet()) {
+ for (Map.Entry<String, InsertPlan> entry : tsRecordHashMap.entrySet()) {
try {
- fManager.insert(entry.getValue(), true);
+ dbEngine.insert(entry.getValue(), true);
numInsert.incrementAndGet();
- pointNum = entry.getValue().dataPointList.size();
+ pointNum = entry.getValue().getValues().length;
numPointsInsert.addAndGet(pointNum);
} catch (StorageGroupManagerException e) {
numInsertError.incrementAndGet();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
index b3e695f..6f780cf 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
@@ -30,12 +30,14 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngine;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -54,13 +56,13 @@ public class FileSize implements IStatistic {
private static final Logger LOGGER = LoggerFactory.getLogger(FileSize.class);
private static final long ABNORMAL_VALUE = -1L;
private static final long INIT_VALUE_IF_FILE_NOT_EXIST = 0L;
- private FileNodeManager fileNodeManager;
+ private DatabaseEngine databaseEngine;
@Override
- public Map<String, TSRecord> getAllStatisticsValue() {
+ public Map<String, InsertPlan> getAllStatisticsValue() {
long curTime = System.currentTimeMillis();
TSRecord tsRecord = StatMonitor
- .convertToTSRecord(getStatParamsHashMap(), MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME,
+ .convertToInsertPlan(getStatParamsHashMap(), MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME,
curTime);
HashMap<String, TSRecord> ret = new HashMap<>();
ret.put(MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME, tsRecord);
@@ -77,11 +79,11 @@ public class FileSize implements IStatistic {
hashMap.put(seriesPath, MonitorConstants.DATA_TYPE_INT64);
Path path = new Path(seriesPath);
try {
- fileNodeManager.addTimeSeries(path, TSDataType.valueOf(MonitorConstants.DATA_TYPE_INT64),
+ databaseEngine.addTimeSeries(path, TSDataType.valueOf(MonitorConstants.DATA_TYPE_INT64),
TSEncoding.valueOf("RLE"), CompressionType.valueOf(TSFileConfig.compressor),
Collections.emptyMap());
} catch (StorageGroupManagerException e) {
- LOGGER.error("Register File Size Stats into fileNodeManager Failed.", e);
+ LOGGER.error("Register File Size Stats into databaseEngine Failed.", e);
}
}
StatMonitor.getInstance().registerStatStorageGroup(hashMap);
@@ -114,7 +116,7 @@ public class FileSize implements IStatistic {
}
private FileSize() {
- fileNodeManager = FileNodeManager.getInstance();
+ databaseEngine = DatabaseEngineFactory.getCurrent();
if (config.isEnableStatMonitor()) {
StatMonitor statMonitor = StatMonitor.getInstance();
registerStatMetadata();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
index 2fc51bb..b30da6c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
@@ -31,7 +31,8 @@ import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngine;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -63,8 +64,6 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,11 +72,11 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
private static final Logger LOG = LoggerFactory.getLogger(OverflowQPExecutor.class);
- private FileNodeManager fileNodeManager;
+ private DatabaseEngine databaseEngine;
private MManager mManager = MManager.getInstance();
public OverflowQPExecutor() {
- fileNodeManager = FileNodeManager.getInstance();
+ databaseEngine = DatabaseEngineFactory.getCurrent();
}
public static String checkValue(TSDataType dataType, String value) throws ProcessorException {
@@ -117,9 +116,8 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
return flag;
case INSERT:
InsertPlan insert = (InsertPlan) plan;
- int result = multiInsert(insert.getDeviceId(), insert.getTime(), insert.getMeasurements(),
- insert.getValues());
- return result > 0;
+ multiInsert(insert);
+ return true;
case CREATE_ROLE:
case DELETE_ROLE:
case CREATE_USER:
@@ -215,7 +213,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
mManager.getStorageGroupByPath(fullPath);
TSDataType dataType = mManager.getSeriesType(fullPath);
value = checkValue(dataType, value);
- fileNodeManager.update(deviceId, measurementId, startTime, endTime, dataType, value);
+ databaseEngine.update(deviceId, measurementId, startTime, endTime, dataType, value);
return true;
} catch (PathErrorException e) {
throw new ProcessorException(e.getMessage());
@@ -235,7 +233,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
String.format("Timeseries %s does not exist.", path.getFullPath()));
}
mManager.getStorageGroupByPath(path.getFullPath());
- fileNodeManager.delete(deviceId, measurementId, timestamp);
+ databaseEngine.deleteData(deviceId, measurementId, timestamp);
return true;
} catch (PathErrorException | StorageGroupManagerException e) {
throw new ProcessorException(e);
@@ -243,30 +241,12 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
}
@Override
- public void insert(Path path, long timestamp, String value) throws ProcessorException {
- String deviceId = path.getDevice();
- String measurementId = path.getMeasurement();
-
- try {
- TSDataType type = mManager.getSeriesType(deviceId + "," + measurementId);
- TSRecord tsRecord = new TSRecord(timestamp, deviceId);
- DataPoint dataPoint = DataPoint.getDataPoint(type, measurementId, value);
- tsRecord.addTuple(dataPoint);
- fileNodeManager.insert(tsRecord, false);
-
- } catch (PathErrorException e) {
- throw new ProcessorException("Error in insert: " + e.getMessage());
- } catch (StorageGroupManagerException e) {
- throw new ProcessorException(e);
- }
- }
-
- @Override
- public int multiInsert(String deviceId, long insertTime, String[] measurementList,
- String[] insertValues)
+ public void multiInsert(InsertPlan plan)
throws ProcessorException {
try {
- TSRecord tsRecord = new TSRecord(insertTime, deviceId);
+ String deviceId = plan.getDeviceId();
+ String[] measurementList = plan.getMeasurements();
+ String[] insertValues = plan.getValues();
MNode node = mManager.getNodeByDeviceIdFromCache(deviceId);
@@ -284,12 +264,9 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
}
TSDataType dataType = measurementNode.getSchema().getType();
- String value = insertValues[i];
- value = checkValue(dataType, value);
- DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementList[i], value);
- tsRecord.addTuple(dataPoint);
+ insertValues[i] = checkValue(dataType, insertValues[i]);
}
- return fileNodeManager.insert(tsRecord, false);
+ databaseEngine.insert(plan, false);
} catch (PathErrorException | StorageGroupManagerException e) {
throw new ProcessorException(e.getMessage());
@@ -535,11 +512,11 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
try {
if (isNewMeasurement) {
// add time series to schema
- fileNodeManager.addTimeSeries(path, dataType, encoding, compressor, props);
- //TODO fileNodeManager.addTimeSeries(
+ databaseEngine.addTimeSeries(path, dataType, encoding, compressor, props);
+ //TODO databaseEngine.addTimeSeries(
//TODO path, resultDataType, encoding, compressor, encodingArgs);
}
- // fileNodeManager.closeOneFileNode(namespacePath);
+ // databaseEngine.closeOneFileNode(namespacePath);
} catch (StorageGroupManagerException e) {
throw new ProcessorException(e);
}
@@ -613,10 +590,10 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
closeFileNodes.removeAll(deleteFielNodes);
for (String deleteFileNode : deleteFielNodes) {
// close processor
- fileNodeManager.deleteOneFileNode(deleteFileNode);
+ databaseEngine.deleteStorageGroup(deleteFileNode);
}
for (String closeFileNode : closeFileNodes) {
- fileNodeManager.closeOneFileNode(closeFileNode);
+ databaseEngine.closeStorageGroup(closeFileNode);
}
}
break;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index c89894c..d139f65 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.AggregationPlan;
import org.apache.iotdb.db.qp.physical.crud.FillQueryPlan;
import org.apache.iotdb.db.qp.physical.crud.GroupByPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.QueryPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.executor.EngineQueryRouter;
@@ -172,25 +173,11 @@ public abstract class QueryProcessExecutor {
protected abstract boolean delete(Path path, long deleteTime) throws ProcessorException;
/**
- * insert a single value. Only used in test
+ * execute insert command and return whether the operator is successful.
*
- * @param path seriesPath to be inserted
- * @param insertTime - it's time point but not a range
- * @param value value to be inserted
+ * @param plan the InsertPlan
*/
- public abstract void insert(Path path, long insertTime, String value) throws ProcessorException;
-
- /**
- * executeWithGlobalTimeFilter insert command and return whether the operator is successful.
- *
- * @param deviceId deviceId to be inserted
- * @param insertTime - it's time point but not a range
- * @param measurementList measurements to be inserted
- * @param insertValues values to be inserted
- * @return - Operate Type.
- */
- public abstract int multiInsert(String deviceId, long insertTime, String[] measurementList,
- String[] insertValues) throws ProcessorException;
+ public abstract void multiInsert(InsertPlan plan) throws ProcessorException;
public abstract List<String> getAllPaths(String originPath) throws PathErrorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index ad55b7a..84ceeb0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.query.control;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index 2b75efb..310a850 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
@@ -45,9 +45,9 @@ import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
* QueryResourceManager manages resource (file streams) used by each query job, and assign Ids to the jobs.
* During the life cycle of a query, the following methods must be called in strict order:
* 1. assignJobId - get an Id for the new job.
- * 2. beginQueryOfGivenQueryPaths - remind FileNodeManager that some files are being used
+ * 2. beginQueryOfGivenQueryPaths - remind DatabaseEngine that some files are being used
* 3. (if using filter)beginQueryOfGivenExpression
- * - remind FileNodeManager that some files are being used
+ * - remind DatabaseEngine that some files are being used
* 4. getQueryDataSource - open files for the job or reuse existing readers.
* 5. endQueryForGivenJob - release the resource used by this job.
* </p>
@@ -65,29 +65,29 @@ public class QueryResourceManager {
* <p>
* For example, during a query process Q1, given a query sql <sql>select device_1.sensor_1,
* device_1.sensor_2, device_2.sensor_1, device_2.sensor_2</sql>, we will invoke
- * <code>FileNodeManager.getInstance().beginQuery(device_1)</code> and
- * <code>FileNodeManager.getInstance().beginQuery(device_2)</code> both once. Although there
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery(device_1)</code> and
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery(device_2)</code> both once. Although there
* exists four paths, but the unique devices are only `device_1` and `device_2`. When invoking
- * <code>FileNodeManager.getInstance().beginQuery(device_1)</code>, it returns result token `1`.
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery(device_1)</code>, it returns result token `1`.
* Similarly,
- * <code>FileNodeManager.getInstance().beginQuery(device_2)</code> returns result token `2`.
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery(device_2)</code> returns result token `2`.
*
* In the meanwhile, another query process Q2 aroused by other client is triggered, whose sql
- * statement is same to Q1. Although <code>FileNodeManager.getInstance().beginQuery(device_1)
+ * statement is same to Q1. Although <code>DatabaseEngineFactory.getCurrent().beginQuery(device_1)
* </code>
* and
- * <code>FileNodeManager.getInstance().beginQuery(device_2)</code> will be invoked again, it
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery(device_2)</code> will be invoked again, it
* returns result token `3` and `4` .
*
- * <code>FileNodeManager.getInstance().endQueryForGivenJob(device_1, 1)</code> and
- * <code>FileNodeManager.getInstance().endQueryForGivenJob(device_2, 2)</code> must be invoked no matter how
+ * <code>DatabaseEngineFactory.getCurrent().endQueryForGivenJob(device_1, 1)</code> and
+ * <code>DatabaseEngineFactory.getCurrent().endQueryForGivenJob(device_2, 2)</code> must be invoked no matter how
* query process Q1 exits normally or abnormally. So is Q2,
- * <code>FileNodeManager.getInstance().endQueryForGivenJob(device_1, 3)</code> and
- * <code>FileNodeManager.getInstance().endQueryForGivenJob(device_2, 4)</code> must be invoked
+ * <code>DatabaseEngineFactory.getCurrent().endQueryForGivenJob(device_1, 3)</code> and
+ * <code>DatabaseEngineFactory.getCurrent().endQueryForGivenJob(device_2, 4)</code> must be invoked
*
* Last but no least, to ensure the correctness of write process and query process of IoTDB,
- * <code>FileNodeManager.getInstance().beginQuery()</code> and
- * <code>FileNodeManager.getInstance().endQueryForGivenJob()</code> must be executed rightly.
+ * <code>DatabaseEngineFactory.getCurrent().beginQuery()</code> and
+ * <code>DatabaseEngineFactory.getCurrent().endQueryForGivenJob()</code> must be executed rightly.
* </p>
*/
private ConcurrentHashMap<Long, ConcurrentHashMap<String, List<Integer>>> queryTokensMap;
@@ -125,7 +125,7 @@ public class QueryResourceManager {
for (String deviceId : deviceIdSet) {
putQueryTokenForCurrentRequestThread(jobId, deviceId,
- FileNodeManager.getInstance().beginQuery(deviceId));
+ DatabaseEngineFactory.getCurrent().beginQuery(deviceId));
}
}
@@ -139,7 +139,7 @@ public class QueryResourceManager {
getUniquePaths(expression, deviceIdSet);
for (String deviceId : deviceIdSet) {
putQueryTokenForCurrentRequestThread(jobId, deviceId,
- FileNodeManager.getInstance().beginQuery(deviceId));
+ DatabaseEngineFactory.getCurrent().beginQuery(deviceId));
}
}
@@ -148,7 +148,7 @@ public class QueryResourceManager {
throws StorageGroupManagerException {
SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath, null);
- QueryDataSource queryDataSource = FileNodeManager.getInstance()
+ QueryDataSource queryDataSource = DatabaseEngineFactory.getCurrent()
.query(singleSeriesExpression, context);
// add used files to current thread request cached map
@@ -168,7 +168,7 @@ public class QueryResourceManager {
}
for (Map.Entry<String, List<Integer>> entry : queryTokensMap.get(jobId).entrySet()) {
for (int token : entry.getValue()) {
- FileNodeManager.getInstance().endQuery(entry.getKey(), token);
+ DatabaseEngineFactory.getCurrent().endQuery(entry.getKey(), token);
}
}
queryTokensMap.remove(jobId);
@@ -198,26 +198,4 @@ public class QueryResourceManager {
}
}
-
-
- /**
- * TODO
- * This is only for test TsFileProcessor now. This method will finally be replaced when
- * fileNodeManager is refactored
- */
- public QueryDataSource getQueryDataSourceByTsFileProcessor(Path selectedPath,
- QueryContext context, TsFileProcessor processor)
- throws IOException, StorageGroupManagerException {
- OverflowSeriesDataSource overflowSeriesDataSource = new OverflowSeriesDataSource(selectedPath);
- overflowSeriesDataSource.setOverflowInsertFileList(Collections.EMPTY_LIST);
-
- SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath, null);
- SeriesDataSource dataSource =processor.query(singleSeriesExpression, context);
- QueryDataSource queryDataSource = new QueryDataSource(dataSource, overflowSeriesDataSource);
- // add used files to current thread request cached map
- filePathsManager.addUsedFilesForGivenJob(context.getJobId(), queryDataSource);
-
- return queryDataSource;
-
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
index d479be9..2819915 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
@@ -92,7 +92,8 @@ public class EngineExecutorWithoutTimeGenerator {
PriorityMergeReader unSeqMergeReader;
try {
unSeqMergeReader = SeriesReaderFactory.getInstance()
- .createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), timeFilter);
+ .createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(),
+ context, timeFilter);
} catch (IOException e) {
throw new StorageGroupManagerException(e);
}
@@ -146,62 +147,8 @@ public class EngineExecutorWithoutTimeGenerator {
PriorityMergeReader unSeqMergeReader;
try {
unSeqMergeReader = SeriesReaderFactory.getInstance()
- .createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), null);
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
-
- // merge sequence data with unsequence data.
- readersOfSelectedSeries.add(new AllDataReader(tsFilesReader, unSeqMergeReader));
- }
-
- try {
- return new EngineDataSetWithoutTimeGenerator(queryExpression.getSelectedSeries(), dataTypes,
- readersOfSelectedSeries);
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
- }
- /**
- * TODO
- * This is only for test TsFileProcessor now. This method will finally be replaced when
- * fileNodeManager is refactored
- */
- public QueryDataSet executeWithoutFilter(QueryContext context, TsFileProcessor processor)
- throws StorageGroupManagerException, IOException {
-
- List<IPointReader> readersOfSelectedSeries = new ArrayList<>();
- List<TSDataType> dataTypes = new ArrayList<>();
-
- QueryResourceManager.getInstance()
- .beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries());
-
- for (Path path : queryExpression.getSelectedSeries()) {
-
- QueryDataSource queryDataSource = QueryResourceManager.getInstance()
- .getQueryDataSourceByTsFileProcessor(path, context, processor);
-
- // add data type
- try {
- dataTypes.add(MManager.getInstance().getSeriesType(path.getFullPath()));
- } catch (PathErrorException e) {
- throw new StorageGroupManagerException(e);
- }
-
- // sequence insert data
- SequenceDataReader tsFilesReader;
- try {
- tsFilesReader = new SequenceDataReader(queryDataSource.getSeqDataSource(),
- null, context);
- } catch (IOException e) {
- throw new StorageGroupManagerException(e);
- }
-
- // unseq insert data
- PriorityMergeReader unSeqMergeReader;
- try {
- unSeqMergeReader = SeriesReaderFactory.getInstance()
- .createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), null);
+ .createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), context,
+ null);
} catch (IOException e) {
throw new StorageGroupManagerException(e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
index 930e87f..c808b0b 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
@@ -220,21 +220,4 @@ public class EngineQueryRouter {
return merged;
}
- /**
- * TODO
- * This is only for test TsFileProcessor now. This method will finally be replaced when
- * fileNodeManager is refactored
- */
- public QueryDataSet query(QueryExpression queryExpression, TsFileProcessor processor,
- QueryContext context)
- throws StorageGroupManagerException, IOException {
-
- if (queryExpression.hasQueryFilter()) {
- throw new NotImplementedException("this function is just for test...");
- } else {
- EngineExecutorWithoutTimeGenerator engineExecutor = new EngineExecutorWithoutTimeGenerator(
- queryExpression);
- return engineExecutor.executeWithoutFilter(context, processor);
- }
- }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
index 608c623..c6d34dc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
@@ -19,10 +19,11 @@
package org.apache.iotdb.db.query.factory;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
@@ -51,8 +52,10 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderByTimestamp;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithFilter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
public class SeriesReaderFactory {
@@ -70,7 +73,7 @@ public class SeriesReaderFactory {
* process, it's no need to maintain the opened file stream.
*/
public PriorityMergeReader createUnSeqMergeReader(
- SeriesDataSource overflowSeriesDataSource, QueryContext context)
+ SeriesDataSource overflowSeriesDataSource, QueryContext context, Filter filter)
throws IOException {
PriorityMergeReader unSeqMergeReader = new PriorityMergeReader();
@@ -87,7 +90,12 @@ public class SeriesReaderFactory {
for (ChunkMetaData chunkMetaData : metaDataList) {
Chunk chunk = chunkLoader.getChunk(chunkMetaData);
- ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
+ ChunkReader chunkReader;
+ if (filter == null) {
+ chunkReader = new ChunkReaderWithoutFilter(chunk);
+ } else {
+ chunkReader = new ChunkReaderWithFilter(chunk, filter);
+ }
unSeqMergeReader
.addReaderWithPriority(new EngineChunkReader(chunkReader, tsFileSequenceReader),
@@ -114,7 +122,7 @@ public class SeriesReaderFactory {
SequenceDataReader seqReader = new SequenceDataReader(seqDataSource, null, context);
// unSequence merge reader
- IPointReader unSeqReader = createUnSeqMergeReader(overflowDataSource, context);
+ IPointReader unSeqReader = createUnSeqMergeReader(overflowDataSource, context, null);
return new AllDataReader(seqReader, unSeqReader);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
index 0c44297..e38d673 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.IAggregateReader;
import org.apache.iotdb.db.query.reader.IBatchReader;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReaderByTimestamp.java
index e58583c..5778c50 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReaderByTimestamp.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/CloseMergeService.java b/iotdb/src/main/java/org/apache/iotdb/db/service/CloseMergeService.java
index 0915cd3..3743ad0 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/CloseMergeService.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/CloseMergeService.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.StartupException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,7 +177,7 @@ public class CloseMergeService implements IService {
+ "time interval is {}s.", startDateTime, endDateTime, timeInterval / 1000);
mergeAllLastTime = System.currentTimeMillis();
try {
- FileNodeManager.getInstance().mergeAll();
+ DatabaseEngineFactory.getCurrent().mergeAll();
} catch (Exception e) {
LOGGER.error("Merge all error.", e);
}
@@ -204,7 +203,7 @@ public class CloseMergeService implements IService {
+ "time interval is {}s.", startDateTime, endDateTime, timeInterval / 1000);
closeAllLastTime = System.currentTimeMillis();
try {
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngineFactory.getCurrent().closeAll();
} catch (Exception e) {
LOGGER.error("close all error.", e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 86c667c..e6f768e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.concurrent.IoTDBDefaultThreadExceptionHandler;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -82,7 +81,7 @@ public class IoTDB implements IoTDBMBean {
Runtime.getRuntime().addShutdownHook(new IoTDBShutdownHook());
setUncaughtExceptionHandler();
- FileNodeManager.getInstance().recovery();
+ DatabaseEngineFactory.getCurrent().recovery();
try {
systemDataRecovery();
} catch (RecoverException e) {
@@ -98,7 +97,7 @@ public class IoTDB implements IoTDBMBean {
StatMonitor.getInstance().recovery();
}
- registerManager.register(FileNodeManager.getInstance());
+ registerManager.register(DatabaseEngineFactory.getCurrent());
registerManager.register(MultiFileLogNodeManager.getInstance());
registerManager.register(JMXService.getInstance());
registerManager.register(JDBCService.getInstance());
@@ -154,7 +153,7 @@ public class IoTDB implements IoTDBMBean {
for (String filenodeName : filenodeNames) {
if (writeLogManager.hasWAL(filenodeName)) {
try {
- FileNodeManager.getInstance().recoverFileNode(filenodeName);
+ DatabaseEngineFactory.getCurrent().recoverFileNode(filenodeName);
} catch (StorageGroupManagerException e) {
throw new RecoverException(e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index ffaefdb..18e1008 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -34,7 +34,7 @@ import org.apache.iotdb.db.auth.authorizer.LocalFileAuthorizer;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -376,17 +376,17 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
switch (statement) {
case "flush":
try {
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngineFactory.getCurrent().closeAll();
} catch (StorageGroupManagerException e) {
- LOGGER.error("meet error while FileNodeManager closing all!", e);
+ LOGGER.error("meet error while DataBaseEngine closing all!", e);
throw new IOException(e);
}
return true;
case "merge":
try {
- FileNodeManager.getInstance().mergeAll();
+ DatabaseEngineFactory.getCurrent().mergeAll();
} catch (StorageGroupManagerException e) {
- LOGGER.error("meet error while FileNodeManager merging all!", e);
+ LOGGER.error("meet error while DataBaseEngine merging all!", e);
throw new IOException(e);
}
return true;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java
index a946112..01d9448 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java
@@ -25,7 +25,7 @@ import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.sync.conf.Constans;
public class SyncUtils {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
index d933dbb..a7292bf 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
@@ -43,9 +43,9 @@ import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.engine.filenode.OverflowChangeType;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.DatabaseEngine;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -81,7 +81,7 @@ public class SyncServiceImpl implements SyncService.Iface {
private static final Logger logger = LoggerFactory.getLogger(SyncServiceImpl.class);
- private static final FileNodeManager fileNodeManager = FileNodeManager.getInstance();
+ private static final DatabaseEngine databaseEngine = DatabaseEngineFactory.getCurrent();
/**
* Metadata manager
**/
@@ -474,12 +474,12 @@ public class SyncServiceImpl implements SyncService.Iface {
);
// call interface of load external file
try {
- if (!fileNodeManager.appendFileToFileNode(storageGroup, fileNode, path)) {
+ if (!databaseEngine.appendFileToStorageGroup(storageGroup, fileNode, path)) {
// it is a file with overflow data
if (config.isUpdate_historical_data_possibility()) {
loadOldData(path);
} else {
- List<String> overlapFiles = fileNodeManager.getOverlapFilesFromFileNode(
+ List<String> overlapFiles = databaseEngine.getOverlapFilesFromStorageGroup(
storageGroup,
fileNode, uuid.get());
if (overlapFiles.isEmpty()) {
@@ -555,11 +555,9 @@ public class SyncServiceImpl implements SyncService.Iface {
}
}
}
- if (insertExecutor
- .multiInsert(deviceId, record.getTimestamp(), measurementList.toArray(new String[]{}),
- insertValues.toArray(new String[]{})) <= 0) {
- throw new IOException("Inserting series data to IoTDB engine has failed.");
- }
+ insertExecutor
+ .multiInsert(new InsertPlan(deviceId, record.getTimestamp(),
+ measurementList.toArray(new String[]{}), insertValues.toArray(new String[]{})));
}
}
} catch (IOException e) {
@@ -633,19 +631,13 @@ public class SyncServiceImpl implements SyncService.Iface {
/** If there has no overlap data with the timeseries, inserting all data in the sync file **/
if (originDataPoints.isEmpty()) {
for (InsertPlan insertPlan : newDataPoints) {
- if (insertExecutor.multiInsert(insertPlan.getDeviceId(), insertPlan.getTime(),
- insertPlan.getMeasurements(), insertPlan.getValues()) <= 0) {
- throw new IOException("Inserting series data to IoTDB engine has failed.");
- }
+ insertExecutor.multiInsert(insertPlan);
}
} else {
/** Compare every data to get valid data **/
for (InsertPlan insertPlan : newDataPoints) {
if (!originDataPoints.contains(insertPlan)) {
- if (insertExecutor.multiInsert(insertPlan.getDeviceId(), insertPlan.getTime(),
- insertPlan.getMeasurements(), insertPlan.getValues()) <= 0) {
- throw new IOException("Inserting series data to IoTDB engine has failed.");
- }
+ insertExecutor.multiInsert(insertPlan);
}
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
index db5b528..580987f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
@@ -31,7 +31,8 @@ import java.util.List;
import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.DatabaseEngine;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
@@ -57,7 +58,7 @@ public class LoadDataUtils {
private int writeInstanceThreshold;
private boolean hasExtra = false;
private long totalPointCount = 0;
- private FileNodeManager fileNodeManager;
+ private DatabaseEngine databaseEngine;
private IoTDBConfig conf = IoTDBDescriptor.getInstance().getConfig();
/**
@@ -65,7 +66,7 @@ public class LoadDataUtils {
*/
public LoadDataUtils() {
writeInstanceMap = new HashSet<>();
- fileNodeManager = FileNodeManager.getInstance();
+ databaseEngine = DatabaseEngineFactory.getCurrent();
writeInstanceThreshold = conf.getWriteInstanceThreshold();
}
@@ -151,9 +152,9 @@ public class LoadDataUtils {
}
// appeared before, insert directly
try {
- fileNodeManager.insert(record, false);
+ databaseEngine.insert(record, false);
} catch (StorageGroupManagerException e) {
- logger.error("failed when insert into fileNodeManager, record:{}", line, e);
+ logger.error("failed when insert into databaseEngine, record:{}", line, e);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index c9621fe..8b2d66f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.utils;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.sgmanager.TsFileResource;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.query.context.QueryContext;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
index 75c5580..3e55df1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
@@ -31,7 +31,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.RecoverException;
@@ -285,7 +284,7 @@ public class ExclusiveLogRecoverPerformer implements RecoverPerformer {
+ " logs failed to recover, see logs above for details");
}
try {
- FileNodeManager.getInstance().closeOneFileNode(writeLogNode.getFileNodeName());
+ DatabaseEngineFactory.getCurrent().closeOneFileNode(writeLogNode.getFileNodeName());
} catch (StorageGroupManagerException e) {
logger.error("Log node {} cannot perform flush after replaying logs! Because {}",
writeLogNode.getIdentifier(), e.getMessage());
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
index b41542d..033b63c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
@@ -17,8 +17,6 @@
* under the License.
*/
package org.apache.iotdb.db.writelog.recover;
-
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.RecoverException;
import org.slf4j.Logger;
@@ -42,7 +40,7 @@ public class FileNodeRecoverPerformer implements RecoverPerformer {
@Override
public void recover() throws RecoverException {
try {
- FileNodeManager.getInstance().recoverFileNode(getFileNodeName());
+ DatabaseEngineFactory.getCurrent().recoverFileNode(getFileNodeName());
} catch (StorageGroupManagerException e) {
logger.error("Cannot recover filenode {}", identifier);
throw new RecoverException(e);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
index 267ee39..9ea6ed3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
@@ -17,11 +17,10 @@
* under the License.
*/
package org.apache.iotdb.db.writelog.replay;
-
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.StorageGroupManagerException;
+import org.apache.iotdb.db.engine.DatabaseEngineFactory;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
@@ -30,8 +29,6 @@ import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
public class ConcreteLogReplayer implements LogReplayer {
@@ -61,27 +58,14 @@ public class ConcreteLogReplayer implements LogReplayer {
}
private void multiInsert(InsertPlan insertPlan)
- throws PathErrorException, StorageGroupManagerException {
- String deviceId = insertPlan.getDeviceId();
- long insertTime = insertPlan.getTime();
- String[] measurementList = insertPlan.getMeasurements();
- String[] insertValues = insertPlan.getValues();
-
- TSRecord tsRecord = new TSRecord(insertTime, deviceId);
- for (int i = 0; i < measurementList.length; i++) {
- String pathKey = deviceId + "." + measurementList[i];
- TSDataType dataType = MManager.getInstance().getSeriesType(pathKey);
- String value = insertValues[i];
- DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementList[i], value);
- tsRecord.addTuple(dataPoint);
- }
- FileNodeManager.getInstance().insert(tsRecord, true);
+ throws StorageGroupManagerException {
+ DatabaseEngineFactory.getCurrent().insert(insertPlan, true);
}
private void update(UpdatePlan updatePlan) throws StorageGroupManagerException, PathErrorException {
TSDataType dataType = MManager.getInstance().getSeriesType(updatePlan.getPath().getFullPath());
for (Pair<Long, Long> timePair : updatePlan.getIntervals()) {
- FileNodeManager.getInstance().update(updatePlan.getPath().getDevice(),
+ DatabaseEngineFactory.getCurrent().update(updatePlan.getPath().getDevice(),
updatePlan.getPath().getMeasurement(), timePair.left, timePair.right, dataType,
updatePlan.getValue());
}
@@ -90,10 +74,10 @@ public class ConcreteLogReplayer implements LogReplayer {
private void delete(DeletePlan deletePlan, boolean isOverflow) throws StorageGroupManagerException {
for (Path path : deletePlan.getPaths()) {
if (isOverflow) {
- FileNodeManager.getInstance().deleteOverflow(path.getDevice(), path.getMeasurement(),
+ DatabaseEngineFactory.getCurrent().deleteInOverflow(path.getDevice(), path.getMeasurement(),
deletePlan.getDeleteTime());
} else {
- FileNodeManager.getInstance().deleteBufferWrite(path.getDevice(), path.getMeasurement(),
+ DatabaseEngineFactory.getCurrent().deleteInSeqFile(path.getDevice(), path.getMeasurement(),
deletePlan.getDeleteTime());
}
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteBenchmark.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteBenchmark.java
index 7c62ff7..d90eff0 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteBenchmark.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteBenchmark.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -75,22 +76,22 @@ public class BufferWriteBenchmark {
public static void main(String[] args) throws BufferWriteProcessorException, IOException {
before();
Map<String, Action> parameters = new HashMap<>();
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, new Action() {
+ parameters.put(EngingeConstants.BUFFERWRITE_FLUSH_ACTION, new Action() {
@Override
public void act() throws ActionException {
- System.out.println(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION);
+ System.out.println(EngingeConstants.BUFFERWRITE_FLUSH_ACTION);
}
});
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, new Action() {
+ parameters.put(EngingeConstants.BUFFERWRITE_CLOSE_ACTION, new Action() {
@Override
public void act() throws ActionException {
- System.out.println(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION);
+ System.out.println(EngingeConstants.BUFFERWRITE_CLOSE_ACTION);
}
});
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, new Action() {
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, new Action() {
@Override
public void act() throws ActionException {
- System.out.println(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
+ System.out.println(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
}
});
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
index 6299215..90a101b 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorNewTest.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.iotdb.db.conf.directories.Directories;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
@@ -85,9 +86,9 @@ public class BufferWriteProcessorNewTest {
@Before
public void setUp() throws Exception {
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
MetadataManagerHelper.initMetadata();
EnvironmentUtils.envSetUp();
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
index 3956736..2ad558e 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/BufferWriteProcessorTest.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.iotdb.db.conf.directories.Directories;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.PathUtils;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -96,9 +97,9 @@ public class BufferWriteProcessorTest {
@Before
public void setUp() throws Exception {
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
// origin value
groupSizeInByte = TsFileConf.groupSizeInByte;
// new value
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
index f2c393a..3054b05 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/bufferwrite/RestorableTsFileIOWriterTest.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.MemTableFlushUtil;
import org.apache.iotdb.db.engine.memtable.MemTableTestUtils;
import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
+import org.apache.iotdb.db.engine.tsfiledata.RestorableTsFileIOWriter;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
index 6e066af..951523c 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
@@ -21,7 +21,7 @@ package org.apache.iotdb.db.engine.filenodev2;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -36,7 +36,7 @@ import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
/**
* Bench The filenode manager with mul-thread and get its performance.
*/
-public class FileNodeManagerBenchmark {
+public class DatabaseEngineBenchmark {
private static int numOfWoker = 10;
private static int numOfDevice = 10;
@@ -113,7 +113,7 @@ public class FileNodeManagerBenchmark {
long time = RandomNum.getRandomLong(1, seed);
String deltaObject = devices[(int) (time % numOfDevice)];
TSRecord tsRecord = getRecord(deltaObject, time);
- FileNodeManager.getInstance().insert(tsRecord, true);
+ DatabaseEngine.getInstance().insert(tsRecord, true);
}
} catch (StorageGroupManagerException e) {
e.printStackTrace();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteFileSizeControlTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteFileSizeControlTest.java
index ca9e59d..56a130a 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteFileSizeControlTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteFileSizeControlTest.java
@@ -33,7 +33,7 @@ import org.apache.iotdb.db.engine.PathUtils;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
@@ -130,9 +130,9 @@ public class BufferwriteFileSizeControlTest {
new File(filename).delete();
Map<String, Action> parameters = new HashMap<>();
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
try {
processor = new BufferWriteProcessor(Directories.getInstance().getTsFolderForTest(), nsp,
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteMetaSizeControlTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteMetaSizeControlTest.java
index a800a80..86a16b0 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteMetaSizeControlTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/BufferwriteMetaSizeControlTest.java
@@ -33,7 +33,7 @@ import org.apache.iotdb.db.engine.PathUtils;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
@@ -130,9 +130,9 @@ public class BufferwriteMetaSizeControlTest {
new File(filename).delete();
Map<String, Action> parameters = new HashMap<>();
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_FLUSH_ACTION, bfflushaction);
+ parameters.put(EngingeConstants.BUFFERWRITE_CLOSE_ACTION, bfcloseaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fnflushaction);
try {
processor = new BufferWriteProcessor(Directories.getInstance().getTsFolderForTest(), nsp,
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowFileSizeControlTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowFileSizeControlTest.java
index ddf7a3b..72b4658 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowFileSizeControlTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowFileSizeControlTest.java
@@ -29,7 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
import org.apache.iotdb.db.exception.OverflowProcessorException;
@@ -78,14 +78,14 @@ public class OverflowFileSizeControlTest {
}
};
- private Action filenodemanagerbackupaction = new Action() {
+ private Action DatabaseEnginebackupaction = new Action() {
@Override
public void act() throws ActionException {
}
};
- private Action filenodemanagerflushaction = new Action() {
+ private Action DatabaseEngineflushaction = new Action() {
@Override
public void act() throws ActionException {
@@ -95,8 +95,8 @@ public class OverflowFileSizeControlTest {
@Before
public void setUp() throws Exception {
parameters = new HashMap<>();
- parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
+ parameters.put(EngingeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
overflowFileSize = dbConfig.getOverflowFileSizeThreshold();
groupSize = tsconfig.groupSizeInByte;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowMetaSizeControlTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowMetaSizeControlTest.java
index 2334ee7..8235954 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowMetaSizeControlTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/memcontrol/OverflowMetaSizeControlTest.java
@@ -29,7 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
import org.apache.iotdb.db.exception.OverflowProcessorException;
@@ -78,14 +78,14 @@ public class OverflowMetaSizeControlTest {
}
};
- private Action filenodemanagerbackupaction = new Action() {
+ private Action DatabaseEnginebackupaction = new Action() {
@Override
public void act() throws ActionException {
}
};
- private Action filenodemanagerflushaction = new Action() {
+ private Action DatabaseEngineflushaction = new Action() {
@Override
public void act() throws ActionException {
@@ -95,8 +95,8 @@ public class OverflowMetaSizeControlTest {
@Before
public void setUp() throws Exception {
parameters = new HashMap<String, Action>();
- parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
+ parameters.put(EngingeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
overflowFileSize = dbConfig.getOverflowMetaSizeThreshold();
groupSize = tsconfig.groupSizeInByte;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 47b0d89..2f22670 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -31,7 +31,7 @@ import java.util.Collections;
import java.util.Iterator;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
@@ -77,7 +77,7 @@ public class DeletionFileNodeTest {
for (int i = 0; i < 10; i++) {
MManager.getInstance().addPathToMTree(processorName + "." + measurements[i], dataType,
encoding);
- FileNodeManager.getInstance()
+ DatabaseEngine.getInstance()
.addTimeSeries(new Path(processorName, measurements[i]), TSDataType.valueOf(dataType),
TSEncoding.valueOf(encoding), CompressionType.valueOf(TSFileConfig.compressor),
Collections.emptyMap());
@@ -98,13 +98,13 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
SingleSeriesExpression expression = new SingleSeriesExpression(new Path(processorName,
measurements[5]), null);
@@ -130,13 +130,13 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 40);
- FileNodeManager.getInstance().delete(processorName, measurements[3], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 40);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 30);
Modification[] realModifications = new Modification[]{
new Deletion(processorName + "." + measurements[5], 102, 50),
@@ -173,9 +173,9 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
// insert into Overflow
for (int i = 1; i <= 100; i++) {
@@ -183,13 +183,13 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
SingleSeriesExpression expression = new SingleSeriesExpression(new Path(processorName,
measurements[5]), null);
@@ -218,9 +218,9 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
// insert into Overflow
for (int i = 1; i <= 100; i++) {
@@ -228,13 +228,13 @@ public class DeletionFileNodeTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 40);
- FileNodeManager.getInstance().delete(processorName, measurements[3], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 40);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 30);
Modification[] realModifications = new Modification[]{
new Deletion(processorName + "." + measurements[5], 103, 50),
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index 70e2de4..21f00db 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@ -26,7 +26,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
@@ -72,7 +72,7 @@ public class DeletionQueryTest {
for (int i = 0; i < 10; i++) {
MManager.getInstance().addPathToMTree(processorName + "." + measurements[i], dataType,
encoding);
- FileNodeManager.getInstance()
+ DatabaseEngine.getInstance()
.addTimeSeries(new Path(processorName, measurements[i]), TSDataType.valueOf(dataType),
TSEncoding.valueOf(encoding), CompressionType.valueOf(TSFileConfig.compressor),
Collections.emptyMap());
@@ -93,13 +93,13 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
List<Path> pathList = new ArrayList<>();
pathList.add(new Path(processorName, measurements[3]));
@@ -124,13 +124,13 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 40);
- FileNodeManager.getInstance().delete(processorName, measurements[3], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 40);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 30);
List<Path> pathList = new ArrayList<>();
pathList.add(new Path(processorName, measurements[3]));
@@ -156,9 +156,9 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
// insert into Overflow
for (int i = 1; i <= 100; i++) {
@@ -166,13 +166,13 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
List<Path> pathList = new ArrayList<>();
pathList.add(new Path(processorName, measurements[3]));
@@ -198,9 +198,9 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
// insert into Overflow
for (int i = 1; i <= 100; i++) {
@@ -208,13 +208,13 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 40);
- FileNodeManager.getInstance().delete(processorName, measurements[3], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 40);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 30);
List<Path> pathList = new ArrayList<>();
pathList.add(new Path(processorName, measurements[3]));
@@ -240,45 +240,45 @@ public class DeletionQueryTest {
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().forceFlush(UsageLevel.DANGEROUS);
+ DatabaseEngine.getInstance().forceFlush(UsageLevel.DANGEROUS);
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 250);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 250);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 230);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 250);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 250);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 250);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 230);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 250);
- FileNodeManager.getInstance().forceFlush(UsageLevel.DANGEROUS);
+ DatabaseEngine.getInstance().forceFlush(UsageLevel.DANGEROUS);
for (int i = 201; i <= 300; i++) {
TSRecord record = new TSRecord(i, processorName);
for (int j = 0; j < 10; j++) {
record.addTuple(new DoubleDataPoint(measurements[j], i * 1.0));
}
- FileNodeManager.getInstance().insert(record, false);
+ DatabaseEngine.getInstance().insert(record, false);
}
- FileNodeManager.getInstance().delete(processorName, measurements[3], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[4], 50);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 30);
- FileNodeManager.getInstance().delete(processorName, measurements[5], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[3], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[4], 50);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 30);
+ DatabaseEngine.getInstance().delete(processorName, measurements[5], 50);
- FileNodeManager.getInstance().closeAll();
+ DatabaseEngine.getInstance().closeAll();
List<Path> pathList = new ArrayList<>();
pathList.add(new Path(processorName, measurements[3]));
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorBenchmark.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorBenchmark.java
index 4fac693..8942e7c 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorBenchmark.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorBenchmark.java
@@ -27,7 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.OverflowProcessorException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -82,16 +82,16 @@ public class OverflowProcessorBenchmark {
public static void main(String[] args) throws IOException, OverflowProcessorException {
Map<String, Action> parameters = new HashMap<>();
- parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, new Action() {
+ parameters.put(EngingeConstants.OVERFLOW_FLUSH_ACTION, new Action() {
@Override
public void act() throws ActionException {
- System.out.println(FileNodeConstants.OVERFLOW_FLUSH_ACTION);
+ System.out.println(EngingeConstants.OVERFLOW_FLUSH_ACTION);
}
});
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, new Action() {
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, new Action() {
@Override
public void act() throws ActionException {
- System.out.println(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
+ System.out.println(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION);
}
});
OverflowProcessor overflowProcessor = new OverflowProcessor("Overflow_bench", parameters,
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
index 9877f76..b366c21 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflow/io/OverflowProcessorTest.java
@@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.iotdb.db.engine.PathUtils;
import org.apache.iotdb.db.engine.bufferwrite.Action;
import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.EngingeConstants;
import org.apache.iotdb.db.engine.querycontext.MergeSeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
@@ -67,8 +67,8 @@ public class OverflowProcessorTest {
public void setUp() throws Exception {
EnvironmentUtils.envSetUp();
parameters = new HashMap<String, Action>();
- parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
+ parameters.put(EngingeConstants.OVERFLOW_FLUSH_ACTION, overflowflushaction);
+ parameters.put(EngingeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, filenodeflushaction);
}
@After
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
index cd470c9..d701c7a 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
@@ -25,7 +25,7 @@ import static org.apache.iotdb.db.utils.EnvironmentUtils.TEST_QUERY_JOB_ID;
import java.io.IOException;
import java.util.Collections;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -50,7 +50,7 @@ import org.junit.Test;
public class QueryDataFromUnclosedTsFileIT {
long bufferWriteFileSize;
- FileNodeManager sgManager;
+ DatabaseEngine sgManager;
MManager mManager;
EngineQueryRouter queryManager;
@Before
@@ -61,7 +61,7 @@ public class QueryDataFromUnclosedTsFileIT {
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
bufferWriteFileSize = IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold();
//IoTDBDescriptor.getInstance().getConfig().setBufferwriteFileSizeThreshold(100);
- sgManager = FileNodeManager.getInstance();
+ sgManager = DatabaseEngine.getInstance();
mManager = MManager.getInstance();
queryManager = new EngineQueryRouter();
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
index 5fa1728..f02381d 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
@@ -28,7 +28,7 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
@@ -63,22 +63,22 @@ public class MonitorTest {
}
@Test
- public void testFileNodeManagerMonitorAndAddMetadata() {
- FileNodeManager fManager = FileNodeManager.getInstance();
+ public void testDatabaseEngineMonitorAndAddMetadata() {
+ DatabaseEngine dbEngine = DatabaseEngine.getInstance();
FileSize fileSize = FileSize.getInstance();
statMonitor = StatMonitor.getInstance();
statMonitor.registerStatStorageGroup();
- fManager.getStatParamsHashMap().forEach((key, value) -> value.set(0));
+ dbEngine.getStatParamsHashMap().forEach((key, value) -> value.set(0));
fileSize.getStatParamsHashMap().forEach((key, value) -> value.set(0));
statMonitor.clearIStatisticMap();
- statMonitor.registerStatistics(fManager.getClass().getSimpleName(), fManager);
+ statMonitor.registerStatistics(dbEngine.getClass().getSimpleName(), dbEngine);
statMonitor
.registerStatistics(MonitorConstants.FILE_SIZE_STORAGE_GROUP_NAME, FileSize.getInstance());
// add metadata
MManager mManager = MManager.getInstance();
- fManager.registerStatMetadata();
+ dbEngine.registerStatMetadata();
fileSize.registerStatMetadata();
- Map<String, AtomicLong> statParamsHashMap = fManager.getStatParamsHashMap();
+ Map<String, AtomicLong> statParamsHashMap = dbEngine.getStatParamsHashMap();
Map<String, AtomicLong> fileSizeStatsHashMap = fileSize.getStatParamsHashMap();
for (String statParam : statParamsHashMap.keySet()) {
assertTrue(mManager.pathExist(
@@ -103,10 +103,10 @@ public class MonitorTest {
// Get stat data and test right
- Map<String, TSRecord> statHashMap = fManager.getAllStatisticsValue();
+ Map<String, TSRecord> statHashMap = dbEngine.getAllStatisticsValue();
Map<String, TSRecord> fileSizeStatMap = fileSize.getAllStatisticsValue();
- String path = fManager.getAllPathForStatistic().get(0);
+ String path = dbEngine.getAllPathForStatistic().get(0);
String fileSizeStatPath = fileSize.getAllPathForStatistic().get(0);
int pos = path.lastIndexOf('.');
int fileSizeStatPos = fileSizeStatPath.lastIndexOf('.');
@@ -138,7 +138,7 @@ public class MonitorTest {
}
try {
- fManager.deleteAll();
+ dbEngine.deleteAll();
} catch (StorageGroupManagerException e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 3f9bffd..0c28697 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -28,7 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.cache.RowGroupBlockMetaDataCache;
import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenode.DatabaseEngine;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.StartupException;
@@ -73,7 +73,7 @@ public class EnvironmentUtils {
// tsFileConfig.duplicateIncompletedPage = false;
// clean filenode manager
try {
- if (!FileNodeManager.getInstance().deleteAll()) {
+ if (!DatabaseEngine.getInstance().deleteAll()) {
LOGGER.error("Can't close the filenode manager in EnvironmentUtils");
Assert.fail();
}
@@ -81,7 +81,7 @@ public class EnvironmentUtils {
throw new IOException(e);
}
StatMonitor.getInstance().close();
- FileNodeManager.getInstance().resetFileNodeManager();
+ DatabaseEngine.getInstance().resetDatabaseEngine();
// clean wal
MultiFileLogNodeManager.getInstance().stop();
// clean cache
@@ -92,7 +92,7 @@ public class EnvironmentUtils {
MManager.getInstance().flushObjectToFile();
// delete all directory
cleanAllDir();
- // FileNodeManager.getInstance().reset();
+ // DatabaseEngine.getInstance().reset();
// reset MemController
BasicMemController.getInstance().close();
}
@@ -168,7 +168,7 @@ public class EnvironmentUtils {
} catch (AuthException e) {
throw new StartupException(e.getMessage());
}
- FileNodeManager.getInstance().resetFileNodeManager();
+ DatabaseEngine.getInstance().resetDatabaseEngine();
MultiFileLogNodeManager.getInstance().start();
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId();
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
[incubator-iotdb] 03/05: rename StorageEngine to DatabaseEngine
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 245cc54f7739ef3fb6a83b7447c60d213fe1a32d
Author: 江天 <jt...@163.com>
AuthorDate: Tue May 28 12:50:27 2019 +0800
rename StorageEngine to DatabaseEngine
---
.../engine/{sgmanager/StorageEngine.java => DatabaseEngine.java} | 6 +++---
.../StorageEngineFactory.java => DatabaseEngineFactory.java} | 8 +++++---
.../org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java | 3 ++-
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
similarity index 97%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
index 72c7dcf..7dc436e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngine.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngine.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.sgmanager;
+package org.apache.iotdb.db.engine;
import java.util.List;
import java.util.Map;
@@ -34,9 +34,9 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
/**
- * StorageEngine is an abstraction of IoTDB storage-level interfaces.
+ * DatabaseEngine is an abstraction of IoTDB storage-level interfaces.
*/
-public interface StorageEngine {
+public interface DatabaseEngine {
/**
* This function is just for unit test.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngineFactory.java
similarity index 82%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngineFactory.java
index 85218a7..f67ef54 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageEngineFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/DatabaseEngineFactory.java
@@ -17,11 +17,13 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.sgmanager;
+package org.apache.iotdb.db.engine;
-public class StorageEngineFactory {
+import org.apache.iotdb.db.engine.sgmanager.StorageGroupManager;
- public static StorageEngine getCurrent() {
+public class DatabaseEngineFactory {
+
+ public static DatabaseEngine getCurrent() {
return StorageGroupManager.getInstance();
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index 22d37b8..b1488cc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
+import org.apache.iotdb.db.engine.DatabaseEngine;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
@@ -67,7 +68,7 @@ import org.slf4j.LoggerFactory;
* StorageGroupManager provides top-level interfaces to access IoTDB storage engine. It decides
* which StorageGroup(s) to access in order to complete a query.
*/
-public class StorageGroupManager implements IStatistic, IService, StorageEngine {
+public class StorageGroupManager implements IStatistic, IService, DatabaseEngine {
private static final Logger LOGGER = LoggerFactory.getLogger(StorageGroupManager.class);
private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();