You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/05/17 11:21:48 UTC
[incubator-iotdb] branch refactor_overflow updated: partially
migrate query and merge
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/refactor_overflow by this push:
new e7b534e partially migrate query and merge
e7b534e is described below
commit e7b534ea69c999d5962f460c12bf8c3ced9892b3
Author: 江天 <jt...@163.com>
AuthorDate: Fri May 17 19:19:34 2019 +0800
partially migrate query and merge
---
.../iotdb/db/engine/filenode/FileNodeManager.java | 158 ++--
.../db/engine/filenode/FileNodeProcessor.java | 3 +-
.../iotdb/db/engine/filenode/TsFileResource.java | 72 +-
.../apache/iotdb/db/engine/merge/MergeTask.java | 64 ++
.../db/engine/overflowdata/OverflowProcessor.java | 2 +-
.../apache/iotdb/db/engine/pool/MergeManager.java | 5 +
.../db/engine/querycontext/QueryDataSource.java | 6 +-
.../db/engine/querycontext/SeriesDataSource.java | 34 +-
.../iotdb/db/engine/sgmanager/OperationResult.java | 2 -
.../StorageGroupManager.java} | 645 ++++++-----------
.../db/engine/sgmanager/StorageGroupProcessor.java | 792 +++++++++++++++++++--
.../db/engine/tsfiledata/TsFileProcessor.java | 129 +++-
...tion.java => StorageGroupManagerException.java} | 10 +-
.../StorageGroupProcessorException.java} | 24 +-
.../java/org/apache/iotdb/db/metadata/MGraph.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 22 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 18 +-
.../apache/iotdb/db/monitor/MonitorConstants.java | 5 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 11 +-
.../iotdb/db/monitor/collector/FileSize.java | 4 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 36 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 13 +-
.../iotdb/db/query/control/JobFileManager.java | 6 +-
.../db/query/control/QueryResourceManager.java | 12 +-
.../db/query/executor/AggregateEngineExecutor.java | 6 +-
.../executor/EngineExecutorWithTimeGenerator.java | 10 +-
.../EngineExecutorWithoutTimeGenerator.java | 32 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 14 +-
.../db/query/executor/FillEngineExecutor.java | 4 +-
.../GroupByWithOnlyTimeFilterDataSetDataSet.java | 4 +-
.../GroupByWithValueFilterDataSetDataSet.java | 4 +-
.../db/query/factory/SeriesReaderFactory.java | 156 ++--
.../db/query/reader/merge/PriorityMergeReader.java | 3 -
.../overflow/OverflowSeriesSingleFileReader.java | 77 --
.../query/reader/sequence/SealedTsFilesReader.java | 36 +-
.../query/reader/sequence/SequenceDataReader.java | 12 +-
.../sequence/SequenceDataReaderByTimestamp.java | 6 +-
.../query/timegenerator/EngineNodeConstructor.java | 10 +-
.../query/timegenerator/EngineTimeGenerator.java | 6 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 8 +-
.../org/apache/iotdb/db/service/IoTDBMBean.java | 4 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 13 +-
.../java/org/apache/iotdb/db/sync/SyncUtils.java | 63 ++
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 11 +-
.../org/apache/iotdb/db/utils/LoadDataUtils.java | 6 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 47 ++
.../recover/ExclusiveLogRecoverPerformer.java | 4 +-
.../writelog/recover/FileNodeRecoverPerformer.java | 4 +-
.../db/writelog/replay/ConcreteLogReplayer.java | 9 +-
.../db/engine/filenode/TsFileResourceTest.java | 2 +-
.../filenodev2/FileNodeManagerBenchmark.java | 8 +-
.../engine/modification/DeletionFileNodeTest.java | 14 +-
.../db/engine/modification/DeletionQueryTest.java | 16 +-
.../engine/overflowdata/OverflowProcessorTest.java | 5 +-
.../db/engine/tsfiledata/TsFileProcessorTest.java | 9 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 10 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 8 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 10 +-
.../integration/QueryDataFromUnclosedTsFileIT.java | 9 +-
.../iotdb/db/metadata/MManagerAdvancedTest.java | 7 +-
.../iotdb/db/metadata/MManagerImproveTest.java | 4 +-
.../org/apache/iotdb/db/metadata/MTreeTest.java | 11 +-
.../org/apache/iotdb/db/monitor/MonitorTest.java | 4 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 10 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 8 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 6 +-
67 files changed, 1666 insertions(+), 1107 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index 516abdc..e291e88 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
@@ -43,13 +43,14 @@ import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
import org.apache.iotdb.db.engine.pool.FlushManager;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -99,8 +100,8 @@ public class FileNodeManager implements IStatistic, IService {
processorMap = new ConcurrentHashMap<>();
statParamsHashMap = new HashMap<>();
//label: A
- for (MonitorConstants.FileNodeManagerStatConstants fileNodeManagerStatConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants fileNodeManagerStatConstant :
+ StorageGroupManagerStatConstants.values()) {
statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
}
@@ -126,9 +127,9 @@ public class FileNodeManager implements IStatistic, IService {
}
private void updateStatHashMapWhenFail(TSRecord tsRecord) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_FAIL.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_FAIL.name())
.incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_FAIL.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS_FAIL.name())
.addAndGet(tsRecord.dataPointList.size());
}
@@ -145,8 +146,8 @@ public class FileNodeManager implements IStatistic, IService {
@Override
public List<String> getAllPathForStatistic() {
List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants statConstant :
+ StorageGroupManagerStatConstants.values()) {
list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name());
}
@@ -170,8 +171,8 @@ public class FileNodeManager implements IStatistic, IService {
@Override
public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants statConstant :
+ StorageGroupManagerStatConstants.values()) {
hashMap
.put(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
@@ -193,24 +194,24 @@ public class FileNodeManager implements IStatistic, IService {
* @param filenodeName storage name, e.g., root.a.b
*/
private FileNodeProcessor constructNewProcessor(String filenodeName)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
try {
return new FileNodeProcessor(baseDir, filenodeName);
} catch (FileNodeProcessorException e) {
LOGGER.error("Can't construct the FileNodeProcessor, the filenode is {}", filenodeName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
private FileNodeProcessor getProcessor(String path, boolean isWriteLock)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
String filenodeName;
try {
// return the stroage name
- filenodeName = MManager.getInstance().getFileNameByPath(path);
+ filenodeName = MManager.getInstance().getStorageGroupByPath(path);
} catch (PathErrorException e) {
LOGGER.error("MManager get filenode name error, seriesPath is {}", path);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
FileNodeProcessor processor;
processor = processorMap.get(filenodeName);
@@ -242,7 +243,7 @@ public class FileNodeManager implements IStatistic, IService {
public void recovery() {
List<String> filenodeNames = null;
try {
- filenodeNames = MManager.getInstance().getAllFileNames();
+ filenodeNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
LOGGER.error("Restoring all FileNodes failed.", e);
return;
@@ -256,7 +257,7 @@ public class FileNodeManager implements IStatistic, IService {
filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
fileNodeProcessor.fileNodeRecovery();
}
- } catch (FileNodeManagerException | FileNodeProcessorException e) {
+ } catch (StorageGroupManagerException | FileNodeProcessorException e) {
LOGGER.error("Restoring fileNode {} failed.", filenodeName, e);
} finally {
if (fileNodeProcessor != null) {
@@ -273,9 +274,8 @@ public class FileNodeManager implements IStatistic, IService {
* @param tsRecord input Data
* @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
* be recorded. if false, the statParamsHashMap will be updated.
- * @return an int value represents the insert type
*/
- public int insert(TSRecord tsRecord, boolean isMonitor) throws FileNodeManagerException {
+ public void insert(TSRecord tsRecord, boolean isMonitor) throws StorageGroupManagerException {
long timestamp = tsRecord.time;
String deviceId = tsRecord.deviceId;
@@ -283,21 +283,18 @@ public class FileNodeManager implements IStatistic, IService {
updateStat(isMonitor, tsRecord);
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- int insertType;
try {
long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
if (timestamp < lastUpdateTime) {
insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
- insertType = 1;
} else {
insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
- insertType = 2;
}
} catch (FileNodeProcessorException e) {
LOGGER.error(String.format("Encounter an error when closing the buffer write processor %s.",
fileNodeProcessor.getProcessorName()), e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -309,17 +306,16 @@ public class FileNodeManager implements IStatistic, IService {
fileNodeProcessor.getStatParamsHashMap()
.get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_SUCCESS.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
statParamsHashMap
- .get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
+ .get(StorageGroupManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
.addAndGet(tsRecord.dataPointList.size());
}
- return insertType;
}
private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
try {
if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
String[] measurementList = new String[tsRecord.dataPointList.size()];
@@ -337,27 +333,27 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
- private void checkTimestamp(TSRecord tsRecord) throws FileNodeManagerException {
+ private void checkTimestamp(TSRecord tsRecord) throws StorageGroupManagerException {
if (tsRecord.time < 0) {
LOGGER.error("The insert time lt 0, {}.", tsRecord);
- throw new FileNodeManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
+ throw new StorageGroupManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
}
}
private void updateStat(boolean isMonitor, TSRecord tsRecord) {
if (!isMonitor) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS.name())
.addAndGet(tsRecord.dataPointList.size());
}
}
private void insertOverflow(FileNodeProcessor fileNodeProcessor, long timestamp,
TSRecord tsRecord, boolean isMonitor, String deviceId)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
// get overflow processor
OverflowProcessor overflowProcessor;
String filenodeName = fileNodeProcessor.getProcessorName();
@@ -369,7 +365,7 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// write wal
writeLog(tsRecord, isMonitor, overflowProcessor.getLogNode());
@@ -383,13 +379,13 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
private void insertBufferWrite(FileNodeProcessor fileNodeProcessor, long timestamp,
boolean isMonitor, TSRecord tsRecord, String deviceId)
- throws FileNodeManagerException, FileNodeProcessorException {
+ throws StorageGroupManagerException, FileNodeProcessorException {
// get bufferwrite processor
BufferWriteProcessor bufferWriteProcessor;
String filenodeName = fileNodeProcessor.getProcessorName();
@@ -401,7 +397,7 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// Add a new interval file to newfilelist
if (bufferWriteProcessor.isNewProcessor()) {
@@ -414,7 +410,7 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
// write wal
@@ -435,7 +431,7 @@ public class FileNodeManager implements IStatistic, IService {
if (!isMonitor) {
updateStatHashMapWhenFail(tsRecord);
}
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
if (bufferWriteProcessor
@@ -459,7 +455,7 @@ public class FileNodeManager implements IStatistic, IService {
*/
public void update(String deviceId, String measurementId, long startTime, long endTime,
TSDataType type, String v)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
@@ -483,7 +479,7 @@ public class FileNodeManager implements IStatistic, IService {
"Get the overflow processor failed, the filenode is {}, "
+ "insert time range is from {} to {}",
filenodeName, startTime, finalEndTime);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
overflowProcessor.update(deviceId, measurementId, startTime, finalEndTime, type, v);
// change the type of tsfile to overflowed
@@ -498,7 +494,7 @@ public class FileNodeManager implements IStatistic, IService {
+ "." + measurementId)));
}
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} finally {
fileNodeProcessor.writeUnlock();
@@ -509,7 +505,7 @@ public class FileNodeManager implements IStatistic, IService {
* delete data.
*/
public void delete(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
@@ -536,7 +532,7 @@ public class FileNodeManager implements IStatistic, IService {
} catch (IOException | FileNodeProcessorException e) {
LOGGER.error("Getting the processor failed, the filenode is {}, delete time is {}.",
filenodeName, timestamp);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
try {
overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
@@ -544,14 +540,14 @@ public class FileNodeManager implements IStatistic, IService {
bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
new Path(deviceId + "." + measurementId)));
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
try {
fileNodeProcessor.delete(deviceId, measurementId, timestamp);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// change the type of tsfile to overflowed
fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
@@ -565,7 +561,7 @@ public class FileNodeManager implements IStatistic, IService {
private void delete(String processorName,
Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
//TODO do we need to call processorIterator.remove() ?
LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
@@ -574,7 +570,7 @@ public class FileNodeManager implements IStatistic, IService {
LOGGER.info("Try to delete the filenode processor {}.", processorName);
FileNodeProcessor processor = processorMap.get(processorName);
if (!processor.tryWriteLock()) {
- throw new FileNodeManagerException(String
+ throw new StorageGroupManagerException(String
.format("Can't delete the filenode processor %s because Can't get the write lock.",
processorName));
}
@@ -591,7 +587,7 @@ public class FileNodeManager implements IStatistic, IService {
processorIterator.remove();
} catch (ProcessorException e) {
LOGGER.error("Delete the filenode processor {} by iterator error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} finally {
processor.writeUnlock();
@@ -602,12 +598,12 @@ public class FileNodeManager implements IStatistic, IService {
* Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
*/
public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
fileNodeProcessor.deleteBufferWrite(deviceId, measurementId, timestamp);
} catch (BufferWriteProcessorException | IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -620,12 +616,12 @@ public class FileNodeManager implements IStatistic, IService {
* Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
*/
public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
fileNodeProcessor.deleteOverflow(deviceId, measurementId, timestamp);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -640,7 +636,7 @@ public class FileNodeManager implements IStatistic, IService {
* @param deviceId queried deviceId
* @return a query token for the device.
*/
- public int beginQuery(String deviceId) throws FileNodeManagerException {
+ public int beginQuery(String deviceId) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
LOGGER.debug("Get the FileNodeProcessor: filenode is {}, begin query.",
@@ -655,7 +651,7 @@ public class FileNodeManager implements IStatistic, IService {
* query data.
*/
public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
String deviceId = seriesExpression.getSeriesPath().getDevice();
String measurementId = seriesExpression.getSeriesPath().getMeasurement();
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, false);
@@ -670,7 +666,7 @@ public class FileNodeManager implements IStatistic, IService {
} catch (IOException e) {
LOGGER.error("Get the overflow processor failed, the filenode is {}, query is {},{}",
fileNodeProcessor.getProcessorName(), deviceId, measurementId);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
try {
@@ -678,7 +674,7 @@ public class FileNodeManager implements IStatistic, IService {
} catch (FileNodeProcessorException e) {
LOGGER.error("Query error: the deviceId {}, the measurementId {}", deviceId, measurementId,
e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// return query structure
return queryDataSource;
@@ -690,7 +686,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* end query.
*/
- public void endQuery(String deviceId, int token) throws FileNodeManagerException {
+ public void endQuery(String deviceId, int token) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
try {
@@ -699,7 +695,7 @@ public class FileNodeManager implements IStatistic, IService {
fileNodeProcessor.decreaseMultiPassCount(token);
} catch (FileNodeProcessorException e) {
LOGGER.error("Failed to end query: the deviceId {}, token {}.", deviceId, token, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -713,7 +709,7 @@ public class FileNodeManager implements IStatistic, IService {
* @param appendFile the appended tsfile information
*/
public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
- String appendFilePath) throws FileNodeManagerException {
+ String appendFilePath) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
try {
// check append file
@@ -728,7 +724,7 @@ public class FileNodeManager implements IStatistic, IService {
fileNodeProcessor.appendFile(appendFile, appendFilePath);
} catch (FileNodeProcessorException e) {
LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -742,13 +738,13 @@ public class FileNodeManager implements IStatistic, IService {
* @param appendFile the appended tsfile information
*/
public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
- String uuid) throws FileNodeManagerException {
+ String uuid) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
List<String> overlapFiles;
try {
overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
} catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -758,9 +754,9 @@ public class FileNodeManager implements IStatistic, IService {
/**
* merge all overflowed filenode.
*
- * @throws FileNodeManagerException FileNodeManagerException
+ * @throws StorageGroupManagerException StorageGroupManagerException
*/
- public void mergeAll() throws FileNodeManagerException {
+ public void mergeAll() throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
fileNodeManagerStatus);
@@ -771,10 +767,10 @@ public class FileNodeManager implements IStatistic, IService {
LOGGER.info("Start to merge all overflowed filenode");
List<String> allFileNodeNames;
try {
- allFileNodeNames = MManager.getInstance().getAllFileNames();
+ allFileNodeNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
LOGGER.error("Get all storage group seriesPath error,", e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
List<Future<?>> futureTasks = new ArrayList<>();
for (String fileNodeName : allFileNodeNames) {
@@ -820,7 +816,7 @@ public class FileNodeManager implements IStatistic, IService {
}
if (!mergeException.isEmpty()) {
// just throw the first exception
- throw new FileNodeManagerException(mergeException.get(0));
+ throw new StorageGroupManagerException(mergeException.get(0));
}
fileNodeManagerStatus = FileNodeManagerStatus.NONE;
LOGGER.info("End to merge all overflowed filenode");
@@ -833,7 +829,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* try to close the filenode processor. The name of filenode processor is processorName
*/
- private boolean closeOneProcessor(String processorName) throws FileNodeManagerException {
+ private boolean closeOneProcessor(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
return true;
}
@@ -849,7 +845,7 @@ public class FileNodeManager implements IStatistic, IService {
}
} catch (ProcessorException e) {
LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
}
@@ -861,7 +857,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* delete one filenode.
*/
- public void deleteOneFileNode(String processorName) throws FileNodeManagerException {
+ public void deleteOneFileNode(String processorName) throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
return;
}
@@ -883,7 +879,7 @@ public class FileNodeManager implements IStatistic, IService {
.deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
} catch (IOException e) {
LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeManagerStatus = FileNodeManagerStatus.NONE;
}
@@ -908,7 +904,7 @@ public class FileNodeManager implements IStatistic, IService {
}
}
- private void deleteFileNodeBlocked(String processorName) throws FileNodeManagerException {
+ private void deleteFileNodeBlocked(String processorName) throws StorageGroupManagerException {
LOGGER.info("Forced to delete the filenode processor {}", processorName);
FileNodeProcessor processor = processorMap.get(processorName);
while (true) {
@@ -927,7 +923,7 @@ public class FileNodeManager implements IStatistic, IService {
}
} catch (ProcessorException e) {
LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
}
@@ -960,7 +956,7 @@ public class FileNodeManager implements IStatistic, IService {
*/
public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor,
- Map<String, String> props) throws FileNodeManagerException {
+ Map<String, String> props) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
try {
fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
@@ -973,7 +969,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* Force to close the filenode processor.
*/
- public void closeOneFileNode(String processorName) throws FileNodeManagerException {
+ public void closeOneFileNode(String processorName) throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
return;
}
@@ -1000,7 +996,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* try to close the filenode processor.
*/
- private void close(String processorName) throws FileNodeManagerException {
+ private void close(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
return;
@@ -1018,7 +1014,7 @@ public class FileNodeManager implements IStatistic, IService {
processor.close();
} catch (ProcessorException e) {
LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} else {
LOGGER.warn("The filenode processor {} can't be closed.", processorName);
@@ -1031,7 +1027,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* delete all filenode.
*/
- public synchronized boolean deleteAll() throws FileNodeManagerException {
+ public synchronized boolean deleteAll() throws StorageGroupManagerException {
LOGGER.info("Start deleting all filenode");
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.info("Failed to delete all filenode processor because of merge operation");
@@ -1056,7 +1052,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* Try to close All.
*/
- public void closeAll() throws FileNodeManagerException {
+ public void closeAll() throws StorageGroupManagerException {
LOGGER.info("Start closing all filenode processor");
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.info("Failed to close all filenode processor because of merge operation");
@@ -1165,7 +1161,7 @@ public class FileNodeManager implements IStatistic, IService {
public void stop() {
try {
closeAll();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("Failed to close file node manager because .", e);
}
}
@@ -1191,14 +1187,14 @@ public class FileNodeManager implements IStatistic, IService {
* recover filenode.
*/
public void recoverFileNode(String filenodeName)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(filenodeName, true);
LOGGER.info("Recover the filenode processor, the filenode is {}, the status is {}",
filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
try {
fileNodeProcessor.fileNodeRecovery();
} catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
index acaea24..508d689 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
@@ -1237,8 +1237,7 @@ public class FileNodeProcessor extends Processor implements IStatistic {
endTimeMap.put(deviceId, tsFileResource.getEndTime(deviceId));
}
}
- TsFileResource node = new TsFileResource(startTimeMap, endTimeMap,
- tsFileResource.getOverflowChangeType(), tsFileResource.getFile());
+ TsFileResource node = new TsFileResource(startTimeMap, endTimeMap, tsFileResource.getFile());
result.add(node);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java
index 3d6d74a..d8ba95f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/TsFileResource.java
@@ -46,8 +46,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
*/
public class TsFileResource {
- private OverflowChangeType overflowChangeType;
-
//the file index of `settled` folder in the Directories.
private int baseDirIndex;
private File file;
@@ -61,7 +59,7 @@ public class TsFileResource {
* @param autoRead whether read the file to initialize startTimeMap and endTimeMap
*/
public TsFileResource(File file, boolean autoRead) throws IOException {
- this(new HashMap<>(), new HashMap<>(), OverflowChangeType.NO_CHANGE, file);
+ this(new HashMap<>(), new HashMap<>(), file);
if (autoRead) {
//init startTime and endTime
try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) {
@@ -87,7 +85,7 @@ public class TsFileResource {
* @param writer an unclosed TsFile Writer
*/
public TsFileResource(File file, RestorableTsFileIOWriter writer) {
- this(new HashMap<>(), new HashMap<>(), OverflowChangeType.NO_CHANGE, file);
+ this(new HashMap<>(), new HashMap<>(), file);
initTimeMapFromChunGroupMetaDatas(writer.getChunkGroupMetaDatas());
}
@@ -109,10 +107,8 @@ public class TsFileResource {
}
- public TsFileResource(Map<String, Long> startTimeMap, Map<String, Long> endTimeMap,
- OverflowChangeType type, File file) {
+ public TsFileResource(Map<String, Long> startTimeMap, Map<String, Long> endTimeMap, File file) {
- this.overflowChangeType = type;
if (file != null) {
this.baseDirIndex = Directories.getInstance()
.getTsFileFolderIndex(file.getParentFile().getParent());
@@ -126,7 +122,6 @@ public class TsFileResource {
}
public void serialize(OutputStream outputStream) throws IOException {
- ReadWriteIOUtils.write(this.overflowChangeType.serialize(), outputStream);
ReadWriteIOUtils.write(this.baseDirIndex, outputStream);
ReadWriteIOUtils.writeIsNull(this.file, outputStream);
if (this.file != null) {
@@ -149,8 +144,6 @@ public class TsFileResource {
}
public static TsFileResource deSerialize(InputStream inputStream) throws IOException {
- OverflowChangeType overflowChangeType = OverflowChangeType
- .deserialize(ReadWriteIOUtils.readShort(inputStream));
int baseDirIndex = ReadWriteIOUtils.readInt(inputStream);
boolean hasRelativePath = ReadWriteIOUtils.readIsNull(inputStream);
@@ -179,7 +172,7 @@ public class TsFileResource {
String path = ReadWriteIOUtils.readString(inputStream);
mergeChanaged.add(path);
}
- TsFileResource tsFileResource = new TsFileResource(startTimes, endTimes, overflowChangeType, file);
+ TsFileResource tsFileResource = new TsFileResource(startTimes, endTimes, file);
tsFileResource.mergeChanged = mergeChanaged;
return tsFileResource;
}
@@ -261,16 +254,6 @@ public class TsFileResource {
startTimeMap.clear();
endTimeMap.clear();
mergeChanged.clear();
- overflowChangeType = OverflowChangeType.NO_CHANGE;
- }
-
- public void changeTypeToChanged(FileNodeProcessorStatus fileNodeProcessorState) {
-
- if (fileNodeProcessorState == FileNodeProcessorStatus.MERGING_WRITE) {
- overflowChangeType = OverflowChangeType.MERGING_CHANGE;
- } else {
- overflowChangeType = OverflowChangeType.CHANGED;
- }
}
public void addMergeChanged(String deviceId) {
@@ -299,7 +282,7 @@ public class TsFileResource {
Map<String, Long> startTimeMapCopy = new HashMap<>(this.startTimeMap);
Map<String, Long> endTimeMapCopy = new HashMap<>(this.endTimeMap);
return new TsFileResource(startTimeMapCopy,
- endTimeMapCopy, overflowChangeType, file);
+ endTimeMapCopy, file);
}
public Set<String> getDevices() {
@@ -308,42 +291,13 @@ public class TsFileResource {
@Override
public int hashCode() {
-
- final int prime = 31;
- int result = 1;
- result = prime * result + ((endTimeMap == null) ? 0 : endTimeMap.hashCode());
- result = prime * result + ((file == null) ? 0 : file.hashCode());
- result = prime * result + ((overflowChangeType == null) ? 0 : overflowChangeType.hashCode());
- result = prime * result + ((startTimeMap == null) ? 0 : startTimeMap.hashCode());
- return result;
+ return super.hashCode();
}
@Override
public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof TsFileResource)) {
- return false;
- }
- TsFileResource that = (TsFileResource) o;
- return baseDirIndex == that.baseDirIndex &&
- overflowChangeType == that.overflowChangeType &&
- Objects.equals(file, that.file) &&
- Objects.equals(startTimeMap, that.startTimeMap) &&
- Objects.equals(endTimeMap, that.endTimeMap) &&
- Objects.equals(mergeChanged, that.mergeChanged) &&
- Objects.equals(modFile, that.modFile);
- }
-
-
- public OverflowChangeType getOverflowChangeType() {
- return overflowChangeType;
- }
-
- public void setOverflowChangeType(OverflowChangeType overflowChangeType) {
- this.overflowChangeType = overflowChangeType;
+ return this == o;
}
public synchronized ModificationFile getModFile() {
@@ -387,4 +341,16 @@ public class TsFileResource {
public String getFilePath() {
return this.getFile().getAbsolutePath();
}
+
+ public boolean overlaps(TsFileResource another) {
+ for (Entry<String, Long> entry : another.getStartTimeMap().entrySet()) {
+ if (this.getStartTimeMap().containsKey(entry.getKey()) &&
+ this.getEndTime(entry.getKey()) >= entry.getValue()
+ && this.getStartTime(entry.getKey()) <= another
+ .getEndTime(entry.getKey())) {
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
new file mode 100644
index 0000000..e0fb2cc
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge;
+
+import static java.time.ZonedDateTime.ofInstant;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.concurrent.Callable;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.sgmanager.StorageGroupProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MergeTask implements Callable<Exception> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MergeTask.class);
+
+ private StorageGroupProcessor processor;
+
+ public MergeTask(StorageGroupProcessor processor) {
+ this.processor = processor;
+ }
+
+ @Override
+ public Exception call() {
+ try {
+ ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
+ long mergeStartTime = System.currentTimeMillis();
+ processor.merge();
+ long mergeEndTime = System.currentTimeMillis();
+ long intervalTime = mergeEndTime - mergeStartTime;
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(
+ "The filenode processor {} merge start time is {}, "
+ + "merge end time is {}, merge consumes {}ms.",
+ processor.getProcessorName(), ofInstant(Instant.ofEpochMilli(mergeStartTime),
+ zoneId), ofInstant(Instant.ofEpochMilli(mergeEndTime),
+ zoneId), intervalTime);
+ }
+
+ } catch (Exception e) {
+ return e;
+ }
+ return null;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessor.java
index eae9799..b96a16d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessor.java
@@ -39,7 +39,7 @@ public class OverflowProcessor extends TsFileProcessor {
*/
public OverflowProcessor(String processorName, VersionController versionController,
FileSchema fileSchemaRef)
- throws TsFileProcessorException, IOException {
+ throws TsFileProcessorException {
super(processorName, versionController,
fileSchemaRef);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
index 75190f2..81b0dcb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/pool/MergeManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.engine.pool;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -109,6 +110,10 @@ public class MergeManager {
return pool.submit(task);
}
+ public Future<?> submit(Callable task) {
+ return pool.submit(task);
+ }
+
public int getActiveCnt() {
return ((ThreadPoolExecutor) pool).getActiveCount();
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
index 84dac5f..97f0d1c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
@@ -24,10 +24,10 @@ public class QueryDataSource {
private SeriesDataSource seriesDataSource;
// unSequence data source
- private OverflowSeriesDataSource overflowSeriesDataSource;
+ private SeriesDataSource overflowSeriesDataSource;
public QueryDataSource(SeriesDataSource seriesDataSource,
- OverflowSeriesDataSource overflowSeriesDataSource) {
+ SeriesDataSource overflowSeriesDataSource) {
this.seriesDataSource = seriesDataSource;
this.overflowSeriesDataSource = overflowSeriesDataSource;
}
@@ -36,7 +36,7 @@ public class QueryDataSource {
return seriesDataSource;
}
- public OverflowSeriesDataSource getOverflowSeriesDataSource() {
+ public SeriesDataSource getOverflowSeriesDataSource() {
return overflowSeriesDataSource;
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
index 933c0bc..fd7280f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/SeriesDataSource.java
@@ -27,46 +27,46 @@ public class SeriesDataSource {
private Path seriesPath;
// sealed tsfile
- private List<TsFileResource> sealedTsFiles;
+ private List<TsFileResource> sealedFiles;
// unsealed tsfile
- private UnsealedTsFile unsealedTsFile;
+ private UnsealedTsFile unsealedFile;
// seq mem-table
private ReadOnlyMemChunk readableChunk;
- public SeriesDataSource(Path seriesPath, List<TsFileResource> sealedTsFiles,
- UnsealedTsFile unsealedTsFile,
+ public SeriesDataSource(Path seriesPath, List<TsFileResource> sealedFiles,
+ UnsealedTsFile unsealedFile,
ReadOnlyMemChunk readableChunk) {
this.seriesPath = seriesPath;
- this.sealedTsFiles = sealedTsFiles;
- this.unsealedTsFile = unsealedTsFile;
+ this.sealedFiles = sealedFiles;
+ this.unsealedFile = unsealedFile;
this.readableChunk = readableChunk;
}
public boolean hasSealedTsFiles() {
- return sealedTsFiles != null && !sealedTsFiles.isEmpty();
+ return sealedFiles != null && !sealedFiles.isEmpty();
}
- public List<TsFileResource> getSealedTsFiles() {
- return sealedTsFiles;
+ public List<TsFileResource> getSealedFiles() {
+ return sealedFiles;
}
- public void setSealedTsFiles(List<TsFileResource> sealedTsFiles) {
- this.sealedTsFiles = sealedTsFiles;
+ public void setSealedFiles(List<TsFileResource> sealedFiles) {
+ this.sealedFiles = sealedFiles;
}
- public boolean hasUnsealedTsFile() {
- return unsealedTsFile != null;
+ public boolean hasUnsealedFile() {
+ return unsealedFile != null;
}
- public UnsealedTsFile getUnsealedTsFile() {
- return unsealedTsFile;
+ public UnsealedTsFile getUnsealedFile() {
+ return unsealedFile;
}
- public void setUnsealedTsFile(UnsealedTsFile unsealedTsFile) {
- this.unsealedTsFile = unsealedTsFile;
+ public void setUnsealedFile(UnsealedTsFile unsealedFile) {
+ this.unsealedFile = unsealedFile;
}
public boolean hasRawSeriesChunk() {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/OperationResult.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/OperationResult.java
index 1a2deb0..1da7259 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/OperationResult.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/OperationResult.java
@@ -28,6 +28,4 @@ public enum OperationResult {
WRITE_REJECT_BY_TIME,
//the write operation is reject because there is no available memory
WRITE_REJECT_BY_MEM,
- //attempts to write a closed FileProcessor
- WRITE_REJECT_BY_CLOSED_PROCESSOR,
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
similarity index 53%
copy from iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
copy to iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
index 516abdc..51ed0e3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupManager.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.filenode;
+package org.apache.iotdb.db.engine.sgmanager;
import java.io.File;
import java.io.IOException;
@@ -37,30 +37,28 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
+import org.apache.iotdb.db.engine.filenode.FileNodeProcessor;
+import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
import org.apache.iotdb.db.engine.pool.FlushManager;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -68,52 +66,46 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class FileNodeManager implements IStatistic, IService {
+/**
+ * StorageGroupManager provides top-level interfaces to access IoTDB storage engine. It decides
+ * which StorageGroup(s) to access in order to complete a query.
+ */
+public class StorageGroupManager implements IStatistic, IService {
- private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeManager.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(StorageGroupManager.class);
private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
private static final Directories directories = Directories.getInstance();
- /**
- * a folder that persist FileNodeProcessorStore classes. Each stroage group will have a subfolder.
- * by default, it is system/info
- */
- private final String baseDir;
/**
- * This map is used to manage all filenode processor,<br> the key is filenode name which is
+ * This map is used to manage all StorageGroupProcessors,<br> the key is filenode name which is
* storage group seriesPath.
+ * e.g. if the user executes a query "SET STORAGE GROUP TO root.beijing", then "root.beijing" will
+ * be a key.
*/
- private ConcurrentHashMap<String, FileNodeProcessor> processorMap;
+ private ConcurrentHashMap<String, StorageGroupProcessor> processorMap;
+
/**
- * This set is used to store overflowed filenode name.<br> The overflowed filenode will be merge.
+ * fileNodeManagerStatus indicates whether the Manager is merging or being closed.
*/
private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // There is no need to add concurrently
+
private HashMap<String, AtomicLong> statParamsHashMap;
- private FileNodeManager(String baseDir) {
+ private StorageGroupManager() {
processorMap = new ConcurrentHashMap<>();
+ initStat();
+ }
+
+ private void initStat() {
statParamsHashMap = new HashMap<>();
- //label: A
- for (MonitorConstants.FileNodeManagerStatConstants fileNodeManagerStatConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants fileNodeManagerStatConstant :
+ StorageGroupManagerStatConstants.values()) {
statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
}
- String normalizedBaseDir = baseDir;
- if (normalizedBaseDir.charAt(normalizedBaseDir.length() - 1) != File.separatorChar) {
- normalizedBaseDir += Character.toString(File.separatorChar);
- }
- this.baseDir = normalizedBaseDir;
- File dir = new File(normalizedBaseDir);
- if (dir.mkdirs()) {
- LOGGER.info("{} dir home doesn't exist, create it", dir.getPath());
- }
- //TODO merge this with label A
if (TsFileDBConf.isEnableStatMonitor()) {
StatMonitor statMonitor = StatMonitor.getInstance();
registerStatMetadata();
@@ -121,14 +113,14 @@ public class FileNodeManager implements IStatistic, IService {
}
}
- public static FileNodeManager getInstance() {
+ public static StorageGroupManager getInstance() {
return FileNodeManagerHolder.INSTANCE;
}
private void updateStatHashMapWhenFail(TSRecord tsRecord) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_FAIL.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_FAIL.name())
.incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_FAIL.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS_FAIL.name())
.addAndGet(tsRecord.dataPointList.size());
}
@@ -145,8 +137,8 @@ public class FileNodeManager implements IStatistic, IService {
@Override
public List<String> getAllPathForStatistic() {
List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants statConstant :
+ StorageGroupManagerStatConstants.values()) {
list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name());
}
@@ -170,8 +162,8 @@ public class FileNodeManager implements IStatistic, IService {
@Override
public void registerStatMetadata() {
Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants statConstant :
+ StorageGroupManagerStatConstants.values()) {
hashMap
.put(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
+ statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
@@ -192,27 +184,28 @@ public class FileNodeManager implements IStatistic, IService {
/**
* @param filenodeName storage name, e.g., root.a.b
*/
- private FileNodeProcessor constructNewProcessor(String filenodeName)
- throws FileNodeManagerException {
+ private StorageGroupProcessor constructNewProcessor(String filenodeName)
+ throws StorageGroupManagerException {
try {
- return new FileNodeProcessor(baseDir, filenodeName);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Can't construct the FileNodeProcessor, the filenode is {}", filenodeName, e);
- throw new FileNodeManagerException(e);
+ return new StorageGroupProcessor(filenodeName);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String.format("Can't construct the "
+ + "StorageGroupProcessor, the StorageGroup is %s", filenodeName), e);
}
}
- private FileNodeProcessor getProcessor(String path, boolean isWriteLock)
- throws FileNodeManagerException {
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ private StorageGroupProcessor getProcessor(String path, boolean isWriteLock)
+ throws StorageGroupManagerException {
String filenodeName;
try {
// return the stroage name
- filenodeName = MManager.getInstance().getFileNameByPath(path);
+ filenodeName = MManager.getInstance().getStorageGroupByPath(path);
} catch (PathErrorException e) {
- LOGGER.error("MManager get filenode name error, seriesPath is {}", path);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(
+ String.format("MManager get StorageGroup name error, seriesPath is %s", path), e);
}
- FileNodeProcessor processor;
+ StorageGroupProcessor processor;
processor = processorMap.get(filenodeName);
if (processor != null) {
processor.lock(isWriteLock);
@@ -225,8 +218,10 @@ public class FileNodeManager implements IStatistic, IService {
processor.lock(isWriteLock);
} else {
// calculate the value with the key monitor
- LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
- filenodeName, Thread.currentThread().getId());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
+ filenodeName, Thread.currentThread().getId());
+ }
processor = constructNewProcessor(filenodeName);
processor.lock(isWriteLock);
processorMap.put(filenodeName, processor);
@@ -237,361 +232,162 @@ public class FileNodeManager implements IStatistic, IService {
}
/**
- * recovery the filenode processor.
+ * recover the StorageGroupProcessors.
*/
- public void recovery() {
- List<String> filenodeNames = null;
+ public void recover() throws StorageGroupManagerException {
+ List<String> filenodeNames;
try {
- filenodeNames = MManager.getInstance().getAllFileNames();
+ filenodeNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
- LOGGER.error("Restoring all FileNodes failed.", e);
- return;
+ throw new StorageGroupManagerException("Restoring all FileNodes failed.", e);
}
for (String filenodeName : filenodeNames) {
- FileNodeProcessor fileNodeProcessor = null;
+ StorageGroupProcessor fileNodeProcessor = null;
try {
fileNodeProcessor = getProcessor(filenodeName, true);
- if (fileNodeProcessor.shouldRecovery()) {
- LOGGER.info("Recovery the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- fileNodeProcessor.fileNodeRecovery();
- }
- } catch (FileNodeManagerException | FileNodeProcessorException e) {
- LOGGER.error("Restoring fileNode {} failed.", filenodeName, e);
+ } catch (StorageGroupManagerException e) {
+ throw new StorageGroupManagerException(String.format("Restoring fileNode %s failed.",
+ filenodeName), e);
} finally {
if (fileNodeProcessor != null) {
fileNodeProcessor.writeUnlock();
}
}
- // add index check sum
}
}
/**
* insert TsRecord into storage group.
*
- * @param tsRecord input Data
+ * @param plan an insert plan
* @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
* be recorded. if false, the statParamsHashMap will be updated.
- * @return an int value represents the insert type
*/
- public int insert(TSRecord tsRecord, boolean isMonitor) throws FileNodeManagerException {
- long timestamp = tsRecord.time;
+ public void insert(InsertPlan plan, boolean isMonitor) throws StorageGroupManagerException {
+ long timestamp = plan.getTime();
- String deviceId = tsRecord.deviceId;
- checkTimestamp(tsRecord);
- updateStat(isMonitor, tsRecord);
+ String deviceId = plan.getDeviceId();
+ checkTimestamp(timestamp);
+ updateStat(isMonitor, plan);
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- int insertType;
+ StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
- long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
- if (timestamp < lastUpdateTime) {
- insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
- insertType = 1;
- } else {
- insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
- insertType = 2;
- }
- } catch (FileNodeProcessorException e) {
- LOGGER.error(String.format("Encounter an error when closing the buffer write processor %s.",
- fileNodeProcessor.getProcessorName()), e);
- throw new FileNodeManagerException(e);
+ OperationResult result = processor.insert(plan);
+ if (result != OperationResult.WRITE_SUCCESS) {
+ throw new StorageGroupManagerException(String.format("Insertion failed due to %s",
+ result.toString()));
+ }
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String.format("Fail to write in SG %s",
+ processor.getProcessorName()), e);
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
// Modify the insert
if (!isMonitor) {
- fileNodeProcessor.getStatParamsHashMap()
+ processor.getStatParamsHashMap()
.get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
- fileNodeProcessor.getStatParamsHashMap()
+ .addAndGet(plan.getValues().length);
+ processor.getStatParamsHashMap()
.get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_SUCCESS.name())
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_REQ_SUCCESS.name())
.incrementAndGet();
statParamsHashMap
- .get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
+ .get(StorageGroupManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
+ .addAndGet(plan.getValues().length);
}
- return insertType;
}
- private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
- throws FileNodeManagerException {
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- String[] measurementList = new String[tsRecord.dataPointList.size()];
- String[] insertValues = new String[tsRecord.dataPointList.size()];
- int i=0;
- for (DataPoint dp : tsRecord.dataPointList) {
- measurementList[i] = dp.getMeasurementId();
- insertValues[i] = dp.getValue().toString();
- i++;
- }
- logNode.write(new InsertPlan(2, tsRecord.deviceId, tsRecord.time, measurementList,
- insertValues));
- }
- } catch (IOException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- }
-
- private void checkTimestamp(TSRecord tsRecord) throws FileNodeManagerException {
- if (tsRecord.time < 0) {
- LOGGER.error("The insert time lt 0, {}.", tsRecord);
- throw new FileNodeManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
+ private void checkTimestamp(long time) throws StorageGroupManagerException {
+ if (time < 0) {
+ throw new StorageGroupManagerException(String.format("The insert time %s is lt 0 ",
+ time));
}
}
- private void updateStat(boolean isMonitor, TSRecord tsRecord) {
+ private void updateStat(boolean isMonitor, InsertPlan plan) {
if (!isMonitor) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
- }
-
- private void insertOverflow(FileNodeProcessor fileNodeProcessor, long timestamp,
- TSRecord tsRecord, boolean isMonitor, String deviceId)
- throws FileNodeManagerException {
- // get overflow processor
- OverflowProcessor overflowProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- // write wal
- writeLog(tsRecord, isMonitor, overflowProcessor.getLogNode());
- // write overflow data
- try {
- overflowProcessor.insert(tsRecord);
- fileNodeProcessor.changeTypeToChanged(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- } catch (IOException e) {
- LOGGER.error("Insert into overflow error, the reason is {}", e);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
+ statParamsHashMap.get(StorageGroupManagerStatConstants.TOTAL_POINTS.name())
+ .addAndGet(plan.getValues().length);
}
}
- private void insertBufferWrite(FileNodeProcessor fileNodeProcessor, long timestamp,
- boolean isMonitor, TSRecord tsRecord, String deviceId)
- throws FileNodeManagerException, FileNodeProcessorException {
- // get bufferwrite processor
- BufferWriteProcessor bufferWriteProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName, timestamp);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Get the bufferwrite processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- // Add a new interval file to newfilelist
- if (bufferWriteProcessor.isNewProcessor()) {
- bufferWriteProcessor.setNewProcessor(false);
- String bufferwriteBaseDir = bufferWriteProcessor.getBaseDir();
- String bufferwriteRelativePath = bufferWriteProcessor.getFileRelativePath();
- try {
- fileNodeProcessor.addIntervalFileNode(new File(new File(bufferwriteBaseDir), bufferwriteRelativePath));
- } catch (Exception e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- }
- // write wal
- writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
- // Write data
- long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
- long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
-
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId);
- fileNodeProcessor.setLastUpdateTime(deviceId, timestamp);
- try {
- if (!bufferWriteProcessor.write(tsRecord)) {
- // undo time update
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId, prevStartTime);
- fileNodeProcessor.setLastUpdateTime(deviceId, prevUpdateTime);
- }
- } catch (BufferWriteProcessorException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
-
- if (bufferWriteProcessor
- .getFileSize() > IoTDBDescriptor.getInstance()
- .getConfig().getBufferwriteFileSizeThreshold()) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "The filenode processor {} will close the bufferwrite processor, "
- + "because the size[{}] of tsfile {} reaches the threshold {}",
- filenodeName, MemUtils.bytesCntToStr(bufferWriteProcessor.getFileSize()),
- bufferWriteProcessor.getInsertFilePath(), MemUtils.bytesCntToStr(
- IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold()));
- }
-
- fileNodeProcessor.closeBufferWrite();
- }
- }
/**
* update data.
*/
public void update(String deviceId, String measurementId, long startTime, long endTime,
TSDataType type, String v)
- throws FileNodeManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
-
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- if (startTime > lastUpdateTime) {
- LOGGER.warn("The update range is error, startTime {} is great than lastUpdateTime {}",
- startTime,
- lastUpdateTime);
- return;
- }
- long finalEndTime = endTime > lastUpdateTime ? lastUpdateTime : endTime;
-
- String filenodeName = fileNodeProcessor.getProcessorName();
- // get overflow processor
- OverflowProcessor overflowProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error(
- "Get the overflow processor failed, the filenode is {}, "
- + "insert time range is from {} to {}",
- filenodeName, startTime, finalEndTime);
- throw new FileNodeManagerException(e);
- }
- overflowProcessor.update(deviceId, measurementId, startTime, finalEndTime, type, v);
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChanged(deviceId, startTime, finalEndTime);
- fileNodeProcessor.setOverflowed(true);
-
- // write wal
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- overflowProcessor.getLogNode().write(
- new UpdatePlan(startTime, finalEndTime, v, new Path(deviceId
- + "." + measurementId)));
- }
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
+ throws StorageGroupManagerException {
+ throw new UnsupportedOperationException("Method unimplemented");
}
/**
- * delete data.
+ * Delete data whose timestamp <= 'timestamp' of time series 'deviceId'.'measurementId'.
+ * @param deviceId
+ * @param measurementId
+ * @param timestamp
+ * @throws StorageGroupManagerException
*/
- public void delete(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
+ public void deleteProcessor(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
+ StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- // no tsfile data, the delete operation is invalid
+ long lastUpdateTime;
+ try {
+ lastUpdateTime = processor.getLastInsetTime(deviceId);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(e);
+ }
+ // no TsFile data, the delete operation is invalid
if (lastUpdateTime == -1) {
- LOGGER.warn("The last update time is -1, delete overflow is invalid, "
- + "the filenode processor is {}",
- fileNodeProcessor.getProcessorName());
+ LOGGER.debug("The last update time is -1, deletion is invalid, "
+ + "the StorageGroup processor is {}",
+ processor.getProcessorName());
} else {
- // write wal
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // get processors for wal
- String filenodeName = fileNodeProcessor.getProcessorName();
- OverflowProcessor overflowProcessor;
- BufferWriteProcessor bufferWriteProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // in case that no BufferWriteProcessor is available, a new BufferWriteProcessor is
- // needed to access LogNode.
- // TODO this may make the time range of the next TsFile a little wider
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName,
- lastUpdateTime + 1);
- } catch (IOException | FileNodeProcessorException e) {
- LOGGER.error("Getting the processor failed, the filenode is {}, delete time is {}.",
- filenodeName, timestamp);
- throw new FileNodeManagerException(e);
- }
- try {
- overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- }
-
try {
- fileNodeProcessor.delete(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
+ processor.delete(deviceId, measurementId, timestamp);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String.format("Fail to delete %s.%s at %d",
+ deviceId, measurementId, timestamp), e);
}
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
-
}
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
}
- private void delete(String processorName,
+ private void deleteProcessor(String processorName,
Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
- //TODO do we need to call processorIterator.remove() ?
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
+ LOGGER.warn("The processorMap doesn't contain the StorageGroupProcessor {}.", processorName);
return;
}
- LOGGER.info("Try to delete the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
+ LOGGER.info("Try to delete the StorageGroupProcessor {}.", processorName);
+ StorageGroupProcessor processor = processorMap.get(processorName);
if (!processor.tryWriteLock()) {
- throw new FileNodeManagerException(String
- .format("Can't delete the filenode processor %s because Can't get the write lock.",
+ throw new StorageGroupManagerException(String
+ .format("Can't delete the StorageGroupProcessor %s because Can't get the write lock.",
processorName));
}
try {
if (!processor.canBeClosed()) {
- LOGGER.warn("The filenode processor {} can't be deleted.", processorName);
+ LOGGER.warn("The StorageGroupProcessor {} can't be deleted.", processorName);
return;
}
try {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
+ LOGGER.info("Delete the StorageGroupProcessor {}.", processorName);
+ processor.close();
processorIterator.remove();
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} by iterator error.", processorName, e);
- throw new FileNodeManagerException(e);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(String.format("Delete the StorageGroupProcessor %s"
+ + " error.", processorName), e);
}
} finally {
processor.writeUnlock();
@@ -599,109 +395,96 @@ public class FileNodeManager implements IStatistic, IService {
}
/**
- * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
+ * Similar to delete(), but only deletes data in sequence files. Only used by WAL recovery.
*/
- public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
+ public void deleteInSeqFile(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException {
+ StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
- fileNodeProcessor.deleteBufferWrite(deviceId, measurementId, timestamp);
- } catch (BufferWriteProcessorException | IOException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
+ processor.deleteInSeqFile(deviceId, measurementId, timestamp);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(e);
}
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
}
/**
* Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
*/
- public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
+ public void deleteInOverflow(String deviceId, String measurementId, long timestamp)
+ throws StorageGroupManagerException {
+ StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
- fileNodeProcessor.deleteOverflow(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
+ processor.deleteInOverflow(deviceId, measurementId, timestamp);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(e);
}
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
}
/**
- * begin query.
+ * Get a StorageGroup-level token for this query so that the StorageGroupProcessor may know which
+ * queries are occupying resources.
*
* @param deviceId queried deviceId
* @return a query token for the device.
*/
- public int beginQuery(String deviceId) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
+ public int beginQuery(String deviceId) throws StorageGroupManagerException {
+ StorageGroupProcessor processor = getProcessor(deviceId, true);
try {
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, begin query.",
- fileNodeProcessor.getProcessorName());
- return fileNodeProcessor.addMultiPassCount();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Get the FileNodeProcessor: storage group is {}, begin query.",
+ processor.getProcessorName());
+ }
+ return processor.addMultiPassCount();
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
}
/**
- * query data.
+ * Notify the storage group of 'deviceId' that query 'token' has ended and its resource can be
+ * released.
*/
- public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
- throws FileNodeManagerException {
- String deviceId = seriesExpression.getSeriesPath().getDevice();
- String measurementId = seriesExpression.getSeriesPath().getMeasurement();
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, false);
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
- fileNodeProcessor.getProcessorName());
+ public void endQuery(String deviceId, int token) throws StorageGroupManagerException {
+
+ StorageGroupProcessor processorrocessor = getProcessor(deviceId, true);
try {
- QueryDataSource queryDataSource;
- // query operation must have overflow processor
- if (!fileNodeProcessor.hasOverflowProcessor()) {
- try {
- fileNodeProcessor.getOverflowProcessor(fileNodeProcessor.getProcessorName());
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, query is {},{}",
- fileNodeProcessor.getProcessorName(), deviceId, measurementId);
- throw new FileNodeManagerException(e);
- }
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Get the FileNodeProcessor: {} end query.",
+ processorrocessor.getProcessorName());
}
- try {
- queryDataSource = fileNodeProcessor.query(deviceId, measurementId, context);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Query error: the deviceId {}, the measurementId {}", deviceId, measurementId,
- e);
- throw new FileNodeManagerException(e);
- }
- // return query structure
- return queryDataSource;
+ processorrocessor.decreaseMultiPassCount(token);
} finally {
- fileNodeProcessor.readUnlock();
+ processorrocessor.writeUnlock();
}
}
/**
- * end query.
+ * Find sealed files, unsealed file and memtable data in SeqFiles and OverflowFiles that contains the
+ * given series.
+ * @param seriesExpression provides the path of the series.
+ * @param context provides shared modifications across a query.
+ * @return sealed files, unsealed file and memtable data in SeqFiles or OverflowFiles
+ * @throws StorageGroupManagerException
*/
- public void endQuery(String deviceId, int token) throws FileNodeManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
+ public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
+ throws StorageGroupManagerException {
+ String deviceId = seriesExpression.getSeriesPath().getDevice();
+ StorageGroupProcessor processor = getProcessor(deviceId, false);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
+ processor.getProcessorName());
+ }
try {
- LOGGER.debug("Get the FileNodeProcessor: {} end query.",
- fileNodeProcessor.getProcessorName());
- fileNodeProcessor.decreaseMultiPassCount(token);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Failed to end query: the deviceId {}, token {}.", deviceId, token, e);
- throw new FileNodeManagerException(e);
+ QueryDataSource queryDataSource;
+ try {
+ queryDataSource = processor.query(seriesExpression, context);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupManagerException(e);
+ }
+ // return query structure
+ return queryDataSource;
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.readUnlock();
}
}
@@ -713,24 +496,24 @@ public class FileNodeManager implements IStatistic, IService {
* @param appendFile the appended tsfile information
*/
public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
- String appendFilePath) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
+ String appendFilePath) throws StorageGroupManagerException {
+ StorageGroupProcessor processor = getProcessor(fileNodeName, true);
try {
// check append file
for (Map.Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- if (fileNodeProcessor.getLastUpdateTime(entry.getKey()) >= entry.getValue()) {
+ if (processor.getLastInsetTime(entry.getKey()) >= entry.getValue()) {
return false;
}
}
// close bufferwrite file
- fileNodeProcessor.closeBufferWrite();
+ processor.close();
// append file to storage group.
- fileNodeProcessor.appendFile(appendFile, appendFilePath);
- } catch (FileNodeProcessorException e) {
+ processor.appendFile(appendFile, appendFilePath);
+ } catch (TsFileProcessorException e) {
LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
- fileNodeProcessor.writeUnlock();
+ processor.writeUnlock();
}
return true;
}
@@ -742,13 +525,13 @@ public class FileNodeManager implements IStatistic, IService {
* @param appendFile the appended tsfile information
*/
public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
- String uuid) throws FileNodeManagerException {
+ String uuid) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
List<String> overlapFiles;
try {
overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
} catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -758,9 +541,9 @@ public class FileNodeManager implements IStatistic, IService {
/**
* merge all overflowed filenode.
*
- * @throws FileNodeManagerException FileNodeManagerException
+ * @throws StorageGroupManagerException StorageGroupManagerException
*/
- public void mergeAll() throws FileNodeManagerException {
+ public void mergeAll() throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
fileNodeManagerStatus);
@@ -771,10 +554,10 @@ public class FileNodeManager implements IStatistic, IService {
LOGGER.info("Start to merge all overflowed filenode");
List<String> allFileNodeNames;
try {
- allFileNodeNames = MManager.getInstance().getAllFileNames();
+ allFileNodeNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
LOGGER.error("Get all storage group seriesPath error,", e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
List<Future<?>> futureTasks = new ArrayList<>();
for (String fileNodeName : allFileNodeNames) {
@@ -820,7 +603,7 @@ public class FileNodeManager implements IStatistic, IService {
}
if (!mergeException.isEmpty()) {
// just throw the first exception
- throw new FileNodeManagerException(mergeException.get(0));
+ throw new StorageGroupManagerException(mergeException.get(0));
}
fileNodeManagerStatus = FileNodeManagerStatus.NONE;
LOGGER.info("End to merge all overflowed filenode");
@@ -833,7 +616,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* try to close the filenode processor. The name of filenode processor is processorName
*/
- private boolean closeOneProcessor(String processorName) throws FileNodeManagerException {
+ private boolean closeOneProcessor(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
return true;
}
@@ -849,7 +632,7 @@ public class FileNodeManager implements IStatistic, IService {
}
} catch (ProcessorException e) {
LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
}
@@ -861,7 +644,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* delete one filenode.
*/
- public void deleteOneFileNode(String processorName) throws FileNodeManagerException {
+ public void deleteOneFileNode(String processorName) throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
return;
}
@@ -883,7 +666,7 @@ public class FileNodeManager implements IStatistic, IService {
.deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
} catch (IOException e) {
LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeManagerStatus = FileNodeManagerStatus.NONE;
}
@@ -908,7 +691,7 @@ public class FileNodeManager implements IStatistic, IService {
}
}
- private void deleteFileNodeBlocked(String processorName) throws FileNodeManagerException {
+ private void deleteFileNodeBlocked(String processorName) throws StorageGroupManagerException {
LOGGER.info("Forced to delete the filenode processor {}", processorName);
FileNodeProcessor processor = processorMap.get(processorName);
while (true) {
@@ -927,7 +710,7 @@ public class FileNodeManager implements IStatistic, IService {
}
} catch (ProcessorException e) {
LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
processor.writeUnlock();
}
@@ -960,7 +743,7 @@ public class FileNodeManager implements IStatistic, IService {
*/
public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor,
- Map<String, String> props) throws FileNodeManagerException {
+ Map<String, String> props) throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
try {
fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
@@ -973,7 +756,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* Force to close the filenode processor.
*/
- public void closeOneFileNode(String processorName) throws FileNodeManagerException {
+ public void closeOneFileNode(String processorName) throws StorageGroupManagerException {
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
return;
}
@@ -1000,7 +783,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* try to close the filenode processor.
*/
- private void close(String processorName) throws FileNodeManagerException {
+ private void close(String processorName) throws StorageGroupManagerException {
if (!processorMap.containsKey(processorName)) {
LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
return;
@@ -1018,7 +801,7 @@ public class FileNodeManager implements IStatistic, IService {
processor.close();
} catch (ProcessorException e) {
LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} else {
LOGGER.warn("The filenode processor {} can't be closed.", processorName);
@@ -1031,7 +814,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* delete all filenode.
*/
- public synchronized boolean deleteAll() throws FileNodeManagerException {
+ public synchronized boolean deleteAll() throws StorageGroupManagerException {
LOGGER.info("Start deleting all filenode");
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.info("Failed to delete all filenode processor because of merge operation");
@@ -1044,7 +827,7 @@ public class FileNodeManager implements IStatistic, IService {
.iterator();
while (processorIterator.hasNext()) {
Map.Entry<String, FileNodeProcessor> processorEntry = processorIterator.next();
- delete(processorEntry.getKey(), processorIterator);
+ deleteProcessor(processorEntry.getKey(), processorIterator);
}
return processorMap.isEmpty();
} finally {
@@ -1056,7 +839,7 @@ public class FileNodeManager implements IStatistic, IService {
/**
* Try to close All.
*/
- public void closeAll() throws FileNodeManagerException {
+ public void closeAll() throws StorageGroupManagerException {
LOGGER.info("Start closing all filenode processor");
if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
LOGGER.info("Failed to close all filenode processor because of merge operation");
@@ -1165,7 +948,7 @@ public class FileNodeManager implements IStatistic, IService {
public void stop() {
try {
closeAll();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("Failed to close file node manager because .", e);
}
}
@@ -1191,14 +974,14 @@ public class FileNodeManager implements IStatistic, IService {
* recover filenode.
*/
public void recoverFileNode(String filenodeName)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
FileNodeProcessor fileNodeProcessor = getProcessor(filenodeName, true);
LOGGER.info("Recover the filenode processor, the filenode is {}, the status is {}",
filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
try {
fileNodeProcessor.fileNodeRecovery();
} catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
} finally {
fileNodeProcessor.writeUnlock();
}
@@ -1213,7 +996,7 @@ public class FileNodeManager implements IStatistic, IService {
private FileNodeManagerHolder() {
}
- private static final FileNodeManager INSTANCE = new FileNodeManager(
+ private static final StorageGroupManager INSTANCE = new StorageGroupManager(
TsFileDBConf.getFileNodeDir());
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
index 07674f5..77fc48d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/sgmanager/StorageGroupProcessor.java
@@ -19,21 +19,45 @@
package org.apache.iotdb.db.engine.sgmanager;
+import static java.time.ZonedDateTime.ofInstant;
+
import java.io.File;
import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.Processor;
+import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
+import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.engine.merge.MergeTask;
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.overflowdata.OverflowProcessor;
+import org.apache.iotdb.db.engine.pool.MergeManager;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
+import org.apache.iotdb.db.exception.PathErrorException;
+import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.IStatistic;
@@ -42,34 +66,80 @@ import org.apache.iotdb.db.monitor.StatMonitor;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
+import org.apache.iotdb.db.query.reader.IReader;
+import org.apache.iotdb.db.utils.ImmediateFuture;
+import org.apache.iotdb.db.utils.MemUtils;
+import org.apache.iotdb.db.utils.TimeValuePair;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
+import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StorageGroupProcessor extends Processor implements IStatistic {
private static final Logger LOGGER = LoggerFactory.getLogger(StorageGroupProcessor.class);
+ private final String statStorageGroupName;
private TsFileProcessor tsFileProcessor;
private OverflowProcessor overflowProcessor;
- //the version controller is shared by tsfile and overflow processor.
+ private FileSchema fileSchema;
private VersionController versionController;
+ private long lastMergeTime;
+
+ /**
+ * oldMultiPassTokenSet records how many queries are progressing before the last merge starts.
+ */
+ private Set<Integer> oldMultiPassTokenSet = null;
+
+ /**
+ * newMultiPassTokenSet records how many queries are progressing since the last merge starts.
+ */
+ private Set<Integer> newMultiPassTokenSet = new HashSet<>();
+
+ /**
+ * newMultiPassCount assign an Id to each query to track whether this query has ended and its
+ * resource can be released (whether files can be removed).
+ */
+ private AtomicInteger newMultiPassCount = new AtomicInteger(0);
+
+ private boolean isMerging = false;
+
+ /**
+ * This is the modification file of the result of the current merge.
+ */
+ private ModificationFile mergingModification;
+
+ private TsFileIOWriter mergeFileWriter = null;
+ private String mergeOutputPath = null;
+ private String mergeBaseDir = null;
+ private String mergeFileName = null;
+ private boolean mergeIsChunkGroupHasData = false;
+ private long mergeStartPos;
- private FileSchema fileSchema;
/**
- * Construct processor using name space seriesPath
+ * Construct processor using StorageGroup name
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
- public StorageGroupProcessor(String processorName)
- throws IOException, WriteProcessException, TsFileProcessorException {
+ StorageGroupProcessor(String processorName)
+ throws TsFileProcessorException {
super(processorName);
- this.fileSchema = constructFileSchema(processorName);
+ fileSchema = constructFileSchema(processorName);
File systemFolder = new File(IoTDBDescriptor.getInstance().getConfig().getFileNodeDir(),
processorName);
@@ -78,10 +148,21 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
LOGGER.info("The directory of the filenode processor {} doesn't exist. Create new directory {}",
getProcessorName(), systemFolder.getAbsolutePath());
}
- versionController = new SimpleFileVersionController(systemFolder.getAbsolutePath());
+ //the version controller is shared by tsfile and overflow processor.
+ try {
+ versionController = new SimpleFileVersionController(systemFolder.getAbsolutePath());
+ } catch (IOException e) {
+ throw new TsFileProcessorException(String.format("Unable to create VersionController for %s",
+ processorName), e);
+ }
tsFileProcessor = new TsFileProcessor(processorName, versionController, fileSchema);
overflowProcessor = new OverflowProcessor(processorName, versionController, fileSchema);
+ statStorageGroupName =
+ MonitorConstants.STAT_STORAGE_GROUP_PREFIX + MonitorConstants.MONITOR_PATH_SEPARATOR
+ + MonitorConstants.FILE_NODE_PATH + MonitorConstants.MONITOR_PATH_SEPARATOR
+ + processorName.replaceAll("\\.", "_");
+
// RegisterStatService
if (IoTDBDescriptor.getInstance().getConfig().isEnableStatMonitor()) {
String statStorageDeltaName =
@@ -94,7 +175,7 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
}
}
- public OperationResult insert(InsertPlan insertPlan) throws IOException, TsFileProcessorException {
+ public OperationResult insert(InsertPlan insertPlan) throws TsFileProcessorException {
OperationResult result = getTsFileProcessor().insert(insertPlan);
if (result == OperationResult.WRITE_REJECT_BY_TIME) {
result = getOverflowProcessor().insert(insertPlan);
@@ -109,15 +190,15 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
public void delete(String device, String measurementId, long timestamp)
throws TsFileProcessorException {
- try {
- getTsFileProcessor().delete(device, measurementId, timestamp);
- } catch (IOException e) {
- throw new TsFileProcessorException(e);
- }
- try {
- getOverflowProcessor().delete(device, measurementId, timestamp);
- } catch (IOException e) {
- throw new TsFileProcessorException(e);
+ deleteInOverflow(device, measurementId, timestamp);
+ deleteInSeqFile(device, measurementId, timestamp);
+ if (mergingModification != null) {
+ try {
+ mergingModification.write(new Deletion(device + IoTDBConstant.PATH_SEPARATOR + measurementId,
+ versionController.nextVersion(), timestamp));
+ } catch (IOException e) {
+ throw new TsFileProcessorException(e);
+ }
}
}
@@ -126,65 +207,85 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
*/
public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
throws TsFileProcessorException {
+ SeriesDataSource tsfileData;
+ SeriesDataSource overflowData;
try {
- SeriesDataSource tsfileData = getTsFileProcessor().query(seriesExpression, context);
+ tsfileData = getTsFileProcessor().query(seriesExpression, context);
} catch (IOException e) {
throw new TsFileProcessorException(e);
}
try {
- SeriesDataSource overflowData = getOverflowProcessor().query(seriesExpression, context);
+ overflowData = getOverflowProcessor().query(seriesExpression, context);
} catch (IOException e) {
throw new TsFileProcessorException(e);
}
- return null;
- }
-
-
- public void addExternalTsFile() {
-
- }
-
- public void addExternalOverflowFile() {
-
+ return new QueryDataSource(tsfileData, overflowData);
}
@Override
public boolean canBeClosed() {
- tsFileProcessor.canBeClosed();
- return false;
+ return tsFileProcessor.canBeClosed() && overflowProcessor.canBeClosed();
}
@Override
- public Future<Boolean> flush() {
- return null;
+ public Future<Boolean> flush() throws IOException {
+ Future<Boolean> tsFileFuture = tsFileProcessor.flush();
+ Future<Boolean> overflowFuture = overflowProcessor.flush();
+
+ return new StorageGroupFlushFuture(tsFileFuture, overflowFuture);
}
@Override
- public void close() {
-
+ public void close() throws TsFileProcessorException {
+ tsFileProcessor.close();
+ overflowProcessor.close();
}
@Override
public long memoryUsage() {
- return 0;
+ return tsFileProcessor.memoryUsage() + overflowProcessor.memoryUsage();
}
@Override
public Map<String, TSRecord> getAllStatisticsValue() {
- return null;
+ Long curTime = System.currentTimeMillis();
+ HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
+ TSRecord tsRecord = new TSRecord(curTime, statStorageGroupName);
+
+ Map<String, AtomicLong> hashMap = getStatParamsHashMap();
+ tsRecord.dataPointList = new ArrayList<>();
+ for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
+ tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(), entry.getValue().get()));
+ }
+
+ tsRecordHashMap.put(statStorageGroupName, tsRecord);
+ return tsRecordHashMap;
}
@Override
public void registerStatMetadata() {
-
+ Map<String, String> hashMap = new HashMap<>();
+ for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
+ MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ hashMap
+ .put(statStorageGroupName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
+ MonitorConstants.DATA_TYPE_INT64);
+ }
+ StatMonitor.getInstance().registerStatStorageGroup(hashMap);
}
@Override
public List<String> getAllPathForStatistic() {
- return null;
+ List<String> list = new ArrayList<>();
+ for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
+ MonitorConstants.FileNodeProcessorStatConstants.values()) {
+ list.add(
+ statStorageGroupName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
+ }
+ return list;
}
@Override
@@ -204,17 +305,622 @@ public class StorageGroupProcessor extends Processor implements IStatistic {
}
- public TsFileProcessor getTsFileProcessor() throws TsFileProcessorException {
+ private TsFileProcessor getTsFileProcessor() throws TsFileProcessorException {
if (tsFileProcessor.isClosed()) {
tsFileProcessor.reopen();
}
return tsFileProcessor;
}
- public OverflowProcessor getOverflowProcessor() throws TsFileProcessorException {
+ private OverflowProcessor getOverflowProcessor() throws TsFileProcessorException {
if (overflowProcessor.isClosed()) {
overflowProcessor.reopen();
}
return overflowProcessor;
}
+
+ public class StorageGroupFlushFuture implements Future<Boolean>{
+ private Future<Boolean> tsFileFuture;
+ private Future<Boolean> overflowFuture;
+
+ StorageGroupFlushFuture(Future<Boolean> tsFileFuture, Future<Boolean> overflowFuture) {
+ this.tsFileFuture = tsFileFuture != null ? tsFileFuture : new ImmediateFuture<>(true);
+ this.overflowFuture = overflowFuture != null ? tsFileFuture : new ImmediateFuture<>(true);
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ boolean tsCanceled = tsFileFuture.cancel(mayInterruptIfRunning);
+ boolean overflowCanceled = overflowFuture.cancel(mayInterruptIfRunning);
+ return tsCanceled && overflowCanceled;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return tsFileFuture.isCancelled() && overflowFuture.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return tsFileFuture.isDone() && overflowFuture.isDone();
+ }
+
+ @Override
+ public Boolean get() throws InterruptedException, ExecutionException {
+ boolean tsFileOK = tsFileFuture.get();
+ boolean overflowOK = overflowFuture.get();
+ return tsFileOK && overflowOK;
+ }
+
+ @Override
+ public Boolean get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ boolean tsFileOK = tsFileFuture.get(timeout, unit);
+ boolean overflowOK = overflowFuture.get(timeout, unit);
+ return tsFileOK && overflowOK;
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return this == o;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ long getLastInsetTime(String deviceId) throws TsFileProcessorException {
+ return getTsFileProcessor().getLastInsertTime(deviceId);
+ }
+
+ void deleteInSeqFile(String deviceId, String measurementId, long timestamp)
+ throws TsFileProcessorException {
+ try {
+ getTsFileProcessor().delete(deviceId, measurementId, timestamp);
+ } catch (IOException e) {
+ throw new TsFileProcessorException(e);
+ }
+ }
+
+ void deleteInOverflow(String deviceId, String measurementId, long timestamp)
+ throws TsFileProcessorException {
+ try {
+ getOverflowProcessor().delete(deviceId, measurementId, timestamp);
+ } catch (IOException e) {
+ throw new TsFileProcessorException(e);
+ }
+ }
+
+ /**
+ * Assign a token to a new query and put it into the newSet so that we know which
+ */
+ public int addMultiPassCount() {
+ LOGGER.debug("Add MultiPassCount: read lock newMultiPassCount.");
+ Integer token = newMultiPassCount.incrementAndGet();
+
+ newMultiPassTokenSet.add(token);
+ LOGGER.debug("Add multi token:{}, nsPath:{}.", token, getProcessorName());
+ return token;
+ }
+
+ /**
+ * decrease multiple pass count.
+ */
+ public void decreaseMultiPassCount(int token) {
+ if (newMultiPassTokenSet.contains(token)) {
+ newMultiPassTokenSet.remove(token);
+ LOGGER.debug("Remove multi token:{}, nspath:{}, new set:{}, count:{}", token,
+ getProcessorName(),
+ newMultiPassTokenSet, newMultiPassCount);
+ } else if (oldMultiPassTokenSet != null && oldMultiPassTokenSet.contains(token)) {
+ // remove token first, then unlock
+ oldMultiPassTokenSet.remove(token);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Remove multi token:{}, old set:{}", token, oldMultiPassTokenSet);
+ }
+ } else {
+ LOGGER.error("remove token error:{},new set:{}, old set:{}", token, newMultiPassTokenSet,
+ oldMultiPassTokenSet);
+ // should add throw exception
+ }
+ }
+
+ /**
+ * append one specified TsFile to this StorageGroup.
+ *
+ * @param destFileResource the information of destination file
+ * @param srcFilePath the path of the source file
+ */
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ public void appendFile(TsFileResource destFileResource, String srcFilePath)
+ throws TsFileProcessorException {
+ try {
+ if (!destFileResource.getFile().getParentFile().exists()) {
+ destFileResource.getFile().getParentFile().mkdirs();
+ }
+ // move file
+ File srcFile = new File(srcFilePath);
+ File destFile = destFileResource.getFile();
+ if (!srcFile.exists()) {
+ throw new FileNodeProcessorException(
+ String.format("The source file %s does not exist.", srcFilePath));
+ }
+ if (destFile.exists()) {
+ throw new FileNodeProcessorException(
+ String.format("The destination file %s already exists.",
+ destFileResource.getFile().getAbsolutePath()));
+ }
+ if (!srcFile.renameTo(destFile)) {
+ LOGGER.warn("File renaming failed when appending new file. Origin: {}, Target: {}",
+ srcFile.getPath(), destFile.getPath());
+ }
+ // append the new tsfile
+ this.tsFileProcessor.appendFile(destFileResource);
+ // reconstruct the inverted index of the newFileNodes
+ } catch (Exception e) {
+ LOGGER.error("Failed to append the tsfile {} to filenode processor {}.", destFileResource,
+ getProcessorName());
+ throw new TsFileProcessorException(e);
+ }
+ }
+
+ /**
+ * get overlap tsfiles which are conflict with the appendFile.
+ *
+ * @param appendFile the appended tsfile information
+ */
+ public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
+ throws FileNodeProcessorException {
+ return tsFileProcessor.getOverlapFiles(appendFile, uuid);
+ }
+
+ /**
+ * register a timeseries in this StorageGroup
+ * @param measurementId
+ * @param dataType
+ * @param encoding
+ * @param compressor
+ * @param props
+ */
+ public void addTimeSeries(String measurementId, TSDataType dataType, TSEncoding encoding,
+ CompressionType compressor, Map<String, String> props) {
+ fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
+ compressor, props));
+ }
+
+ /**
+ * submit the merge task to the <code>MergePool</code>.
+ *
+ * @return null -can't submit the merge task, because this filenode is not overflowed or it is
+ * merging now. Future - submit the merge task successfully.
+ */
+ Future submitToMerge() {
+ ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
+ if (lastMergeTime > 0 && LOGGER.isInfoEnabled()) {
+ long thisMergeTime = System.currentTimeMillis();
+ long mergeTimeInterval = thisMergeTime - lastMergeTime;
+ LOGGER.info(
+ "The filenode {} last merge time is {}, this merge time is {}, "
+ + "merge time interval is {}s",
+ processorName, ofInstant(Instant.ofEpochMilli(lastMergeTime),
+ zoneId), ofInstant(Instant.ofEpochMilli(thisMergeTime),
+ zoneId), mergeTimeInterval / 1000);
+ }
+ lastMergeTime = System.currentTimeMillis();
+
+ if (overflowProcessor.totalFileSize() < IoTDBDescriptor.getInstance()
+ .getConfig().getOverflowFileSizeThreshold()) {
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(
+ "Skip this merge taks submission, because the total file size {} of overflow processor {} "
+ + "does not reaches the threshold {}.",
+ MemUtils.bytesCntToStr(overflowProcessor.totalFileSize()), getProcessorName(),
+ MemUtils.bytesCntToStr(
+ IoTDBDescriptor.getInstance().getConfig().getOverflowFileSizeThreshold()));
+ }
+ return null;
+ }
+
+ if (!isMerging) {
+ Callable<Exception> mergeThread;
+ mergeThread = new MergeTask(this);
+ LOGGER.info("Submit the merge task, the merge filenode is {}", processorName);
+ return MergeManager.getInstance().submit(mergeThread);
+ } else {
+ LOGGER.warn(
+ "Skip this merge task submission, because last merge task is not over yet, "
+ + "the merge filenode processor is {}",
+ processorName);
+ }
+ return null;
+ }
+
+ /**
+ * Divide seqFiles and unseqFiles into k groups separately such that seqFiles in the ith group
+ * overlp some unseqFiles in the ith group but overlap none unseqFiles in other groups.
+ * @param seqFiles
+ * @param unseqFiles
+ */
+ private Pair<List<List<TsFileResource>>, List<List<TsFileResource>>> groupMergeFiles(
+ List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles) {
+ List<List<TsFileResource>> seqGroups = new ArrayList<>();
+ List<List<TsFileResource>> unseqGroups = new ArrayList<>();
+ boolean[] seqVisited = new boolean[seqFiles.size()];
+ boolean[] unseqVisited = new boolean[unseqFiles.size()];
+ for (int i = 0; i < seqFiles.size(); i++) {
+ seqVisited[i] = false;
+ }
+ for (int i = 0; i < unseqFiles.size(); i++) {
+ unseqVisited[i] = false;
+ }
+
+ for (int i = 0; i < seqFiles.size(); i++) {
+ if (seqVisited[i]) {
+ continue;
+ }
+
+ List<TsFileResource> currSeqGroup = new ArrayList<>();
+ List<TsFileResource> currUnseqGroup = new ArrayList<>();
+
+ // we only need to check new files (range [lastIndex:]) added by last iteration,
+ // older files can be skipped
+ int lastSeqIndex = 0;
+ int lastUnseqIndex = 0;
+
+ currSeqGroup.add(seqFiles.get(i));
+ boolean updated = true;
+ while (updated) {
+ // find unseqFiles that overlap current seqFiles
+ updated = searchOverlap(unseqFiles, unseqVisited, currSeqGroup,
+ lastSeqIndex, currUnseqGroup);
+ lastSeqIndex = currSeqGroup.size();
+ // find seqFiles that overlap current unseqFiles
+ updated = updated || searchOverlap(seqFiles, seqVisited, currUnseqGroup,
+ lastUnseqIndex, currSeqGroup);
+ lastUnseqIndex = currUnseqGroup.size();
+ }
+
+ if (currUnseqGroup.size() > 0) {
+ seqGroups.add(currSeqGroup);
+ unseqGroups.add(currUnseqGroup);
+ }
+ }
+ return new Pair<>(seqGroups, unseqGroups);
+ }
+
+ /**
+ * For each TsFile in 'candidates', ignoring the ith element if 'candidatesVisited[i]' == true,
+ * find out whether it overlaps another TsFile in 'targets[offset:]' and add the candidate
+ * into result.
+ * @param candidates
+ * @param candidatesVisited
+ * @param targets
+ * @param offset
+ * @param result
+ * @return
+ */
+ private boolean searchOverlap(List<TsFileResource> candidates, boolean[] candidatesVisited,
+ List<TsFileResource> targets, int offset, List<TsFileResource> result) {
+ boolean found = false;
+ for (int j = 0; j < candidates.size(); j++) {
+ if (candidatesVisited[j]) {
+ continue;
+ }
+ TsFileResource candidate = candidates.get(j);
+ for (int k = offset; k < targets.size(); k++) {
+ if (candidate.overlaps(targets.get(k))) {
+ result.add(candidate);
+ candidatesVisited[j] = true;
+ found = true;
+ }
+ }
+ }
+ return found;
+ }
+
+ /**
+ * Merge this storage group, merge the tsfile data with overflow data.
+ */
+ public void merge() throws StorageGroupProcessorException {
+ writeLock();
+ Pair<List<List<TsFileResource>>, List<List<TsFileResource>>> fileGroups;
+ try {
+ // close seqFile and overflow(unseqFile), prepare for merge
+ LOGGER.info("The StorageGroupProcessor {} begins to merge.", processorName);
+ close();
+ // change status from work to merge
+ isMerging = true;
+
+ // find what files are to be processed in this merge
+ List<TsFileResource> seqFiles = tsFileProcessor.getTsFileResources();
+ List<TsFileResource> unseqFiles = overflowProcessor.getTsFileResources();
+
+ fileGroups = groupMergeFiles(seqFiles, unseqFiles);
+ } catch (TsFileProcessorException e) {
+ throw new StorageGroupProcessorException(e);
+ } finally {
+ writeUnlock();
+ isMerging = false;
+ }
+
+ List<List<TsFileResource>> seqGroups = fileGroups.left;
+ List<List<TsFileResource>> unseqGroups = fileGroups.right;
+
+ LOGGER.info("{}: There are {} groups to be merged", processorName, seqGroups.size());
+
+ for (int i = 0; i < seqGroups.size(); i++) {
+ List<TsFileResource> seqGroup = seqGroups.get(i);
+ List<TsFileResource> unseqGroup = unseqGroups.get(i);
+
+ int seqFileSize = seqGroup.size();
+ int unseqFileSize = unseqGroup.size();
+
+ long startTime = System.currentTimeMillis();
+ LOGGER.info("{}: Starting merging the {} group, there are {} seq groups and {} unseq groups",
+ processorName, i, seqFileSize, unseqFileSize);
+ TsFileResource newResource = mergeFileGroup(seqGroup, unseqGroup);
+ long endTime = System.currentTimeMillis();
+ }
+
+
+ // change status from merge to wait
+ switchMergeToWaiting(backupIntervalFiles, needEmtpy);
+
+ // change status from wait to work
+ switchWaitingToWorking();
+ }
+
+ private TsFileResource mergeFileGroup(List<TsFileResource> seqGroup, List<TsFileResource> unseqGroup)
+ throws StorageGroupProcessorException {
+ // find all timeseries (possibly) of each device (the key) contained in these files.
+ Map<String, List<Path>> pathMap = null;
+ try {
+ pathMap = resolveMergePaths(seqGroup, unseqGroup);
+ } catch (PathErrorException e) {
+ throw new StorageGroupProcessorException(e);
+ }
+ // min data time of all files
+ long minimumTime = resolveMinimumTime(seqGroup, unseqGroup);
+
+ mergeBaseDir = Directories.getInstance().getNextFolderForTsfile();
+ mergeFileName = minimumTime
+ + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR + System.currentTimeMillis();
+ mergeOutputPath = constructOutputFilePath(mergeBaseDir, getProcessorName(),
+ mergeFileName);
+ mergeFileName = getProcessorName() + File.separatorChar + mergeFileName;
+ try {
+ mergeFileWriter = new TsFileIOWriter(new File(mergeOutputPath));
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
+ mergingModification = new ModificationFile(mergeOutputPath
+ + ModificationFile.FILE_SUFFIX);
+ mergeIsChunkGroupHasData = false;
+ mergeStartPos = -1;
+
+ Map<String, Long> startTimeMap = new HashMap<>();
+ Map<String, Long> endTimeMap = new HashMap<>();
+
+ QueryContext context = new QueryContext();
+ IReader seriesReader = null;
+ ChunkGroupFooter footer;
+ int numOfChunks = 0;
+ for (Entry<String, List<Path>> entry : pathMap.entrySet()) {
+ // start writing a device
+ String deviceId = entry.getKey();
+ List<Path> paths = entry.getValue();
+
+ for (Path path : paths) {
+ // start writing a series
+ try {
+ // sequence
+ SeriesDataSource seqSource = new SeriesDataSource(path, seqGroup, null, null);
+ SeriesDataSource unseqSource = new SeriesDataSource(path, unseqGroup, null, null);
+
+ seriesReader = SeriesReaderFactory.getInstance().createSeriesReaderForMerge(seqSource,
+ unseqSource, context);
+ TSDataType dataType = MManager.getInstance().getSeriesType(path.getFullPath());
+ // end writing a series
+ numOfChunks += queryAndWriteSeries(seriesReader, path, null, dataType,
+ startTimeMap, endTimeMap);
+ } catch (PathErrorException | IOException e) {
+ LOGGER.error("{}: error occurred when merging {}", processorName, path.getFullPath(), e);
+ } finally {
+ if (seriesReader != null) {
+ try {
+ seriesReader.close();
+ } catch (IOException e) {
+ LOGGER.error("{}: cannot close series reader of {} when merge", processorName,
+ path.getFullPath(), e);
+ }
+ }
+ }
+ }
+
+ // end writing a device
+ try {
+ if (mergeIsChunkGroupHasData) {
+ // end the new rowGroupMetadata
+ long size = mergeFileWriter.getPos() - mergeStartPos;
+ footer = new ChunkGroupFooter(deviceId, size, numOfChunks);
+ mergeFileWriter.endChunkGroup(footer, 0);
+ }
+ } catch (IOException e) {
+ LOGGER.error("{}: Cannot end ChunkGroup of {} when merge", processorName, deviceId, e);
+ }
+
+ }
+
+ try {
+ mergeFileWriter.endFile(fileSchema);
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
+ TsFileResource newResource;
+ try {
+ newResource = new TsFileResource(
+ new File(mergeBaseDir + File.separator + mergeFileName), false);
+ } catch (IOException e) {
+ throw new StorageGroupProcessorException(e);
+ }
+ newResource.setStartTimeMap(startTimeMap);
+ newResource.setEndTimeMap(endTimeMap);
+ newResource.setModFile(mergingModification);
+ mergingModification = null;
+ mergeFileWriter = null;
+ mergeOutputPath = null;
+ mergeBaseDir = null;
+ mergeFileName = null;
+ return newResource;
+ }
+
+ private Map<String, List<Path>> resolveMergePaths(List<TsFileResource> seqGroup, List<TsFileResource> unseqGroup)
+ throws PathErrorException {
+ Map<String, List<Path>> paths = new HashMap<>();
+
+ resolveMergeGroupPaths(seqGroup, paths);
+ resolveMergeGroupPaths(unseqGroup, paths);
+
+ return paths;
+ }
+
+ private void resolveMergeGroupPaths(List<TsFileResource> fileGroup,
+ Map<String, List<Path>> paths) throws PathErrorException {
+ for (TsFileResource tsFile : fileGroup) {
+ for (String device : tsFile.getDevices()) {
+ if (paths.containsKey(device)) {
+ continue;
+ }
+ List<String> pathStrings = MManager.getInstance().getLeafNodePathInNextLevel(device);
+ for (String string : pathStrings) {
+ paths.computeIfAbsent(device, x -> new ArrayList<>()).add(new Path(string));
+ }
+ }
+ }
+ }
+
+ private long resolveMinimumTime(List<TsFileResource> seqGroup, List<TsFileResource> unseqGroup) {
+ long minTime = Long.MAX_VALUE;
+ for (TsFileResource tsFile : seqGroup) {
+ for (Long startTime : tsFile.getStartTimeMap().values()) {
+ if (startTime < minTime) {
+ minTime = startTime;
+ }
+ }
+ }
+ for (TsFileResource tsFile : unseqGroup) {
+ for (Long startTime : tsFile.getStartTimeMap().values()) {
+ if (startTime < minTime) {
+ minTime = startTime;
+ }
+ }
+ }
+ return minTime;
+ }
+
+ private String constructOutputFilePath(String baseDir, String processorName, String fileName) {
+
+ String localBaseDir = baseDir;
+ if (localBaseDir.charAt(localBaseDir.length() - 1) != File.separatorChar) {
+ localBaseDir = localBaseDir + File.separatorChar + processorName;
+ }
+ File dataDir = new File(localBaseDir);
+ if (!dataDir.exists()) {
+ LOGGER.warn("The bufferwrite processor data dir doesn't exists, create new directory {}",
+ localBaseDir);
+ dataDir.mkdirs();
+ }
+ File outputFile = new File(dataDir, fileName);
+ return outputFile.getPath();
+ }
+
+ private int queryAndWriteSeries(IReader seriesReader, Path path,
+ SingleSeriesExpression seriesFilter, TSDataType dataType,
+ Map<String, Long> startTimeMap, Map<String, Long> endTimeMap)
+ throws IOException {
+ int numOfChunk = 0;
+ if (!seriesReader.hasNext()) {
+ LOGGER.debug(
+ "The time-series {} has no data with the filter {} in the filenode processor {}",
+ path, seriesFilter, getProcessorName());
+ } else {
+ numOfChunk ++;
+ TimeValuePair timeValuePair = seriesReader.next();
+ if (!mergeIsChunkGroupHasData) {
+ // start a new rowGroupMetadata
+ mergeIsChunkGroupHasData = true;
+ // the datasize and numOfChunk is fake
+ // the accurate datasize and numOfChunk will get after write all this device data.
+ mergeFileWriter.startFlushChunkGroup(path.getDevice());
+ mergeStartPos = mergeFileWriter.getPos();
+ }
+ // init the serieswWriteImpl
+ MeasurementSchema measurementSchema = fileSchema
+ .getMeasurementSchema(path.getMeasurement());
+ ChunkBuffer pageWriter = new ChunkBuffer(measurementSchema);
+ int pageSizeThreshold = TSFileConfig.pageSizeInByte;
+ ChunkWriterImpl seriesWriterImpl = new ChunkWriterImpl(measurementSchema, pageWriter,
+ pageSizeThreshold);
+ // write the series data
+ writeOneSeries(path.getDevice(), seriesWriterImpl, dataType,
+ seriesReader, startTimeMap, endTimeMap, timeValuePair);
+ // flush the series data
+ seriesWriterImpl.writeToFileWriter(mergeFileWriter);
+ }
+ return numOfChunk;
+ }
+
+ private void writeOneSeries(String deviceId, ChunkWriterImpl seriesWriterImpl,
+ TSDataType dataType, IReader seriesReader, Map<String, Long> startTimeMap,
+ Map<String, Long> endTimeMap, TimeValuePair firstTVPair) throws IOException {
+ long startTime;
+ long endTime;
+ TimeValuePair localTV = firstTVPair;
+ writeTVPair(seriesWriterImpl, dataType, localTV);
+ startTime = endTime = localTV.getTimestamp();
+ if (!startTimeMap.containsKey(deviceId) || startTimeMap.get(deviceId) > startTime) {
+ startTimeMap.put(deviceId, startTime);
+ }
+ if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
+ endTimeMap.put(deviceId, endTime);
+ }
+ while (seriesReader.hasNext()) {
+ localTV = seriesReader.next();
+ endTime = localTV.getTimestamp();
+ writeTVPair(seriesWriterImpl, dataType, localTV);
+ }
+ if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
+ endTimeMap.put(deviceId, endTime);
+ }
+ }
+
+ private void writeTVPair(ChunkWriterImpl seriesWriterImpl, TSDataType dataType,
+ TimeValuePair timeValuePair) throws IOException {
+ switch (dataType) {
+ case BOOLEAN:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
+ break;
+ case INT32:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
+ break;
+ case INT64:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
+ break;
+ case FLOAT:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
+ break;
+ case DOUBLE:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
+ break;
+ case TEXT:
+ seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
+ break;
+ default:
+ LOGGER.error("Not support data type: {}", dataType);
+ break;
+ }
+ }
+
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
index ce2e803..a06cd76 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessor.java
@@ -30,6 +30,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -58,11 +59,13 @@ import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
import org.apache.iotdb.db.engine.sgmanager.OperationResult;
import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.sync.SyncUtils;
import org.apache.iotdb.db.utils.ImmediateFuture;
import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.db.utils.QueryUtils;
@@ -241,32 +244,32 @@ public class TsFileProcessor extends Processor {
.parseLong(x.getName().split(FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR)[0])));
for (File tsfile : tsFiles) {
- addResource(tsfile, unclosedFileName);
+ if (!tsfile.getName().equals(unclosedFileName)) {
+ addResource(tsfile);
+ }
}
}
// add one TsFiles to tsFileResources and update device inserted time map
- private void addResource(File tsfile, String unclosedFileName) throws TsFileProcessorException {
+ private void addResource(File tsfile) throws TsFileProcessorException {
//TODO we'd better define a file suffix for TsFile, e.g., .ts
String[] names = tsfile.getName().split(FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR);
long time = Long.parseLong(names[0]);
if (fileNamePrefix < time) {
fileNamePrefix = time;
}
- if (!tsfile.getName().equals(unclosedFileName)) {
- TsFileResource resource;
- try {
- resource = new TsFileResource(tsfile, true);
- } catch (IOException e) {
- throw new TsFileProcessorException(e);
- }
- tsFileResources.add(resource);
- //maintain the inverse index and fileNamePrefix
- for (String device : resource.getDevices()) {
- inverseIndexOfResource.computeIfAbsent(device, k -> new ArrayList<>()).add(resource);
- lastFlushedTimeForEachDevice
- .merge(device, resource.getEndTime(device), (x, y) -> x > y ? x : y);
- }
+ TsFileResource resource;
+ try {
+ resource = new TsFileResource(tsfile, true);
+ } catch (IOException e) {
+ throw new TsFileProcessorException(e);
+ }
+ tsFileResources.add(resource);
+ //maintain the inverse index and fileNamePrefix
+ for (String device : resource.getDevices()) {
+ inverseIndexOfResource.computeIfAbsent(device, k -> new ArrayList<>()).add(resource);
+ lastFlushedTimeForEachDevice
+ .merge(device, resource.getEndTime(device), (x, y) -> x > y ? x : y);
}
}
@@ -292,7 +295,7 @@ public class TsFileProcessor extends Processor {
this.insertFile = file;
try {
writer = new RestorableTsFileIOWriter(processorName, insertFile.getAbsolutePath());
- this.currentResource = new TsFileResource(insertFile, writer);
+ currentResource = new TsFileResource(insertFile, writer);
} catch (IOException e) {
throw new TsFileProcessorException(e);
}
@@ -315,15 +318,20 @@ public class TsFileProcessor extends Processor {
* WRITE_REJECT_BY_MEM)
* @throws TsFileProcessorException if a flushing operation occurs and failed.
*/
- public OperationResult insert(InsertPlan plan) throws TsFileProcessorException, IOException {
+ public OperationResult insert(InsertPlan plan) throws TsFileProcessorException {
if (isClosed) {
- return OperationResult.WRITE_REJECT_BY_CLOSED_PROCESSOR;
+ reopen();
}
if (!canWrite(plan.getDeviceId(), plan.getTime())) {
return OperationResult.WRITE_REJECT_BY_TIME;
}
if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- logNode.write(plan);
+ try {
+ logNode.write(plan);
+ } catch (IOException e) {
+ throw new TsFileProcessorException(String.format("Fail to write wal for %s",
+ plan.toString()), e);
+ }
}
long memUsage = 0;
TSDataType type;
@@ -462,6 +470,9 @@ public class TsFileProcessor extends Processor {
*/
@Override
public Future<Boolean> flush() throws IOException {
+ if (isClosed) {
+ return null;
+ }
// waiting for the end of last flush operation.
try {
flushFuture.get();
@@ -611,6 +622,7 @@ public class TsFileProcessor extends Processor {
//very dangerous, how to make sure this function is thread safe (no other functions are running)
private void closeCurrentFile() throws TsFileProcessorException {
try {
+ flush().get();
long closeStartTime = System.currentTimeMillis();
// end file
if (writer.getChunkGroupMetaDatas().isEmpty()){
@@ -661,7 +673,7 @@ public class TsFileProcessor extends Processor {
@Override
public long memoryUsage() {
- return 0;
+ return memSize.get();
}
/**
@@ -782,6 +794,7 @@ public class TsFileProcessor extends Processor {
} catch (IOException e) {
throw new TsFileProcessorException(e);
}
+ isClosed = true;
}
/**
@@ -863,4 +876,78 @@ public class TsFileProcessor extends Processor {
throw new TsFileProcessorException(e);
}
}
+
+ public long getLastInsertTime(String deviceId) {
+ // generally speaking, currentTime > historicalTime, unless currentTime does not exist
+ long historicalTime = lastFlushedTimeForEachDevice.getOrDefault(deviceId, -1L);
+ long currentTime = maxWrittenTimeForEachDeviceInCurrentFile.getOrDefault(deviceId, -1L);
+ return Long.max(historicalTime, currentTime);
+ }
+
+ /**
+ * Append a new TsFile for the sync module. The appended file can only contain new data w.r.t the
+ * existing files.
+ * @param tsFileResource
+ */
+ public void appendFile(TsFileResource tsFileResource) {
+ this.tsFileResources.add(tsFileResource);
+ for (Entry<String, Long> entry : tsFileResource.getEndTimeMap().entrySet()) {
+ lastFlushedTimeForEachDevice.put(entry.getKey(), entry.getValue());
+ maxWrittenTimeForEachDeviceInCurrentFile.put(entry.getKey(), entry.getValue());
+ // the new file may contain new time series
+ if (!minWrittenTimeForEachDeviceInCurrentFile.containsKey(entry.getKey())) {
+ minWrittenTimeForEachDeviceInCurrentFile.put(entry.getKey(), entry.getValue());
+ }
+ inverseIndexOfResource.computeIfAbsent(entry.getKey(),
+ k -> new ArrayList<>()).add(tsFileResource);
+ }
+ }
+
+ /**
+ * get overlap tsfiles which are conflict with the appendFile.
+ *
+ * @param appendFile the appended tsfile information
+ */
+ public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
+ throws FileNodeProcessorException {
+ List<String> overlapFiles = new ArrayList<>();
+ try {
+ for (TsFileResource tsFileResource : tsFileResources) {
+ getOverlapFile(appendFile, tsFileResource, uuid, overlapFiles);
+ }
+ } catch (IOException e) {
+ throw new FileNodeProcessorException(String.format("Failed to get tsfiles "
+ + "which overlap with the appendFile: %s.", appendFile.getFilePath()), e);
+ }
+ return overlapFiles;
+ }
+
+ private void getOverlapFile(TsFileResource appendFile, TsFileResource tsFileResource,
+ String uuid, List<String> overlapFiles) throws IOException {
+ if (!tsFileResource.overlaps(appendFile)) {
+ return;
+ }
+ // create a link to the overlapped file to avoid modifying the original file link
+ File newFile = SyncUtils.linkFile(uuid, tsFileResource);
+ overlapFiles.add(newFile.getPath());
+ }
+
+ /**
+ * the total size of all TsFiles.
+ * @return
+ */
+ public long totalFileSize() {
+ long fileSize = 0;
+ for (TsFileResource resource : tsFileResources) {
+ fileSize += resource.getFile().length();
+ }
+ if (currentResource != null) {
+ fileSize += currentResource.getFile().length();
+ }
+ return fileSize;
+ }
+
+ public List<TsFileResource> getTsFileResources() {
+ return tsFileResources;
+ }
}
\ No newline at end of file
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/exception/FileNodeManagerException.java b/iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupManagerException.java
similarity index 77%
rename from iotdb/src/main/java/org/apache/iotdb/db/exception/FileNodeManagerException.java
rename to iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupManagerException.java
index 0c15c50..a58aa17 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/exception/FileNodeManagerException.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupManagerException.java
@@ -18,23 +18,23 @@
*/
package org.apache.iotdb.db.exception;
-public class FileNodeManagerException extends Exception {
+public class StorageGroupManagerException extends Exception {
private static final long serialVersionUID = 9001649171768311032L;
- public FileNodeManagerException() {
+ public StorageGroupManagerException() {
super();
}
- public FileNodeManagerException(String message, Throwable cause) {
+ public StorageGroupManagerException(String message, Throwable cause) {
super(message, cause);
}
- public FileNodeManagerException(String message) {
+ public StorageGroupManagerException(String message) {
super(message);
}
- public FileNodeManagerException(Throwable cause) {
+ public StorageGroupManagerException(Throwable cause) {
super(cause);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java b/iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupProcessorException.java
similarity index 61%
rename from iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java
rename to iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupProcessorException.java
index 2b6b3bf..7981824 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/exception/StorageGroupProcessorException.java
@@ -16,18 +16,26 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.iotdb.db.exception;
-package org.apache.iotdb.db.query.reader.overflow;
+public class StorageGroupProcessorException extends Exception {
-import java.util.LinkedList;
-import java.util.List;
+ private static final long serialVersionUID = 4182064660150694060L;
-public class OverflowSeriesMultiFileReader {
- private String device;
- private String measurement;
+ public StorageGroupProcessorException() {
+ super();
+ }
- //this list should be sorted.
- private List<ConciseChunkMetadataWithFileName> metadataList = new LinkedList<>();
+ public StorageGroupProcessorException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ public StorageGroupProcessorException(String message) {
+ super(message);
+ }
+
+ public StorageGroupProcessorException(Throwable cause) {
+ super(cause);
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index cc6e1cf..d465498 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@ -321,8 +321,8 @@ public class MGraph implements Serializable {
/**
* Get all file names for given seriesPath
*/
- public List<String> getAllFileNamesByPath(String path) throws PathErrorException {
- return mtree.getAllFileNamesByPath(path);
+ public List<String> getAllStorageGroupsByPath(String path) throws PathErrorException {
+ return mtree.getAllStorageGroupsByPath(path);
}
/**
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index ea4ec6f..98774f4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -644,7 +644,7 @@ public class MManager {
*
* @return A String represented the file name
*/
- public String getFileNameByPath(String path) throws PathErrorException {
+ public String getStorageGroupByPath(String path) throws PathErrorException {
lock.readLock().lock();
try {
@@ -659,7 +659,7 @@ public class MManager {
/**
* function for getting file name by path.
*/
- public String getFileNameByPath(MNode node, String path) throws PathErrorException {
+ public String getStorageGroupByPath(MNode node, String path) throws PathErrorException {
lock.readLock().lock();
try {
@@ -687,11 +687,11 @@ public class MManager {
/**
* function for getting all file names.
*/
- public List<String> getAllFileNames() throws PathErrorException {
+ public List<String> getAllStorageGroups() throws PathErrorException {
lock.readLock().lock();
try {
- Map<String, ArrayList<String>> res = getAllPathGroupByFileName(ROOT_NAME);
+ Map<String, ArrayList<String>> res = getAllStorageGroupsByPath(ROOT_NAME);
return new ArrayList<>(res.keySet());
} finally {
lock.readLock().unlock();
@@ -707,7 +707,7 @@ public class MManager {
lock.readLock().lock();
try {
- return mgraph.getAllFileNamesByPath(path);
+ return mgraph.getAllStorageGroupsByPath(path);
} catch (PathErrorException e) {
throw new PathErrorException(e);
} finally {
@@ -718,7 +718,7 @@ public class MManager {
/**
* return a HashMap contains all the paths separated by File Name.
*/
- public Map<String, ArrayList<String>> getAllPathGroupByFileName(String path)
+ Map<String, ArrayList<String>> getAllStorageGroupsByPath(String path)
throws PathErrorException {
lock.readLock().lock();
try {
@@ -737,7 +737,7 @@ public class MManager {
lock.readLock().lock();
try {
ArrayList<String> res = new ArrayList<>();
- Map<String, ArrayList<String>> pathsGroupByFilename = getAllPathGroupByFileName(path);
+ Map<String, ArrayList<String>> pathsGroupByFilename = getAllStorageGroupsByPath(path);
for (ArrayList<String> ps : pathsGroupByFilename.values()) {
res.addAll(ps);
}
@@ -897,7 +897,7 @@ public class MManager {
lock.readLock().lock();
try {
for (Path p : path) {
- getFileNameByPath(p.getFullPath());
+ getStorageGroupByPath(p.getFullPath());
}
return true;
} finally {
@@ -913,7 +913,7 @@ public class MManager {
lock.readLock().lock();
try {
for (Path p : path) {
- getFileNameByPath(node, p.getFullPath());
+ getStorageGroupByPath(node, p.getFullPath());
}
return true;
} finally {
@@ -928,7 +928,7 @@ public class MManager {
lock.readLock().lock();
try {
- getFileNameByPath(path);
+ getStorageGroupByPath(path);
return true;
} finally {
lock.readLock().unlock();
@@ -942,7 +942,7 @@ public class MManager {
lock.readLock().lock();
try {
- getFileNameByPath(node, path);
+ getStorageGroupByPath(node, path);
return true;
} finally {
lock.readLock().unlock();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index f5bf6a2..584f560 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -585,26 +585,26 @@ public class MTree implements Serializable {
}
/**
- * Get all the storage group seriesPaths for one seriesPath.
+ * Get all the storage group names for one seriesPath.
*
* @return List storage group seriesPath list
* @apiNote :for cluster
*/
- public List<String> getAllFileNamesByPath(String pathReg) throws PathErrorException {
- ArrayList<String> fileNames = new ArrayList<>();
+ public List<String> getAllStorageGroupsByPath(String pathReg) throws PathErrorException {
+ ArrayList<String> storageGroups = new ArrayList<>();
String[] nodes = pathReg.split(DOUB_SEPARATOR);
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException(String.format(SERIES_NOT_CORRECT, pathReg));
}
- findFileName(getRoot(), nodes, 1, "", fileNames);
- return fileNames;
+ findStorageGroup(getRoot(), nodes, 1, "", storageGroups);
+ return storageGroups;
}
/**
- * Recursively find all fileName according to a specific path
+ * Recursively find all storage groups according to a specific path
* @apiNote :for cluster
*/
- private void findFileName(MNode node, String[] nodes, int idx, String parent,
+ private void findStorageGroup(MNode node, String[] nodes, int idx, String parent,
ArrayList<String> paths) {
if (node.isStorageLevel()) {
paths.add(node.getDataFileName());
@@ -619,11 +619,11 @@ public class MTree implements Serializable {
if (!("*").equals(nodeReg)) {
if (node.hasChild(nodeReg)) {
- findFileName(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths);
+ findStorageGroup(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths);
}
} else {
for (MNode child : node.getChildren().values()) {
- findFileName(child, nodes, idx + 1, parent + node.getName() + ".", paths);
+ findStorageGroup(child, nodes, idx + 1, parent + node.getName() + ".", paths);
}
}
return;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
index ecbb27e..a9451a3 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/MonitorConstants.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.util.HashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.monitor.collector.FileSize;
import org.apache.iotdb.db.service.Monitor;
@@ -35,7 +36,7 @@ public class MonitorConstants {
static final String FILENODE_PROCESSOR_CONST = "FILENODE_PROCESSOR_CONST";
private static final String FILENODE_MANAGER_CONST = "FILENODE_MANAGER_CONST";
static final String FILE_SIZE_CONST = "FILE_SIZE_CONST";
- public static final String MONITOR_PATH_SEPARATOR = ".";
+ public static final char MONITOR_PATH_SEPARATOR = IoTDBConstant.PATH_SEPARATOR;
// statistic for file size statistic module
private static final String FILE_SIZE = "file_size";
public static final String FILE_SIZE_STORAGE_GROUP_NAME = STAT_STORAGE_GROUP_PREFIX
@@ -79,7 +80,7 @@ public class MonitorConstants {
return hashMap;
}
- public enum FileNodeManagerStatConstants {
+ public enum StorageGroupManagerStatConstants {
TOTAL_POINTS, TOTAL_REQ_SUCCESS, TOTAL_REQ_FAIL, TOTAL_POINTS_SUCCESS, TOTAL_POINTS_FAIL
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index 052c324..4283e24 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@ -32,12 +32,11 @@ import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
-import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.monitor.MonitorConstants.FileNodeManagerStatConstants;
+import org.apache.iotdb.db.monitor.MonitorConstants.StorageGroupManagerStatConstants;
import org.apache.iotdb.db.monitor.MonitorConstants.FileNodeProcessorStatConstants;
import org.apache.iotdb.db.monitor.collector.FileSize;
import org.apache.iotdb.db.service.IService;
@@ -95,7 +94,7 @@ public class StatMonitor implements IService {
}
private void initTemporaryStatList() {
- for (FileNodeManagerStatConstants constants : FileNodeManagerStatConstants.values()) {
+ for (StorageGroupManagerStatConstants constants : StorageGroupManagerStatConstants.values()) {
temporaryStatList.add(constants.name());
}
for (FileNodeProcessorStatConstants constants : FileNodeProcessorStatConstants.values()) {
@@ -359,7 +358,7 @@ public class StatMonitor implements IService {
}
}
}
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER
.error("Error occurred when deleting statistics information periodically, because",
e);
@@ -382,7 +381,7 @@ public class StatMonitor implements IService {
numInsert.incrementAndGet();
pointNum = entry.getValue().dataPointList.size();
numPointsInsert.addAndGet(pointNum);
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
numInsertError.incrementAndGet();
LOGGER.error("Inserting stat points error.", e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
index f3e3ffe..b3e695f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/collector/FileSize.java
@@ -31,7 +31,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.monitor.IStatistic;
import org.apache.iotdb.db.monitor.MonitorConstants;
import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
@@ -80,7 +80,7 @@ public class FileSize implements IStatistic {
fileNodeManager.addTimeSeries(path, TSDataType.valueOf(MonitorConstants.DATA_TYPE_INT64),
TSEncoding.valueOf("RLE"), CompressionType.valueOf(TSFileConfig.compressor),
Collections.emptyMap());
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("Register File Size Stats into fileNodeManager Failed.", e);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
index b80c393..2fc51bb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
@@ -33,7 +33,7 @@ import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.ArgsErrorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -182,7 +182,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
@Override
public QueryDataSet aggregate(List<Path> paths, List<String> aggres, IExpression expression,
QueryContext context)
- throws ProcessorException, FileNodeManagerException, QueryFilterOptimizationException,
+ throws ProcessorException, StorageGroupManagerException, QueryFilterOptimizationException,
PathErrorException, IOException {
return queryRouter.aggregate(paths, aggres, expression, context);
}
@@ -190,14 +190,14 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
@Override
public QueryDataSet fill(List<Path> fillPaths, long queryTime, Map<TSDataType, IFill> fillTypes,
QueryContext context)
- throws ProcessorException, IOException, PathErrorException, FileNodeManagerException {
+ throws ProcessorException, IOException, PathErrorException, StorageGroupManagerException {
return queryRouter.fill(fillPaths, queryTime, fillTypes, context);
}
@Override
public QueryDataSet groupBy(List<Path> paths, List<String> aggres, IExpression expression,
long unit, long origin, List<Pair<Long, Long>> intervals, QueryContext context)
- throws ProcessorException, FileNodeManagerException, QueryFilterOptimizationException,
+ throws ProcessorException, StorageGroupManagerException, QueryFilterOptimizationException,
PathErrorException, IOException {
return queryRouter.groupBy(paths, aggres, expression, unit, origin, intervals, context);
}
@@ -212,14 +212,14 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
if (!mManager.pathExist(fullPath)) {
throw new ProcessorException(String.format("Timeseries %s does not exist.", fullPath));
}
- mManager.getFileNameByPath(fullPath);
+ mManager.getStorageGroupByPath(fullPath);
TSDataType dataType = mManager.getSeriesType(fullPath);
value = checkValue(dataType, value);
fileNodeManager.update(deviceId, measurementId, startTime, endTime, dataType, value);
return true;
} catch (PathErrorException e) {
throw new ProcessorException(e.getMessage());
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
e.printStackTrace();
throw new ProcessorException(e.getMessage());
}
@@ -234,17 +234,16 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
throw new ProcessorException(
String.format("Timeseries %s does not exist.", path.getFullPath()));
}
- mManager.getFileNameByPath(path.getFullPath());
+ mManager.getStorageGroupByPath(path.getFullPath());
fileNodeManager.delete(deviceId, measurementId, timestamp);
return true;
- } catch (PathErrorException | FileNodeManagerException e) {
+ } catch (PathErrorException | StorageGroupManagerException e) {
throw new ProcessorException(e);
}
}
@Override
- // return 0: failed, 1: Overflow, 2:Bufferwrite
- public int insert(Path path, long timestamp, String value) throws ProcessorException {
+ public void insert(Path path, long timestamp, String value) throws ProcessorException {
String deviceId = path.getDevice();
String measurementId = path.getMeasurement();
@@ -253,12 +252,11 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
TSRecord tsRecord = new TSRecord(timestamp, deviceId);
DataPoint dataPoint = DataPoint.getDataPoint(type, measurementId, value);
tsRecord.addTuple(dataPoint);
- return fileNodeManager.insert(tsRecord, false);
+ fileNodeManager.insert(tsRecord, false);
} catch (PathErrorException e) {
throw new ProcessorException("Error in insert: " + e.getMessage());
- } catch (FileNodeManagerException e) {
- e.printStackTrace();
+ } catch (StorageGroupManagerException e) {
throw new ProcessorException(e);
}
}
@@ -293,7 +291,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
}
return fileNodeManager.insert(tsRecord, false);
- } catch (PathErrorException | FileNodeManagerException e) {
+ } catch (PathErrorException | StorageGroupManagerException e) {
throw new ProcessorException(e.getMessage());
}
}
@@ -508,7 +506,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
throw new ProcessorException("Storage group should be created first");
}
// optimize the speed of adding timeseries
- String fileNodePath = mManager.getFileNameByPath(path.getFullPath());
+ String fileNodePath = mManager.getStorageGroupByPath(path.getFullPath());
// the two map is stored in the storage group node
Map<String, MeasurementSchema> schemaMap = mManager
.getSchemaMapForOneFileNode(fileNodePath);
@@ -542,7 +540,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
//TODO path, resultDataType, encoding, compressor, encodingArgs);
}
// fileNodeManager.closeOneFileNode(namespacePath);
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
throw new ProcessorException(e);
}
}
@@ -560,7 +558,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
}
List<String> newSubPaths = new ArrayList<>();
for (String eachSubPath : subPaths) {
- String filenodeName = mManager.getFileNameByPath(eachSubPath);
+ String filenodeName = mManager.getStorageGroupByPath(eachSubPath);
if (MonitorConstants.STAT_STORAGE_GROUP_PREFIX.equals(filenodeName)) {
continue;
@@ -586,7 +584,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
for (String p : fullPath) {
String nameSpacePath = null;
try {
- nameSpacePath = mManager.getFileNameByPath(p);
+ nameSpacePath = mManager.getStorageGroupByPath(p);
} catch (PathErrorException e) {
throw new ProcessorException(e);
}
@@ -628,7 +626,7 @@ public class OverflowQPExecutor extends QueryProcessExecutor {
default:
throw new ProcessorException("unknown namespace type:" + namespaceType);
}
- } catch (PathErrorException | IOException | FileNodeManagerException e) {
+ } catch (PathErrorException | IOException | StorageGroupManagerException e) {
throw new ProcessorException(e.getMessage());
}
return true;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index 376997e..c89894c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@ -24,7 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -58,7 +58,7 @@ public abstract class QueryProcessExecutor {
* @return QueryDataSet
*/
public QueryDataSet processQuery(QueryPlan queryPlan, QueryContext context)
- throws IOException, FileNodeManagerException, PathErrorException,
+ throws IOException, StorageGroupManagerException, PathErrorException,
QueryFilterOptimizationException, ProcessorException {
QueryExpression queryExpression = QueryExpression.create().setSelectSeries(queryPlan.getPaths())
@@ -104,16 +104,16 @@ public abstract class QueryProcessExecutor {
public abstract QueryDataSet aggregate(List<Path> paths, List<String> aggres,
IExpression expression, QueryContext context) throws ProcessorException, IOException,
- PathErrorException, FileNodeManagerException, QueryFilterOptimizationException;
+ PathErrorException, StorageGroupManagerException, QueryFilterOptimizationException;
public abstract QueryDataSet groupBy(List<Path> paths, List<String> aggres,
IExpression expression, long unit, long origin, List<Pair<Long, Long>> intervals,
QueryContext context) throws ProcessorException, IOException, PathErrorException,
- FileNodeManagerException, QueryFilterOptimizationException;
+ StorageGroupManagerException, QueryFilterOptimizationException;
public abstract QueryDataSet fill(List<Path> fillPaths, long queryTime, Map<TSDataType,
IFill> fillTypes, QueryContext context)
- throws ProcessorException, IOException, PathErrorException, FileNodeManagerException;
+ throws ProcessorException, IOException, PathErrorException, StorageGroupManagerException;
/**
* executeWithGlobalTimeFilter update command and return whether the operator is successful.
@@ -177,9 +177,8 @@ public abstract class QueryProcessExecutor {
* @param path seriesPath to be inserted
* @param insertTime - it's time point but not a range
* @param value value to be inserted
- * @return - Operate Type.
*/
- public abstract int insert(Path path, long insertTime, String value) throws ProcessorException;
+ public abstract void insert(Path path, long insertTime, String value) throws ProcessorException;
/**
* executeWithGlobalTimeFilter insert command and return whether the operator is successful.
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 84e93c6..ad55b7a 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -56,13 +56,13 @@ public class JobFileManager {
* Add the unique file paths to sealedFilePathsMap and unsealedFilePathsMap.
*/
public void addUsedFilesForGivenJob(long jobId, QueryDataSource dataSource) {
- for (TsFileResource tsFileResource : dataSource.getSeqDataSource().getSealedTsFiles()) {
+ for (TsFileResource tsFileResource : dataSource.getSeqDataSource().getSealedFiles()) {
String sealedFilePath = tsFileResource.getFilePath();
addFilePathToMap(jobId, sealedFilePath, true);
}
- if (dataSource.getSeqDataSource().hasUnsealedTsFile()) {
- String unSealedFilePath = dataSource.getSeqDataSource().getUnsealedTsFile().getFilePath();
+ if (dataSource.getSeqDataSource().hasUnsealedFile()) {
+ String unSealedFilePath = dataSource.getSeqDataSource().getUnsealedFile().getFilePath();
addFilePathToMap(jobId, unSealedFilePath, false);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
index cb55dfd..2b75efb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/QueryResourceManager.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.ExpressionType;
@@ -119,7 +119,7 @@ public class QueryResourceManager {
* calculation.
*/
public void beginQueryOfGivenQueryPaths(long jobId, List<Path> queryPaths)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
Set<String> deviceIdSet = new HashSet<>();
queryPaths.forEach(path -> deviceIdSet.add(path.getDevice()));
@@ -134,7 +134,7 @@ public class QueryResourceManager {
* calculation.
*/
public void beginQueryOfGivenExpression(long jobId, IExpression expression)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
Set<String> deviceIdSet = new HashSet<>();
getUniquePaths(expression, deviceIdSet);
for (String deviceId : deviceIdSet) {
@@ -145,7 +145,7 @@ public class QueryResourceManager {
public QueryDataSource getQueryDataSource(Path selectedPath,
QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
SingleSeriesExpression singleSeriesExpression = new SingleSeriesExpression(selectedPath, null);
QueryDataSource queryDataSource = FileNodeManager.getInstance()
@@ -161,7 +161,7 @@ public class QueryResourceManager {
* Whenever the jdbc request is closed normally or abnormally, this method must be invoked. All
* query tokens created by this jdbc request must be cleared.
*/
- public void endQueryForGivenJob(long jobId) throws FileNodeManagerException {
+ public void endQueryForGivenJob(long jobId) throws StorageGroupManagerException {
if (queryTokensMap.get(jobId) == null) {
// no resource need to be released.
return;
@@ -207,7 +207,7 @@ public class QueryResourceManager {
*/
public QueryDataSource getQueryDataSourceByTsFileProcessor(Path selectedPath,
QueryContext context, TsFileProcessor processor)
- throws IOException, FileNodeManagerException {
+ throws IOException, StorageGroupManagerException {
OverflowSeriesDataSource overflowSeriesDataSource = new OverflowSeriesDataSource(selectedPath);
overflowSeriesDataSource.setOverflowInsertFileList(Collections.EMPTY_LIST);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
index ba76c2f..aa494bb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/AggregateEngineExecutor.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -80,7 +80,7 @@ public class AggregateEngineExecutor {
* @param context query context
*/
public QueryDataSet executeWithOutTimeGenerator(QueryContext context)
- throws FileNodeManagerException, IOException, PathErrorException, ProcessorException {
+ throws StorageGroupManagerException, IOException, PathErrorException, ProcessorException {
Filter timeFilter = null;
if (expression != null) {
timeFilter = ((GlobalTimeExpression) expression).getFilter();
@@ -255,7 +255,7 @@ public class AggregateEngineExecutor {
* @param context query context.
*/
public QueryDataSet executeWithTimeGenerator(QueryContext context)
- throws FileNodeManagerException, PathErrorException, IOException, ProcessorException {
+ throws StorageGroupManagerException, PathErrorException, IOException, ProcessorException {
QueryResourceManager
.getInstance().beginQueryOfGivenQueryPaths(context.getJobId(), selectedSeries);
QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithTimeGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithTimeGenerator.java
index 3b40187..d910fea 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithTimeGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithTimeGenerator.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.executor;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -52,9 +52,9 @@ public class EngineExecutorWithTimeGenerator {
*
* @return QueryDataSet object
* @throws IOException IOException
- * @throws FileNodeManagerException FileNodeManagerException
+ * @throws StorageGroupManagerException StorageGroupManagerException
*/
- public QueryDataSet execute(QueryContext context) throws FileNodeManagerException {
+ public QueryDataSet execute(QueryContext context) throws StorageGroupManagerException {
QueryResourceManager.getInstance()
.beginQueryOfGivenQueryPaths(context.getJobId(), queryExpression.getSelectedSeries());
@@ -68,7 +68,7 @@ public class EngineExecutorWithTimeGenerator {
readersOfSelectedSeries = SeriesReaderFactory
.getByTimestampReadersOfSelectedPaths(queryExpression.getSelectedSeries(), context);
} catch (IOException ex) {
- throw new FileNodeManagerException(ex);
+ throw new StorageGroupManagerException(ex);
}
List<TSDataType> dataTypes = new ArrayList<>();
@@ -77,7 +77,7 @@ public class EngineExecutorWithTimeGenerator {
try {
dataTypes.add(MManager.getInstance().getSeriesType(path.getFullPath()));
} catch (PathErrorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
index aec6959..d479be9 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineExecutorWithoutTimeGenerator.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -57,7 +57,7 @@ public class EngineExecutorWithoutTimeGenerator {
* with global time filter.
*/
public QueryDataSet executeWithGlobalTimeFilter(QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
Filter timeFilter = ((GlobalTimeExpression) queryExpression.getExpression()).getFilter();
@@ -76,7 +76,7 @@ public class EngineExecutorWithoutTimeGenerator {
try {
dataTypes.add(MManager.getInstance().getSeriesType(path.getFullPath()));
} catch (PathErrorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// sequence reader for one sealed tsfile
@@ -85,7 +85,7 @@ public class EngineExecutorWithoutTimeGenerator {
tsFilesReader = new SequenceDataReader(queryDataSource.getSeqDataSource(),
timeFilter, context);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// unseq reader for all chunk groups in unSeqFile
@@ -94,7 +94,7 @@ public class EngineExecutorWithoutTimeGenerator {
unSeqMergeReader = SeriesReaderFactory.getInstance()
.createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), timeFilter);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// merge sequence data with unsequence data.
@@ -105,7 +105,7 @@ public class EngineExecutorWithoutTimeGenerator {
return new EngineDataSetWithoutTimeGenerator(queryExpression.getSelectedSeries(), dataTypes,
readersOfSelectedSeries);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
@@ -113,7 +113,7 @@ public class EngineExecutorWithoutTimeGenerator {
* without filter.
*/
public QueryDataSet executeWithoutFilter(QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
List<IPointReader> readersOfSelectedSeries = new ArrayList<>();
List<TSDataType> dataTypes = new ArrayList<>();
@@ -130,7 +130,7 @@ public class EngineExecutorWithoutTimeGenerator {
try {
dataTypes.add(MManager.getInstance().getSeriesType(path.getFullPath()));
} catch (PathErrorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// sequence insert data
@@ -139,7 +139,7 @@ public class EngineExecutorWithoutTimeGenerator {
tsFilesReader = new SequenceDataReader(queryDataSource.getSeqDataSource(),
null, context);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// unseq insert data
@@ -148,7 +148,7 @@ public class EngineExecutorWithoutTimeGenerator {
unSeqMergeReader = SeriesReaderFactory.getInstance()
.createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), null);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// merge sequence data with unsequence data.
@@ -159,7 +159,7 @@ public class EngineExecutorWithoutTimeGenerator {
return new EngineDataSetWithoutTimeGenerator(queryExpression.getSelectedSeries(), dataTypes,
readersOfSelectedSeries);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
/**
@@ -168,7 +168,7 @@ public class EngineExecutorWithoutTimeGenerator {
* fileNodeManager is refactored
*/
public QueryDataSet executeWithoutFilter(QueryContext context, TsFileProcessor processor)
- throws FileNodeManagerException, IOException {
+ throws StorageGroupManagerException, IOException {
List<IPointReader> readersOfSelectedSeries = new ArrayList<>();
List<TSDataType> dataTypes = new ArrayList<>();
@@ -185,7 +185,7 @@ public class EngineExecutorWithoutTimeGenerator {
try {
dataTypes.add(MManager.getInstance().getSeriesType(path.getFullPath()));
} catch (PathErrorException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// sequence insert data
@@ -194,7 +194,7 @@ public class EngineExecutorWithoutTimeGenerator {
tsFilesReader = new SequenceDataReader(queryDataSource.getSeqDataSource(),
null, context);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// unseq insert data
@@ -203,7 +203,7 @@ public class EngineExecutorWithoutTimeGenerator {
unSeqMergeReader = SeriesReaderFactory.getInstance()
.createUnSeqMergeReader(queryDataSource.getOverflowSeriesDataSource(), null);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
// merge sequence data with unsequence data.
@@ -214,7 +214,7 @@ public class EngineExecutorWithoutTimeGenerator {
return new EngineDataSetWithoutTimeGenerator(queryExpression.getSelectedSeries(), dataTypes,
readersOfSelectedSeries);
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
index fb4d106..930e87f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/EngineQueryRouter.java
@@ -24,7 +24,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessor;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -57,7 +57,7 @@ public class EngineQueryRouter {
* execute physical plan.
*/
public QueryDataSet query(QueryExpression queryExpression, QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
if (queryExpression.hasQueryFilter()) {
try {
@@ -76,7 +76,7 @@ public class EngineQueryRouter {
}
} catch (QueryFilterOptimizationException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} else {
EngineExecutorWithoutTimeGenerator engineExecutor = new EngineExecutorWithoutTimeGenerator(
@@ -90,7 +90,7 @@ public class EngineQueryRouter {
*/
public QueryDataSet aggregate(List<Path> selectedSeries, List<String> aggres,
IExpression expression, QueryContext context) throws QueryFilterOptimizationException,
- FileNodeManagerException, IOException, PathErrorException, ProcessorException {
+ StorageGroupManagerException, IOException, PathErrorException, ProcessorException {
if (expression != null) {
IExpression optimizedExpression = ExpressionOptimizer.getInstance()
@@ -123,7 +123,7 @@ public class EngineQueryRouter {
public QueryDataSet groupBy(List<Path> selectedSeries, List<String> aggres,
IExpression expression, long unit, long origin, List<Pair<Long, Long>> intervals,
QueryContext context)
- throws ProcessorException, QueryFilterOptimizationException, FileNodeManagerException,
+ throws ProcessorException, QueryFilterOptimizationException, StorageGroupManagerException,
PathErrorException, IOException {
long nextJobId = context.getJobId();
@@ -188,7 +188,7 @@ public class EngineQueryRouter {
*/
public QueryDataSet fill(List<Path> fillPaths, long queryTime, Map<TSDataType, IFill> fillType,
QueryContext context)
- throws FileNodeManagerException, PathErrorException, IOException {
+ throws StorageGroupManagerException, PathErrorException, IOException {
long nextJobId = context.getJobId();
@@ -227,7 +227,7 @@ public class EngineQueryRouter {
*/
public QueryDataSet query(QueryExpression queryExpression, TsFileProcessor processor,
QueryContext context)
- throws FileNodeManagerException, IOException {
+ throws StorageGroupManagerException, IOException {
if (queryExpression.hasQueryFilter()) {
throw new NotImplementedException("this function is just for test...");
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
index 83c5fa9..7dfa31e 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/FillEngineExecutor.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -58,7 +58,7 @@ public class FillEngineExecutor {
* @param context query context
*/
public QueryDataSet execute(QueryContext context)
- throws FileNodeManagerException, PathErrorException, IOException {
+ throws StorageGroupManagerException, PathErrorException, IOException {
QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries);
List<IFill> fillList = new ArrayList<>();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
index e82b567..c2b85eb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithOnlyTimeFilterDataSetDataSet.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.query.aggregation.AggreResultData;
@@ -74,7 +74,7 @@ public class GroupByWithOnlyTimeFilterDataSetDataSet extends GroupByEngineDataSe
* init reader and aggregate function.
*/
public void initGroupBy(QueryContext context, List<String> aggres, IExpression expression)
- throws FileNodeManagerException, PathErrorException, ProcessorException, IOException {
+ throws StorageGroupManagerException, PathErrorException, ProcessorException, IOException {
initAggreFuction(aggres);
// init reader
QueryResourceManager.getInstance().beginQueryOfGivenQueryPaths(jobId, selectedSeries);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithValueFilterDataSetDataSet.java b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithValueFilterDataSetDataSet.java
index 7a6b3d2..9711203 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithValueFilterDataSetDataSet.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/executor/groupby/GroupByWithValueFilterDataSetDataSet.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.query.aggregation.AggregateFunction;
@@ -70,7 +70,7 @@ public class GroupByWithValueFilterDataSetDataSet extends GroupByEngineDataSet {
* init reader and aggregate function.
*/
public void initGroupBy(QueryContext context, List<String> aggres, IExpression expression)
- throws FileNodeManagerException, PathErrorException, ProcessorException, IOException {
+ throws StorageGroupManagerException, PathErrorException, ProcessorException, IOException {
initAggreFuction(aggres);
QueryResourceManager.getInstance().beginQueryOfGivenExpression(context.getJobId(), expression);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
index b731fc5..608c623 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/factory/SeriesReaderFactory.java
@@ -24,28 +24,26 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.engine.querycontext.SeriesDataSource;
+import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.reader.AllDataReader;
-import org.apache.iotdb.db.query.reader.IBatchReader;
import org.apache.iotdb.db.query.reader.IPointReader;
import org.apache.iotdb.db.query.reader.IReader;
-import org.apache.iotdb.db.query.reader.mem.MemChunkReader;
import org.apache.iotdb.db.query.reader.mem.MemChunkReaderByTimestamp;
import org.apache.iotdb.db.query.reader.merge.EngineReaderByTimeStamp;
import org.apache.iotdb.db.query.reader.merge.PriorityMergeReader;
import org.apache.iotdb.db.query.reader.merge.PriorityMergeReaderByTimestamp;
-import org.apache.iotdb.db.query.reader.sequence.SealedTsFilesReader;
+import org.apache.iotdb.db.query.reader.sequence.SequenceDataReader;
import org.apache.iotdb.db.query.reader.sequence.SequenceDataReaderByTimestamp;
import org.apache.iotdb.db.query.reader.unsequence.EngineChunkReader;
import org.apache.iotdb.db.query.reader.unsequence.EngineChunkReaderByTimestamp;
import org.apache.iotdb.db.utils.QueryUtils;
-import org.apache.iotdb.tsfile.common.constant.StatisticConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Chunk;
@@ -53,22 +51,12 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerier;
import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.read.filter.DigestForFilter;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderByTimestamp;
-import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithFilter;
import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
-import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
-import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class SeriesReaderFactory {
- private static final Logger logger = LoggerFactory.getLogger(SeriesReaderFactory.class);
-
private SeriesReaderFactory() {
}
@@ -82,53 +70,32 @@ public class SeriesReaderFactory {
* process, it's no need to maintain the opened file stream.
*/
public PriorityMergeReader createUnSeqMergeReader(
- OverflowSeriesDataSource overflowSeriesDataSource, Filter filter)
+ SeriesDataSource overflowSeriesDataSource, QueryContext context)
throws IOException {
PriorityMergeReader unSeqMergeReader = new PriorityMergeReader();
int priorityValue = 1;
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
-
- // store only one opened file stream into manager, to avoid too many opened files
- TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance()
- .get(overflowInsertFile.getFilePath(), false);
-
- ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
-
- for (ChunkMetaData chunkMetaData : overflowInsertFile.getChunkMetaDataList()) {
+ for (TsFileResource overflowFile : overflowSeriesDataSource.getSealedFiles()) {
- DigestForFilter digest = new DigestForFilter(chunkMetaData.getStartTime(),
- chunkMetaData.getEndTime(),
- chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MIN_VALUE),
- chunkMetaData.getDigest().getStatistics().get(StatisticConstant.MAX_VALUE),
- chunkMetaData.getTsDataType());
-
- if (filter != null && !filter.satisfy(digest)) {
- continue;
- }
+ TsFileSequenceReader tsFileSequenceReader = FileReaderManager.getInstance()
+ .get(overflowFile.getFilePath(), true);
+ ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(tsFileSequenceReader);
+ List<ChunkMetaData> metaDataList = readFileChunkMetadata(tsFileSequenceReader, context,
+ overflowSeriesDataSource.getSeriesPath(), overflowFile.getModFile());
+ for (ChunkMetaData chunkMetaData : metaDataList) {
Chunk chunk = chunkLoader.getChunk(chunkMetaData);
- ChunkReader chunkReader = filter != null ? new ChunkReaderWithFilter(chunk, filter)
- : new ChunkReaderWithoutFilter(chunk);
+ ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
unSeqMergeReader
- .addReaderWithPriority(new EngineChunkReader(chunkReader, unClosedTsFileReader),
+ .addReaderWithPriority(new EngineChunkReader(chunkReader, tsFileSequenceReader),
priorityValue);
priorityValue++;
}
}
- // add reader for MemTable
- if (overflowSeriesDataSource.hasRawChunk()) {
- unSeqMergeReader.addReaderWithPriority(
- new MemChunkReader(overflowSeriesDataSource.getReadableMemChunk(), filter),
- priorityValue);
- }
-
- // TODO add external sort when needed
return unSeqMergeReader;
}
@@ -138,53 +105,29 @@ public class SeriesReaderFactory {
* This method is used to construct reader for merge process in IoTDB. To merge only one TsFile
* data and one UnSeqFile data.
*/
- public IReader createSeriesReaderForMerge(TsFileResource tsFileResource,
- OverflowSeriesDataSource overflowSeriesDataSource,
- SingleSeriesExpression singleSeriesExpression,
+ public IReader createSeriesReaderForMerge(SeriesDataSource seqDataSource,
+ SeriesDataSource overflowDataSource,
QueryContext context)
throws IOException {
- logger.debug("Create seriesReaders for merge. SeriesFilter = {}. TsFilePath = {}",
- singleSeriesExpression,
- tsFileResource.getFilePath());
-
// sequence reader
- IBatchReader seriesInTsFileReader = createSealedTsFileReaderForMerge(tsFileResource,
- singleSeriesExpression, context);
+ SequenceDataReader seqReader = new SequenceDataReader(seqDataSource, null, context);
// unSequence merge reader
- IPointReader unSeqMergeReader = createUnSeqMergeReader(overflowSeriesDataSource,
- singleSeriesExpression.getFilter());
- if (!seriesInTsFileReader.hasNext()) {
- // only have unsequence data.
- return unSeqMergeReader;
- } else {
- // merge sequence data with unsequence data.
- return new AllDataReader(seriesInTsFileReader, unSeqMergeReader);
- }
+ IPointReader unSeqReader = createUnSeqMergeReader(overflowDataSource, context);
+ return new AllDataReader(seqReader, unSeqReader);
}
- private IBatchReader createSealedTsFileReaderForMerge(TsFileResource fileNode,
- SingleSeriesExpression singleSeriesExpression,
- QueryContext context)
- throws IOException {
-
- TsFileSequenceReader tsFileSequenceReader = FileReaderManager.getInstance()
- .get(fileNode.getFilePath(), true);
- ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(tsFileSequenceReader);
+ private List<ChunkMetaData> readFileChunkMetadata(TsFileSequenceReader tsFileSequenceReader,
+ QueryContext context, Path path, ModificationFile modFile) throws IOException {
MetadataQuerier metadataQuerier = new MetadataQuerierByFileImpl(tsFileSequenceReader);
List<ChunkMetaData> metaDataList = metadataQuerier
- .getChunkMetaDataList(singleSeriesExpression.getSeriesPath());
+ .getChunkMetaDataList(path);
- List<Modification> modifications = context.getPathModifications(fileNode.getModFile(),
- singleSeriesExpression.getSeriesPath().getFullPath());
+ List<Modification> modifications = context.getPathModifications(modFile, path.getFullPath());
QueryUtils.modifyChunkMetaData(metaDataList, modifications);
-
- FileSeriesReader seriesInTsFileReader = new FileSeriesReaderWithFilter(chunkLoader,
- metaDataList,
- singleSeriesExpression.getFilter());
- return new SealedTsFilesReader(seriesInTsFileReader, context);
+ return metaDataList;
}
/**
@@ -195,7 +138,7 @@ public class SeriesReaderFactory {
* @return the list of EngineReaderByTimeStamp
*/
public static List<EngineReaderByTimeStamp> getByTimestampReadersOfSelectedPaths(
- List<Path> paths, QueryContext context) throws IOException, FileNodeManagerException {
+ List<Path> paths, QueryContext context) throws IOException, StorageGroupManagerException {
List<EngineReaderByTimeStamp> readersOfSelectedSeries = new ArrayList<>();
@@ -213,7 +156,7 @@ public class SeriesReaderFactory {
// reader for unSequence data
PriorityMergeReaderByTimestamp unSeqMergeReader = SeriesReaderFactory.getInstance()
- .createUnSeqMergeReaderByTimestamp(queryDataSource.getOverflowSeriesDataSource());
+ .createUnSeqMergeReaderByTimestamp(queryDataSource.getOverflowSeriesDataSource(), context);
mergeReaderByTimestamp.addReaderWithPriority(unSeqMergeReader, 2);
readersOfSelectedSeries.add(mergeReaderByTimestamp);
@@ -226,24 +169,45 @@ public class SeriesReaderFactory {
* This method is used to create unsequence insert reader by timestamp for IoTDB request, such as
* query, aggregation and groupby request.
*/
- public PriorityMergeReaderByTimestamp createUnSeqMergeReaderByTimestamp(
- OverflowSeriesDataSource overflowSeriesDataSource)
+ private PriorityMergeReaderByTimestamp createUnSeqMergeReaderByTimestamp(
+ SeriesDataSource overflowSeriesDataSource, QueryContext context)
throws IOException {
PriorityMergeReaderByTimestamp unSeqMergeReader = new PriorityMergeReaderByTimestamp();
int priorityValue = 1;
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
+ // sealed files
+ for (TsFileResource tsFileResource : overflowSeriesDataSource
+ .getSealedFiles()) {
+
+ TsFileSequenceReader fileSequenceReader = FileReaderManager.getInstance()
+ .get(tsFileResource.getFilePath(), true);
+
+ ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(fileSequenceReader);
+ List<ChunkMetaData> metaDataList = readFileChunkMetadata(fileSequenceReader, context,
+ overflowSeriesDataSource.getSeriesPath(), tsFileResource.getModFile());
- // store only one opened file stream into manager, to avoid too many opened files
- TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance()
- .get(overflowInsertFile.getFilePath(), false);
+ for (ChunkMetaData chunkMetaData : metaDataList) {
+
+ Chunk chunk = chunkLoader.getChunk(chunkMetaData);
+ ChunkReaderByTimestamp chunkReader = new ChunkReaderByTimestamp(chunk);
+
+ unSeqMergeReader
+ .addReaderWithPriority(new EngineChunkReaderByTimestamp(chunkReader), priorityValue);
+ priorityValue++;
+ }
+ }
- ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
+ // unsealed file
+ if (overflowSeriesDataSource.hasUnsealedFile()) {
+ UnsealedTsFile unsealedTsFile = overflowSeriesDataSource.getUnsealedFile();
+ TsFileSequenceReader fileSequenceReader = FileReaderManager.getInstance()
+ .get(unsealedTsFile.getFilePath(), false);
+ List<ChunkMetaData> metaDataList = unsealedTsFile.getChunkMetaDataList();
+ ChunkLoaderImpl chunkLoader = new ChunkLoaderImpl(fileSequenceReader);
- for (ChunkMetaData chunkMetaData : overflowInsertFile.getChunkMetaDataList()) {
+ for (ChunkMetaData chunkMetaData : metaDataList) {
Chunk chunk = chunkLoader.getChunk(chunkMetaData);
ChunkReaderByTimestamp chunkReader = new ChunkReaderByTimestamp(chunk);
@@ -254,10 +218,10 @@ public class SeriesReaderFactory {
}
}
- // add reader for MemTable
- if (overflowSeriesDataSource.hasRawChunk()) {
+ // MemTable
+ if (overflowSeriesDataSource.hasRawSeriesChunk()) {
unSeqMergeReader.addReaderWithPriority(
- new MemChunkReaderByTimestamp(overflowSeriesDataSource.getReadableMemChunk()),
+ new MemChunkReaderByTimestamp(overflowSeriesDataSource.getReadableChunk()),
priorityValue);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/merge/PriorityMergeReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/merge/PriorityMergeReader.java
index e76d142..251bd9d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/merge/PriorityMergeReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/merge/PriorityMergeReader.java
@@ -33,9 +33,6 @@ import org.apache.iotdb.db.utils.TimeValuePair;
*/
public class PriorityMergeReader implements IPointReader {
- public static final int LOW_PRIORITY = 1;
- public static final int HIGH_PRIORITY = 2;
-
private List<IPointReader> readerList = new ArrayList<>();
private List<Integer> priorityList = new ArrayList<>();
private PriorityQueue<Element> heap = new PriorityQueue<>();
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java
deleted file mode 100644
index 6ecbb8f..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with the License. You may obtain
- * a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.reader.overflow;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import org.apache.iotdb.db.exception.TooManyChunksException;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class OverflowSeriesSingleFileReader {
-
- private static final Logger LOGGER = LoggerFactory
- .getLogger(OverflowSeriesSingleFileReader.class);
- private String device;
- private String measurement;
- private File file;
-
- //how many chunks can be read at the same time.
- private int numberOfWays;
-
- //this list should be sorted.
- private List<ConciseChunkMetadata> metadataList = new LinkedList<>();
-
- public OverflowSeriesSingleFileReader(String device, String measurement, File file,
- long memoryLimitInBytes) throws IOException, TooManyChunksException {
- this.device = device;
- this.measurement = measurement;
- this.file = file;
- this.numberOfWays =
- (int) memoryLimitInBytes / (TSFileConfig.pageSizeInByte + (int) ConciseChunkMetadata
- .getOccupiedMemory());
-
- try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) {
- TsFileMetaData fileMetaData = reader.readFileMetadata();
- if (fileMetaData.getMeasurementSchema().containsKey(measurement)) {
- TsDeviceMetadata metadata = reader
- .readTsDeviceMetaData(fileMetaData.getDeviceMetadataIndex(device));
- for (ChunkGroupMetaData cgMetadata : metadata.getChunkGroupMetaDataList()) {
- for (ChunkMetaData metaData : cgMetadata.getChunkMetaDataList()) {
- if (metaData.getMeasurementUid().equals(measurement)) {
- this.metadataList.add(new ConciseChunkMetadata(metaData));
- if (metadataList.size() > numberOfWays) {
- throw new TooManyChunksException(new File[]{file}, device, measurement,
- numberOfWays);
- }
- break;
- }
- }
- }
- }
- Collections.sort(this.metadataList);
- }
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
index 075fa08..0c44297 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SealedTsFilesReader.java
@@ -24,24 +24,15 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
-import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.IAggregateReader;
import org.apache.iotdb.db.query.reader.IBatchReader;
import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.tsfile.file.header.PageHeader;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
-import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter;
-import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithoutFilter;
public class SealedTsFilesReader implements IBatchReader, IAggregateReader {
@@ -132,31 +123,10 @@ public class SealedTsFilesReader implements IBatchReader, IAggregateReader {
private void initSingleTsFileReader(TsFileResource tsfile, QueryContext context)
throws IOException {
-
- // to avoid too many opened files
- TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
- .get(tsfile.getFilePath(), true);
-
- MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
- List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(seriesPath);
-
- List<Modification> pathModifications = context.getPathModifications(tsfile.getModFile(),
- seriesPath.getFullPath());
- if (!pathModifications.isEmpty()) {
- QueryUtils.modifyChunkMetaData(metaDataList, pathModifications);
- }
-
- ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader);
-
- if (isReverse) {
- Collections.reverse(metaDataList);
- }
-
- if (filter == null) {
- seriesReader = new FileSeriesReaderWithoutFilter(chunkLoader, metaDataList);
- } else {
- seriesReader = new FileSeriesReaderWithFilter(chunkLoader, metaDataList, filter);
+ if (seriesReader != null) {
+ seriesReader.close();
}
+ seriesReader = QueryUtils.createTsFileReader(tsfile, context, seriesPath, isReverse, filter);
}
@Override
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
index 668d81e..2a9abb4 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReader.java
@@ -67,27 +67,27 @@ public class SequenceDataReader implements IBatchReader, IAggregateReader {
}
// add reader for unSealed TsFile
- if (sources.hasUnsealedTsFile()) {
- seriesReaders.add(new UnSealedTsFileReader(sources.getUnsealedTsFile(), filter, isReverse));
+ if (sources.hasUnsealedFile()) {
+ seriesReaders.add(new UnSealedTsFileReader(sources.getUnsealedFile(), filter, isReverse));
}
// add reader for sealed TsFiles
if (sources.hasSealedTsFiles()) {
seriesReaders.add(
- new SealedTsFilesReader(sources.getSeriesPath(), sources.getSealedTsFiles(), filter,
+ new SealedTsFilesReader(sources.getSeriesPath(), sources.getSealedFiles(), filter,
context, isReverse));
}
} else {
// add reader for sealed TsFiles
if (sources.hasSealedTsFiles()) {
seriesReaders.add(
- new SealedTsFilesReader(sources.getSeriesPath(), sources.getSealedTsFiles(), filter,
+ new SealedTsFilesReader(sources.getSeriesPath(), sources.getSealedFiles(), filter,
context, isReverse));
}
// add reader for unSealed TsFile
- if (sources.hasUnsealedTsFile()) {
- seriesReaders.add(new UnSealedTsFileReader(sources.getUnsealedTsFile(), filter, isReverse));
+ if (sources.hasUnsealedFile()) {
+ seriesReaders.add(new UnSealedTsFileReader(sources.getUnsealedFile(), filter, isReverse));
}
// add data in memTable
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
index 0eff726..e8a3905 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderByTimestamp.java
@@ -45,13 +45,13 @@ public class SequenceDataReaderByTimestamp implements EngineReaderByTimeStamp {
// add reader for sealed TsFiles
if (sources.hasSealedTsFiles()) {
seriesReaders.add(
- new SealedTsFilesReaderByTimestamp(sources.getSeriesPath(), sources.getSealedTsFiles(),
+ new SealedTsFilesReaderByTimestamp(sources.getSeriesPath(), sources.getSealedFiles(),
context));
}
// add reader for unSealed TsFile
- if (sources.hasUnsealedTsFile()) {
- seriesReaders.add(new UnSealedTsFilesReaderByTimestamp(sources.getUnsealedTsFile()));
+ if (sources.hasUnsealedFile()) {
+ seriesReaders.add(new UnSealedTsFilesReaderByTimestamp(sources.getUnsealedFile()));
}
// add data in memTable
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
index 4ffe62b..86b434f 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineNodeConstructor.java
@@ -25,7 +25,7 @@ import static org.apache.iotdb.tsfile.read.expression.ExpressionType.SERIES;
import java.io.IOException;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
@@ -54,16 +54,16 @@ public class EngineNodeConstructor {
* @param expression expression
* @return Node object
* @throws IOException IOException
- * @throws FileNodeManagerException FileNodeManagerException
+ * @throws StorageGroupManagerException StorageGroupManagerException
*/
public Node construct(IExpression expression, QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
if (expression.getType() == SERIES) {
try {
return new EngineLeafNode(generateSeriesReader((SingleSeriesExpression) expression,
context));
} catch (IOException e) {
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
} else {
Node leftChild;
@@ -85,7 +85,7 @@ public class EngineNodeConstructor {
private IReader generateSeriesReader(SingleSeriesExpression singleSeriesExpression,
QueryContext context)
- throws IOException, FileNodeManagerException {
+ throws IOException, StorageGroupManagerException {
QueryDataSource queryDataSource = QueryResourceManager.getInstance().getQueryDataSource(
singleSeriesExpression.getSeriesPath(), context);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineTimeGenerator.java b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineTimeGenerator.java
index 350ea6f..ac18fba 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineTimeGenerator.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/EngineTimeGenerator.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.query.timegenerator;
import java.io.IOException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
@@ -39,12 +39,12 @@ public class EngineTimeGenerator implements TimeGenerator {
* Constructor of EngineTimeGenerator.
*/
public EngineTimeGenerator(IExpression expression, QueryContext context)
- throws FileNodeManagerException {
+ throws StorageGroupManagerException {
this.expression = expression;
initNode(context);
}
- private void initNode(QueryContext context) throws FileNodeManagerException {
+ private void initNode(QueryContext context) throws StorageGroupManagerException {
EngineNodeConstructor engineNodeConstructor = new EngineNodeConstructor();
this.operatorNode = engineNodeConstructor.construct(expression, context);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index d9a1938..86c667c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -26,7 +26,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.RecoverException;
import org.apache.iotdb.db.exception.StartupException;
@@ -123,7 +123,7 @@ public class IoTDB implements IoTDBMBean {
}
@Override
- public void stop() throws FileNodeManagerException {
+ public void stop() throws StorageGroupManagerException {
deactivate();
}
@@ -147,7 +147,7 @@ public class IoTDB implements IoTDBMBean {
WriteLogNodeManager writeLogManager = MultiFileLogNodeManager.getInstance();
List<String> filenodeNames = null;
try {
- filenodeNames = MManager.getInstance().getAllFileNames();
+ filenodeNames = MManager.getInstance().getAllStorageGroups();
} catch (PathErrorException e) {
throw new RecoverException(e);
}
@@ -155,7 +155,7 @@ public class IoTDB implements IoTDBMBean {
if (writeLogManager.hasWAL(filenodeName)) {
try {
FileNodeManager.getInstance().recoverFileNode(filenodeName);
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
throw new RecoverException(e);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDBMBean.java b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDBMBean.java
index d48523c..a6a4469 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDBMBean.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDBMBean.java
@@ -18,10 +18,10 @@
*/
package org.apache.iotdb.db.service;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
@FunctionalInterface
public interface IoTDBMBean {
- void stop() throws FileNodeManagerException;
+ void stop() throws StorageGroupManagerException;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index f355445..ffaefdb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -36,12 +36,11 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.exception.ArgsErrorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.qp.IllegalASTFormatException;
import org.apache.iotdb.db.exception.qp.QueryProcessorException;
-import org.apache.iotdb.db.metadata.MGraph;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.Metadata;
import org.apache.iotdb.db.qp.QueryProcessor;
@@ -191,13 +190,13 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
releaseQueryResource(req);
clearAllStatusForCurrentRequest();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("Error in closeOperation : {}", e.getMessage());
}
return new TSCloseOperationResp(new TS_Status(TS_StatusCode.SUCCESS_STATUS));
}
- private void releaseQueryResource(TSCloseOperationReq req) throws FileNodeManagerException {
+ private void releaseQueryResource(TSCloseOperationReq req) throws StorageGroupManagerException {
Map<Long, QueryContext> contextMap = contextMapLocal.get();
if (contextMap == null) {
return;
@@ -378,7 +377,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
case "flush":
try {
FileNodeManager.getInstance().closeAll();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("meet error while FileNodeManager closing all!", e);
throw new IOException(e);
}
@@ -386,7 +385,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
case "merge":
try {
FileNodeManager.getInstance().mergeAll();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
LOGGER.error("meet error while FileNodeManager merging all!", e);
throw new IOException(e);
}
@@ -628,7 +627,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
}
private QueryDataSet createNewDataSet(String statement, int fetchSize, TSFetchResultsReq req)
- throws PathErrorException, QueryFilterOptimizationException, FileNodeManagerException,
+ throws PathErrorException, QueryFilterOptimizationException, StorageGroupManagerException,
ProcessorException, IOException {
PhysicalPlan physicalPlan = queryStatus.get().get(statement);
processor.getExecutor().setFetchSize(fetchSize);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java
new file mode 100644
index 0000000..a946112
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/SyncUtils.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.sync;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.iotdb.db.conf.directories.Directories;
+import org.apache.iotdb.db.engine.filenode.TsFileResource;
+import org.apache.iotdb.db.sync.conf.Constans;
+
+public class SyncUtils {
+
+ private SyncUtils() {
+ throw new UnsupportedOperationException("Initialize a util class");
+ }
+
+ /**
+ * create a file link for a 'tsfile' which overlaps a sync source file represented by 'uuid'.
+ * @param uuid uuid of a sync source file
+ * @param tsFile a tsfile that overlaps the source file
+ * @return the file link
+ * @throws IOException
+ */
+ public static File linkFile(String uuid, TsFileResource tsFile) throws IOException {
+ String relativeFilePath =
+ Constans.SYNC_SERVER + File.separatorChar + uuid + File.separatorChar
+ + Constans.BACK_UP_DIRECTORY_NAME
+ + File.separatorChar + tsFile.getRelativePath();
+ File newFile = new File(
+ Directories.getInstance().getTsFileFolder(tsFile.getBaseDirIndex()),
+ relativeFilePath);
+ if (!newFile.getParentFile().exists()) {
+ newFile.getParentFile().mkdirs();
+ }
+ Path link = FileSystems.getDefault().getPath(newFile.getPath());
+ Path target = FileSystems.getDefault()
+ .getPath(tsFile.getFile().getAbsolutePath());
+ Files.createLink(link, target);
+
+ return newFile;
+ }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java b/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
index 400065e..d933dbb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
@@ -46,7 +46,7 @@ import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.filenode.OverflowChangeType;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
@@ -440,7 +440,7 @@ public class SyncServiceImpl implements SyncService.Iface {
* directly. If data in the tsfile is old, it has two strategy to merge.It depends on the
* possibility of updating historical data.
*/
- public void loadData() throws FileNodeManagerException {
+ public void loadData() throws StorageGroupManagerException {
syncDataPath = FilePathUtils.regularizePath(syncDataPath);
int processedNum = 0;
for (String storageGroup : fileNodeMap.get().keySet()) {
@@ -469,8 +469,7 @@ public class SyncServiceImpl implements SyncService.Iface {
// create a new fileNode
String header = syncDataPath;
String relativePath = path.substring(header.length());
- TsFileResource fileNode = new TsFileResource(startTimeMap, endTimeMap,
- OverflowChangeType.NO_CHANGE, new File(
+ TsFileResource fileNode = new TsFileResource(startTimeMap, endTimeMap, new File(
Directories.getInstance().getNextFolderIndexForTsFile() + File.separator + relativePath)
);
// call interface of load external file
@@ -490,9 +489,9 @@ public class SyncServiceImpl implements SyncService.Iface {
}
}
}
- } catch (FileNodeManagerException | IOException | ProcessorException e) {
+ } catch (StorageGroupManagerException | IOException | ProcessorException e) {
logger.error("Can not load external file {}", path);
- throw new FileNodeManagerException(e);
+ throw new StorageGroupManagerException(e);
}
processedNum++;
logger.info(String
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
index 69a6935..db5b528 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/LoadDataUtils.java
@@ -32,7 +32,7 @@ import java.util.Set;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -131,7 +131,7 @@ public class LoadDataUtils {
totalPointCount += record.dataPointList.size();
String nsPath = null;
try {
- nsPath = mmanager.getFileNameByPath(record.deviceId);
+ nsPath = mmanager.getStorageGroupByPath(record.deviceId);
} catch (PathErrorException e) {
logger.error("given seriesPath not found, given deviceId:{}", record.deviceId, e);
}
@@ -152,7 +152,7 @@ public class LoadDataUtils {
// appeared before, insert directly
try {
fileNodeManager.insert(record, false);
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
logger.error("failed when insert into fileNodeManager, record:{}", line, e);
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 8f6fe21..c9621fe 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -19,10 +19,24 @@
package org.apache.iotdb.db.utils;
+import java.io.IOException;
+import java.util.Collections;
import java.util.List;
+import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReader;
+import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithFilter;
+import org.apache.iotdb.tsfile.read.reader.series.FileSeriesReaderWithoutFilter;
public class QueryUtils {
@@ -73,4 +87,37 @@ public class QueryUtils {
}
return false;
}
+
+ public static FileSeriesReader createTsFileReader(TsFileResource tsfile, QueryContext context, Path path,
+ boolean isReverse, Filter filter)
+ throws IOException {
+
+ FileSeriesReader seriesReader;
+
+ // to avoid too many opened files
+ TsFileSequenceReader tsFileReader = FileReaderManager.getInstance()
+ .get(tsfile.getFilePath(), true);
+
+ MetadataQuerierByFileImpl metadataQuerier = new MetadataQuerierByFileImpl(tsFileReader);
+ List<ChunkMetaData> metaDataList = metadataQuerier.getChunkMetaDataList(path);
+
+ List<Modification> pathModifications = context.getPathModifications(tsfile.getModFile(),
+ path.getFullPath());
+ if (!pathModifications.isEmpty()) {
+ QueryUtils.modifyChunkMetaData(metaDataList, pathModifications);
+ }
+
+ ChunkLoader chunkLoader = new ChunkLoaderImpl(tsFileReader);
+
+ if (isReverse) {
+ Collections.reverse(metaDataList);
+ }
+
+ if (filter == null) {
+ seriesReader = new FileSeriesReaderWithoutFilter(chunkLoader, metaDataList);
+ } else {
+ seriesReader = new FileSeriesReaderWithFilter(chunkLoader, metaDataList, filter);
+ }
+ return seriesReader;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
index 893bb1a..75c5580 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/ExclusiveLogRecoverPerformer.java
@@ -32,7 +32,7 @@ import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.RecoverException;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
@@ -286,7 +286,7 @@ public class ExclusiveLogRecoverPerformer implements RecoverPerformer {
}
try {
FileNodeManager.getInstance().closeOneFileNode(writeLogNode.getFileNodeName());
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
logger.error("Log node {} cannot perform flush after replaying logs! Because {}",
writeLogNode.getIdentifier(), e.getMessage());
throw new RecoverException(e);
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
index de339f6..b41542d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/FileNodeRecoverPerformer.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.writelog.recover;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.RecoverException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +43,7 @@ public class FileNodeRecoverPerformer implements RecoverPerformer {
public void recover() throws RecoverException {
try {
FileNodeManager.getInstance().recoverFileNode(getFileNodeName());
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
logger.error("Cannot recover filenode {}", identifier);
throw new RecoverException(e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java b/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
index 6573884..267ee39 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/replay/ConcreteLogReplayer.java
@@ -18,9 +18,8 @@
*/
package org.apache.iotdb.db.writelog.replay;
-import java.util.List;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -62,7 +61,7 @@ public class ConcreteLogReplayer implements LogReplayer {
}
private void multiInsert(InsertPlan insertPlan)
- throws PathErrorException, FileNodeManagerException {
+ throws PathErrorException, StorageGroupManagerException {
String deviceId = insertPlan.getDeviceId();
long insertTime = insertPlan.getTime();
String[] measurementList = insertPlan.getMeasurements();
@@ -79,7 +78,7 @@ public class ConcreteLogReplayer implements LogReplayer {
FileNodeManager.getInstance().insert(tsRecord, true);
}
- private void update(UpdatePlan updatePlan) throws FileNodeManagerException, PathErrorException {
+ private void update(UpdatePlan updatePlan) throws StorageGroupManagerException, PathErrorException {
TSDataType dataType = MManager.getInstance().getSeriesType(updatePlan.getPath().getFullPath());
for (Pair<Long, Long> timePair : updatePlan.getIntervals()) {
FileNodeManager.getInstance().update(updatePlan.getPath().getDevice(),
@@ -88,7 +87,7 @@ public class ConcreteLogReplayer implements LogReplayer {
}
}
- private void delete(DeletePlan deletePlan, boolean isOverflow) throws FileNodeManagerException {
+ private void delete(DeletePlan deletePlan, boolean isOverflow) throws StorageGroupManagerException {
for (Path path : deletePlan.getPaths()) {
if (isOverflow) {
FileNodeManager.getInstance().deleteOverflow(path.getDevice(), path.getMeasurement(),
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenode/TsFileResourceTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenode/TsFileResourceTest.java
index a1c9d24..172ce67 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenode/TsFileResourceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenode/TsFileResourceTest.java
@@ -43,7 +43,7 @@ public class TsFileResourceTest {
Map<String, Long> endTimes = new HashMap<>();
tsFileResource = new TsFileResource(Collections.emptyMap(), Collections.emptyMap(),
- OverflowChangeType.MERGING_CHANGE, new File(relativePath));
+ new File(relativePath));
for (int i = 0; i < 10; i++) {
startTimes.put("d" + i, (long) i);
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
index 55ee355..6e066af 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodev2/FileNodeManagerBenchmark.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
@@ -72,13 +72,13 @@ public class FileNodeManagerBenchmark {
}
}
- private static void tearDown() throws IOException, FileNodeManagerException {
+ private static void tearDown() throws IOException, StorageGroupManagerException {
EnvironmentUtils.cleanEnv();
}
public static void main(String[] args)
throws InterruptedException, IOException, MetadataArgsErrorException,
- PathErrorException, FileNodeManagerException {
+ PathErrorException, StorageGroupManagerException {
tearDown();
prepare();
long startTime = System.currentTimeMillis();
@@ -115,7 +115,7 @@ public class FileNodeManagerBenchmark {
TSRecord tsRecord = getRecord(deltaObject, time);
FileNodeManager.getInstance().insert(tsRecord, true);
}
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
e.printStackTrace();
} finally {
latch.countDown();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 22c8581..47b0d89 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -34,7 +34,7 @@ import org.apache.iotdb.db.conf.directories.Directories;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
@@ -70,7 +70,7 @@ public class DeletionFileNodeTest {
@Before
public void setup() throws MetadataArgsErrorException,
- PathErrorException, IOException, FileNodeManagerException, StartupException {
+ PathErrorException, IOException, StorageGroupManagerException, StartupException {
EnvironmentUtils.envSetUp();
MManager.getInstance().setStorageLevelToMTree(processorName);
@@ -85,13 +85,13 @@ public class DeletionFileNodeTest {
}
@After
- public void teardown() throws IOException, FileNodeManagerException {
+ public void teardown() throws IOException, StorageGroupManagerException {
EnvironmentUtils.cleanEnv();
}
@Test
public void testDeleteInBufferWriteCache() throws
- FileNodeManagerException {
+ StorageGroupManagerException {
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, processorName);
@@ -124,7 +124,7 @@ public class DeletionFileNodeTest {
}
@Test
- public void testDeleteInBufferWriteFile() throws FileNodeManagerException, IOException {
+ public void testDeleteInBufferWriteFile() throws StorageGroupManagerException, IOException {
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, processorName);
for (int j = 0; j < 10; j++) {
@@ -166,7 +166,7 @@ public class DeletionFileNodeTest {
}
@Test
- public void testDeleteInOverflowCache() throws FileNodeManagerException {
+ public void testDeleteInOverflowCache() throws StorageGroupManagerException {
// insert into BufferWrite
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
@@ -211,7 +211,7 @@ public class DeletionFileNodeTest {
}
@Test
- public void testDeleteInOverflowFile() throws FileNodeManagerException, IOException {
+ public void testDeleteInOverflowFile() throws StorageGroupManagerException, IOException {
// insert into BufferWrite
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index 3e9bb99..70e2de4 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@ -28,7 +28,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
@@ -65,7 +65,7 @@ public class DeletionQueryTest {
@Before
public void setup() throws MetadataArgsErrorException,
- PathErrorException, IOException, FileNodeManagerException, StartupException {
+ PathErrorException, IOException, StorageGroupManagerException, StartupException {
EnvironmentUtils.envSetUp();
MManager.getInstance().setStorageLevelToMTree(processorName);
@@ -80,13 +80,13 @@ public class DeletionQueryTest {
}
@After
- public void teardown() throws IOException, FileNodeManagerException {
+ public void teardown() throws IOException, StorageGroupManagerException {
EnvironmentUtils.cleanEnv();
}
@Test
public void testDeleteInBufferWriteCache() throws
- FileNodeManagerException, IOException {
+ StorageGroupManagerException, IOException {
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, processorName);
@@ -118,7 +118,7 @@ public class DeletionQueryTest {
}
@Test
- public void testDeleteInBufferWriteFile() throws FileNodeManagerException, IOException {
+ public void testDeleteInBufferWriteFile() throws StorageGroupManagerException, IOException {
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, processorName);
for (int j = 0; j < 10; j++) {
@@ -149,7 +149,7 @@ public class DeletionQueryTest {
}
@Test
- public void testDeleteInOverflowCache() throws FileNodeManagerException, IOException {
+ public void testDeleteInOverflowCache() throws StorageGroupManagerException, IOException {
// insert into BufferWrite
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
@@ -191,7 +191,7 @@ public class DeletionQueryTest {
}
@Test
- public void testDeleteInOverflowFile() throws FileNodeManagerException, IOException {
+ public void testDeleteInOverflowFile() throws StorageGroupManagerException, IOException {
// insert into BufferWrite
for (int i = 101; i <= 200; i++) {
TSRecord record = new TSRecord(i, processorName);
@@ -234,7 +234,7 @@ public class DeletionQueryTest {
@Test
public void testSuccessiveDeletion()
- throws FileNodeManagerException, IOException, InterruptedException {
+ throws StorageGroupManagerException, IOException, InterruptedException {
for (int i = 1; i <= 100; i++) {
TSRecord record = new TSRecord(i, processorName);
for (int j = 0; j < 10; j++) {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessorTest.java
index 8eb681a..c30ecc4 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/overflowdata/OverflowProcessorTest.java
@@ -30,12 +30,11 @@ import org.apache.iotdb.db.engine.sgmanager.OperationResult;
import org.apache.iotdb.db.engine.tsfiledata.TsFileProcessorTest;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.utils.ImmediateFuture;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.RowRecord;
@@ -64,7 +63,7 @@ public class OverflowProcessorTest extends TsFileProcessorTest {
@Test
public void insert()
- throws BufferWriteProcessorException, IOException, ExecutionException, InterruptedException, FileNodeProcessorException, FileNodeManagerException, PathErrorException, MetadataArgsErrorException {
+ throws BufferWriteProcessorException, IOException, ExecutionException, InterruptedException, FileNodeProcessorException, StorageGroupManagerException, PathErrorException, MetadataArgsErrorException {
String[] s1 = new String[]{"s1"};
String[] s2 = new String[]{"s2"};
String[] value = new String[]{"5.0"};
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessorTest.java
index ac90e75..25ab9af 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/tsfiledata/TsFileProcessorTest.java
@@ -20,7 +20,6 @@
package org.apache.iotdb.db.engine.tsfiledata;
import static org.apache.iotdb.db.utils.EnvironmentUtils.TEST_QUERY_CONTEXT;
-import static org.apache.iotdb.db.utils.EnvironmentUtils.TEST_QUERY_JOB_ID;
import java.io.IOException;
import java.util.Collections;
@@ -34,7 +33,7 @@ import org.apache.iotdb.db.engine.bufferwrite.ActionException;
import org.apache.iotdb.db.engine.sgmanager.OperationResult;
import org.apache.iotdb.db.engine.version.SysTimeVersionController;
import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
@@ -110,7 +109,7 @@ public class TsFileProcessorTest {
@Test
public void insert()
- throws BufferWriteProcessorException, IOException, ExecutionException, InterruptedException, FileNodeProcessorException, FileNodeManagerException, PathErrorException, MetadataArgsErrorException {
+ throws BufferWriteProcessorException, IOException, ExecutionException, InterruptedException, FileNodeProcessorException, StorageGroupManagerException, PathErrorException, MetadataArgsErrorException {
String[] s1 = new String[]{"s1"};
String[] s2 = new String[]{"s2"};
String[] value = new String[]{"5.0"};
@@ -150,7 +149,7 @@ public class TsFileProcessorTest {
@Test
- public void bruteForceTest() throws InterruptedException, FileNodeManagerException, IOException {
+ public void bruteForceTest() throws InterruptedException, StorageGroupManagerException, IOException {
final boolean[] exception = {false, false, false};
final boolean[] goon = {true};
@@ -216,7 +215,7 @@ public class TsFileProcessorTest {
QueryResourceManager.getInstance().endQueryForGivenJob(context.getJobId());
processor.readUnlock();
}
- } catch (IOException | FileNodeManagerException e) {
+ } catch (IOException | StorageGroupManagerException e) {
// we will break out.
LOGGER.error(e.getMessage());
exception[2] = true;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java
index e172971..cfe0187 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBEngineTimeGeneratorIT.java
@@ -28,9 +28,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.JobFileManager;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.timegenerator.EngineTimeGenerator;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.EnvironmentUtils;
@@ -189,7 +187,7 @@ public class IoTDBEngineTimeGeneratorIT {
* value >= 14 && time > 500
*/
@Test
- public void testOneSeriesWithValueAndTimeFilter() throws IOException, FileNodeManagerException {
+ public void testOneSeriesWithValueAndTimeFilter() throws IOException, StorageGroupManagerException {
System.out.println("Test >>> root.vehicle.d0.s0 >= 14 && time > 500");
Path pd0s0 = new Path(Constant.d0s0);
@@ -215,7 +213,7 @@ public class IoTDBEngineTimeGeneratorIT {
* root.vehicle.d1.s0 >= 5, and d1.s0 has no data
*/
@Test
- public void testEmptySeriesWithValueFilter() throws IOException, FileNodeManagerException {
+ public void testEmptySeriesWithValueFilter() throws IOException, StorageGroupManagerException {
System.out.println("Test >>> root.vehicle.d1.s0 >= 5");
Path pd1s0 = new Path(Constant.d1s0);
@@ -237,7 +235,7 @@ public class IoTDBEngineTimeGeneratorIT {
*/
@Test
public void testMultiSeriesWithValueFilterAndTimeFilter()
- throws IOException, FileNodeManagerException {
+ throws IOException, StorageGroupManagerException {
System.out
.println("Test >>> root.vehicle.d0.s0 >= 5 && root.vehicle.d0.s2 >= 11.5 || time > 900");
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
index a4b95a3..3e89303 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
@@ -28,7 +28,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.executor.EngineQueryRouter;
@@ -168,7 +168,7 @@ public class IoTDBSequenceDataQueryIT {
}
@Test
- public void readWithoutFilterTest() throws IOException, FileNodeManagerException {
+ public void readWithoutFilterTest() throws IOException, StorageGroupManagerException {
EngineQueryRouter engineExecutor = new EngineQueryRouter();
QueryExpression queryExpression = QueryExpression.create();
@@ -197,7 +197,7 @@ public class IoTDBSequenceDataQueryIT {
}
@Test
- public void readWithTimeFilterTest() throws IOException, FileNodeManagerException {
+ public void readWithTimeFilterTest() throws IOException, StorageGroupManagerException {
EngineQueryRouter engineExecutor = new EngineQueryRouter();
QueryExpression queryExpression = QueryExpression.create();
queryExpression.addSelectedPath(new Path(Constant.d0s0));
@@ -225,7 +225,7 @@ public class IoTDBSequenceDataQueryIT {
}
@Test
- public void readWithValueFilterTest() throws IOException, FileNodeManagerException {
+ public void readWithValueFilterTest() throws IOException, StorageGroupManagerException {
// select * from root where root.vehicle.d0.s0 >=14
EngineQueryRouter engineExecutor = new EngineQueryRouter();
QueryExpression queryExpression = QueryExpression.create();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index 6b8d413..8b9365d 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@ -30,7 +30,7 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.executor.EngineQueryRouter;
@@ -240,7 +240,7 @@ public class IoTDBSeriesReaderIT {
}
@Test
- public void selectAllTest() throws IOException, FileNodeManagerException {
+ public void selectAllTest() throws IOException, StorageGroupManagerException {
String selectSql = "select * from root";
System.out.println("Test >>> " + selectSql);
@@ -271,7 +271,7 @@ public class IoTDBSeriesReaderIT {
}
@Test
- public void selectOneSeriesWithValueFilterTest() throws IOException, FileNodeManagerException {
+ public void selectOneSeriesWithValueFilterTest() throws IOException, StorageGroupManagerException {
String selectSql = "select s0 from root.vehicle.d0 where s0 >= 20";
System.out.println("Test >>> " + selectSql);
@@ -301,7 +301,7 @@ public class IoTDBSeriesReaderIT {
}
@Test
- public void seriesTimeDigestReadTest() throws IOException, FileNodeManagerException {
+ public void seriesTimeDigestReadTest() throws IOException, StorageGroupManagerException {
String selectSql = "select s0 from root.vehicle.d0 where time >= 22987";
System.out.println("Test >>> " + selectSql);
@@ -329,7 +329,7 @@ public class IoTDBSeriesReaderIT {
}
@Test
- public void crossSeriesReadUpdateTest() throws IOException, FileNodeManagerException {
+ public void crossSeriesReadUpdateTest() throws IOException, StorageGroupManagerException {
System.out.println("Test >>> select s1 from root.vehicle.d0 where s0 < 111");
EngineQueryRouter engineExecutor = new EngineQueryRouter();
QueryExpression queryExpression = QueryExpression.create();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java b/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
index 33dba88..cd470c9 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/QueryDataFromUnclosedTsFileIT.java
@@ -26,7 +26,7 @@ import java.io.IOException;
import java.util.Collections;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.StartupException;
@@ -39,7 +39,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.iotdb.tsfile.write.record.TSRecord;
@@ -55,7 +54,7 @@ public class QueryDataFromUnclosedTsFileIT {
MManager mManager;
EngineQueryRouter queryManager;
@Before
- public void setUp() throws IOException, FileNodeManagerException, StartupException {
+ public void setUp() throws IOException, StorageGroupManagerException, StartupException {
EnvironmentUtils.cleanEnv();
EnvironmentUtils.envSetUp();
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId();
@@ -68,7 +67,7 @@ public class QueryDataFromUnclosedTsFileIT {
}
@After
- public void tearDown() throws FileNodeManagerException, IOException {
+ public void tearDown() throws StorageGroupManagerException, IOException {
IoTDBDescriptor.getInstance().getConfig().setBufferwriteFileSizeThreshold(bufferWriteFileSize);;
EnvironmentUtils.cleanEnv();
@@ -77,7 +76,7 @@ public class QueryDataFromUnclosedTsFileIT {
@Test
public void test()
- throws FileNodeManagerException, IOException, PathErrorException, MetadataArgsErrorException {
+ throws StorageGroupManagerException, IOException, PathErrorException, MetadataArgsErrorException {
mManager.setStorageLevelToMTree("root.test");
mManager.addPathToMTree("root.test.d1.s1", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY, Collections.emptyMap());
mManager.addPathToMTree("root.test.d2.s1", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY, Collections.emptyMap());
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
index 2ead48c..e1e9ce6 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
@@ -72,7 +71,7 @@ public class MManagerAdvancedTest {
try {
// test file name
- List<String> fileNames = mmanager.getAllFileNames();
+ List<String> fileNames = mmanager.getAllStorageGroups();
assertEquals(2, fileNames.size());
if (fileNames.get(0).equals("root.vehicle.d0")) {
assertEquals(fileNames.get(1), "root.vehicle.d1");
@@ -80,9 +79,9 @@ public class MManagerAdvancedTest {
assertEquals(fileNames.get(1), "root.vehicle.d0");
}
// test filename by seriesPath
- assertEquals("root.vehicle.d0", mmanager.getFileNameByPath("root.vehicle.d0.s1"));
+ assertEquals("root.vehicle.d0", mmanager.getStorageGroupByPath("root.vehicle.d0.s1"));
Map<String, ArrayList<String>> map = mmanager
- .getAllPathGroupByFileName("root.vehicle.d1.*");
+ .getAllStorageGroupsByPath("root.vehicle.d1.*");
assertEquals(1, map.keySet().size());
assertEquals(6, map.get("root.vehicle.d1").size());
List<String> paths = mmanager.getPaths("root.vehicle.d0");
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerImproveTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerImproveTest.java
index 999dce5..083378f 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerImproveTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerImproveTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
@@ -56,7 +56,7 @@ public class MManagerImproveTest {
}
@After
- public void after() throws IOException, FileNodeManagerException {
+ public void after() throws IOException, StorageGroupManagerException {
EnvironmentUtils.cleanEnv();
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
index 4f838e1..e33ecff 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MTreeTest.java
@@ -22,9 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-import com.alibaba.fastjson.serializer.SerializerFeature;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -274,12 +271,12 @@ public class MTreeTest {
List<String> list = new ArrayList<>();
list.add("root.laptop.d1");
- assertEquals(list, root.getAllFileNamesByPath("root.laptop.d1.s1"));
- assertEquals(list, root.getAllFileNamesByPath("root.laptop.d1"));
+ assertEquals(list, root.getAllStorageGroupsByPath("root.laptop.d1.s1"));
+ assertEquals(list, root.getAllStorageGroupsByPath("root.laptop.d1"));
list.add("root.laptop.d2");
- assertEquals(list, root.getAllFileNamesByPath("root.laptop"));
- assertEquals(list, root.getAllFileNamesByPath("root"));
+ assertEquals(list, root.getAllStorageGroupsByPath("root.laptop"));
+ assertEquals(list, root.getAllStorageGroupsByPath("root"));
} catch (PathErrorException e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
index 1f959e1..5fa1728 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/monitor/MonitorTest.java
@@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.MonitorConstants.FileSizeConstants;
import org.apache.iotdb.db.monitor.collector.FileSize;
@@ -139,7 +139,7 @@ public class MonitorTest {
try {
fManager.deleteAll();
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
e.printStackTrace();
fail(e.getMessage());
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/qp/plan/QPUpdateTest.java b/iotdb/src/test/java/org/apache/iotdb/db/qp/plan/QPUpdateTest.java
index 7caaa59..2006ca5 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/qp/plan/QPUpdateTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/qp/plan/QPUpdateTest.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.exception.ArgsErrorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.qp.QueryProcessorException;
import org.apache.iotdb.db.qp.QueryProcessor;
@@ -115,7 +115,7 @@ public class QPUpdateTest {
private void testUpdate()
throws QueryProcessorException, ArgsErrorException, ProcessorException, IOException,
- FileNodeManagerException, QueryFilterOptimizationException {
+ StorageGroupManagerException, QueryFilterOptimizationException {
String sqlStr = "update root.qp_update_test.device_1.sensor_1 set value = 33000 where time >= 10 and time <= 10";
PhysicalPlan plan1 = processor.parseSQLToPhysicalPlan(sqlStr);
boolean upRet = processor.getExecutor().processNonQuery(plan1);
@@ -136,7 +136,7 @@ public class QPUpdateTest {
private void testDeletePaths()
throws QueryProcessorException, ProcessorException, ArgsErrorException, IOException,
- FileNodeManagerException, QueryFilterOptimizationException {
+ StorageGroupManagerException, QueryFilterOptimizationException {
String sqlStr = "delete from root.qp_update_test.device_1 where time < 15";
PhysicalPlan plan1 = processor.parseSQLToPhysicalPlan(sqlStr);
boolean upRet = processor.getExecutor().processNonQuery(plan1);
@@ -160,7 +160,7 @@ public class QPUpdateTest {
private void testDelete()
throws QueryProcessorException, ProcessorException, ArgsErrorException, IOException,
- FileNodeManagerException, QueryFilterOptimizationException {
+ StorageGroupManagerException, QueryFilterOptimizationException {
String sqlStr = "delete from root.qp_update_test.device_1.sensor_1 where time < 15";
PhysicalPlan plan1 = processor.parseSQLToPhysicalPlan(sqlStr);
boolean upRet = processor.getExecutor().processNonQuery(plan1);
@@ -184,7 +184,7 @@ public class QPUpdateTest {
private void testInsert()
throws QueryProcessorException, ProcessorException, ArgsErrorException, IOException,
- FileNodeManagerException, QueryFilterOptimizationException {
+ StorageGroupManagerException, QueryFilterOptimizationException {
String sqlStr = "insert into root.qp_update_test.device_1 (timestamp, sensor_1, sensor_2) values (13, 50, 40)";
PhysicalPlan plan1 = processor.parseSQLToPhysicalPlan(sqlStr);
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java b/iotdb/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java
index de521c0..99b6680 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/qp/utils/MemIntQpExecutor.java
@@ -27,7 +27,7 @@ import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.TreeSet;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.qp.constant.SQLConstant;
@@ -111,7 +111,7 @@ public class MemIntQpExecutor extends QueryProcessExecutor {
@Override
public QueryDataSet aggregate(List<Path> paths, List<String> aggres, IExpression expression,
QueryContext context)
- throws ProcessorException, IOException, PathErrorException, FileNodeManagerException,
+ throws ProcessorException, IOException, PathErrorException, StorageGroupManagerException,
QueryFilterOptimizationException {
return null;
}
@@ -119,7 +119,7 @@ public class MemIntQpExecutor extends QueryProcessExecutor {
@Override
public QueryDataSet groupBy(List<Path> paths, List<String> aggres, IExpression expression,
long unit, long origin, List<Pair<Long, Long>> intervals, QueryContext context)
- throws ProcessorException, IOException, PathErrorException, FileNodeManagerException,
+ throws ProcessorException, IOException, PathErrorException, StorageGroupManagerException,
QueryFilterOptimizationException {
return null;
}
@@ -127,7 +127,7 @@ public class MemIntQpExecutor extends QueryProcessExecutor {
@Override
public QueryDataSet fill(List<Path> fillPaths, long queryTime, Map<TSDataType, IFill> fillTypes,
QueryContext context)
- throws ProcessorException, IOException, PathErrorException, FileNodeManagerException {
+ throws ProcessorException, IOException, PathErrorException, StorageGroupManagerException {
return null;
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index b979372..3f9bffd 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.db.engine.cache.RowGroupBlockMetaDataCache;
import org.apache.iotdb.db.engine.cache.TsFileMetaDataCache;
import org.apache.iotdb.db.engine.filenode.FileNodeManager;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.StatMonitor;
@@ -63,7 +63,7 @@ public class EnvironmentUtils {
public static long TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId();
public static QueryContext TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
- public static void cleanEnv() throws IOException, FileNodeManagerException {
+ public static void cleanEnv() throws IOException, StorageGroupManagerException {
QueryResourceManager.getInstance().endQueryForGivenJob(TEST_QUERY_JOB_ID);
@@ -77,7 +77,7 @@ public class EnvironmentUtils {
LOGGER.error("Can't close the filenode manager in EnvironmentUtils");
Assert.fail();
}
- } catch (FileNodeManagerException e) {
+ } catch (StorageGroupManagerException e) {
throw new IOException(e);
}
StatMonitor.getInstance().close();
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
index eb0a117..c76bc0b 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
@@ -20,14 +20,12 @@ package org.apache.iotdb.db.writelog;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
+import org.apache.iotdb.db.exception.StorageGroupManagerException;
import org.apache.iotdb.db.exception.MetadataArgsErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.RecoverException;
-import org.apache.iotdb.db.exception.WALOverSizedException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -123,7 +121,7 @@ public class PerformanceTest {
@Test
public void recoverTest()
- throws IOException, RecoverException, FileNodeManagerException, PathErrorException,
+ throws IOException, RecoverException, StorageGroupManagerException, PathErrorException,
MetadataArgsErrorException {
// this test write 1000000 * 3 logs , recover from them and report elapsed time
if (skip) {