You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/07/01 04:24:46 UTC
[incubator-iotdb] 01/01: resolve conflict after merge master
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a1dbf5a37d2b6b2b6a21b0232e16ff9712f32656
Merge: 91d9c69 bf272d6
Author: qiaojialin <64...@qq.com>
AuthorDate: Mon Jul 1 12:21:31 2019 +0800
resolve conflict after merge master
.mvn/wrapper/maven-wrapper.properties | 2 +-
.travis.yml | 54 +-
docs/Documentation/QuickStart.md | 10 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md | 278 +++
.../db/conf/directories/DirectoryManager.java | 9 +-
.../directories/strategy/DirectoryStrategy.java | 20 +-
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 35 +-
.../strategy/MinDirOccupiedSpaceFirstStrategy.java | 76 -
.../MinFolderOccupiedSpaceFirstStrategy.java | 39 +-
.../directories/strategy/SequenceStrategy.java | 32 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 1317 ------------
.../db/engine/filenode/FileNodeProcessor.java | 2130 --------------------
.../db/engine/filenodeV2/FileNodeManagerV2.java | 3 +-
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 33 +-
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 2 +-
.../iotdb/db/exception/ArgsErrorException.java | 7 +
...on.java => DiskSpaceInsufficientException.java} | 19 +-
...rException.java => MetadataErrorException.java} | 11 +-
.../java/org/apache/iotdb/db/metadata/MGraph.java | 8 +-
.../org/apache/iotdb/db/metadata/MManager.java | 235 ++-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 9 +-
.../org/apache/iotdb/db/qp/QueryProcessor.java | 9 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 136 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 3 +-
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 5 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 23 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 5 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 7 +-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 8 +-
.../iotdb/db/writelog/io/BatchLogReader.java | 4 +-
.../writelog/recover/TsFileRecoverPerformer.java | 1 +
.../iotdb/db/engine/MetadataManagerHelper.java | 77 +-
.../filenodeV2/FileNodeManagerBenchmark.java | 6 +-
.../engine/modification/DeletionFileNodeTest.java | 6 +-
.../db/engine/modification/DeletionQueryTest.java | 6 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 1 -
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 2 +-
.../iotdb/db/metadata/MManagerAdvancedTest.java | 12 +-
.../iotdb/db/metadata/MManagerBasicTest.java | 78 +-
.../iotdb/db/metadata/MManagerEfficiencyTest.java | 6 +-
.../org/apache/iotdb/db/metadata/MetadataTest.java | 11 +-
.../iotdb/db/qp/bench/QueryParseBenchmark.java | 4 +-
.../iotdb/db/qp/other/TSPlanContextAuthorTest.java | 4 +-
.../db/qp/other/TSPlanContextPropertyTest.java | 4 +-
.../transfer/PhysicalPlanLogTransferTest.java | 10 +-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 22 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 38 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 21 +-
.../iotdb/db/qp/plan/TestConcatOptimizer.java | 10 +-
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 1 -
.../apache/iotdb/db/utils/EnvironmentUtils.java | 1 -
.../apache/iotdb/db/writelog/PerformanceTest.java | 5 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
54 files changed, 885 insertions(+), 3975 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
index ac680b9,d792c37..86a6d5f
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
@@@ -85,7 -79,7 +86,7 @@@ public class DirectoryManager
tsfileFolders.set(0, path);
}
- public String getNextFolderForSequenceFile() {
- public String getNextFolderForTsfile() throws DiskSpaceInsufficientException {
++ public String getNextFolderForSequenceFile() throws DiskSpaceInsufficientException {
return getTsFileFolder(getNextFolderIndexForTsFile());
}
@@@ -94,8 -88,8 +95,8 @@@
*
* @return next folder index
*/
- public int getNextFolderIndexForTsFile() {
+ public int getNextFolderIndexForTsFile() throws DiskSpaceInsufficientException {
- return strategy.nextFolderIndex();
+ return tsfileStrategy.nextFolderIndex();
}
public String getTsFileFolder(int index) {
@@@ -111,40 -105,6 +112,40 @@@
}
private static class DirectoriesHolder {
- private static final Directories INSTANCE = new Directories();
+ private static final DirectoryManager INSTANCE = new DirectoryManager();
+ }
+
+ public String getWALFolder() {
+ return IoTDBDescriptor.getInstance().getConfig().getWalFolder();
+ }
+
- public String getNextFolderForUnSequenceFile() {
++ public String getNextFolderForUnSequenceFile() throws DiskSpaceInsufficientException {
+ return getOverflowFileFolder(getNextFolderIndexForOverflowFile());
+ }
+
+ /**
+ * get next folder index for OverflowFile.
+ *
+ * @return next folder index
+ */
- public int getNextFolderIndexForOverflowFile() {
++ public int getNextFolderIndexForOverflowFile() throws DiskSpaceInsufficientException {
+ return overflowStrategy.nextFolderIndex();
+ }
+
+ public String getOverflowFileFolder(int index) {
+ return overflowFolders.get(index);
+ }
+
+ public int getOverflowFileFolderIndex(String folder) {
+ return overflowFolders.indexOf(folder);
+ }
+
+ public List<String> getAllOverflowFileFolders() {
+ return overflowFolders;
+ }
+
+ // only used by test
+ public String getOverflowFolderForTest() {
+ return overflowFolders.get(0);
}
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
index 52b550b,dd397b8..1dd2609
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
@@@ -72,4 -74,14 +74,18 @@@ public abstract class DirectoryStrateg
public void setFolderForTest(String path) {
folders.set(0, path);
}
+
+ protected long getUsableSpace(String dir) {
- long space = new File(dir).getFreeSpace();
++ File file = new File(dir);
++ if (!file.exists() && !file.mkdirs()) {
++ return 0;
++ }
++ long space = file.getFreeSpace();
+ LOGGER.trace("Folder {} has {} available bytes.", dir, space);
+ return space;
+ }
+
+ protected boolean hasSpace(String dir) {
+ return getUsableSpace(dir) > 0;
+ }
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
index 4c68178,4363cc3..0000000
deleted file mode 100644,100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeManager.java
+++ /dev/null
@@@ -1,1317 -1,1252 +1,0 @@@
- ///**
- // * Licensed to the Apache Software Foundation (ASF) under one
- // * or more contributor license agreements. See the NOTICE file
- // * distributed with this work for additional information
- // * regarding copyright ownership. The ASF licenses this file
- // * to you under the Apache License, Version 2.0 (the
- // * "License"); you may not use this file except in compliance
- // * with the License. You may obtain a copy of the License at
- // *
- // * http://www.apache.org/licenses/LICENSE-2.0
- // *
- // * Unless required by applicable law or agreed to in writing,
- // * software distributed under the License is distributed on an
- // * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // * KIND, either express or implied. See the License for the
- // * specific language governing permissions and limitations
- // * under the License.
- // */
- //
- //package org.apache.iotdb.db.engine.filenode;
- //
- //import java.io.File;
- //import java.io.IOException;
- //import java.util.ArrayList;
- //import java.util.HashMap;
- //import java.util.Iterator;
- //import java.util.List;
- //import java.util.Map;
- //import java.util.concurrent.ConcurrentHashMap;
- //import java.util.concurrent.ExecutionException;
- //import java.util.concurrent.Future;
- //import java.util.concurrent.ScheduledExecutorService;
- //import java.util.concurrent.TimeUnit;
- //import java.util.concurrent.atomic.AtomicLong;
- //import org.apache.commons.io.FileUtils;
- //import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
- //import org.apache.iotdb.db.conf.IoTDBConfig;
- //import org.apache.iotdb.db.conf.IoTDBConstant;
- //import org.apache.iotdb.db.conf.IoTDBDescriptor;
- //import org.apache.iotdb.db.conf.directories.DirectoryManager;
- //import org.apache.iotdb.db.engine.Processor;
- //import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
- //import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
- //import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
- //import org.apache.iotdb.db.engine.pool.FlushPoolManager;
- //import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
- //import org.apache.iotdb.db.exception.BufferWriteProcessorException;
- //import org.apache.iotdb.db.exception.FileNodeManagerException;
- //import org.apache.iotdb.db.exception.FileNodeProcessorException;
- //import org.apache.iotdb.db.exception.PathErrorException;
- //import org.apache.iotdb.db.exception.ProcessorException;
- //import org.apache.iotdb.db.metadata.MManager;
- //import org.apache.iotdb.db.monitor.IStatistic;
- //import org.apache.iotdb.db.monitor.MonitorConstants;
- //import org.apache.iotdb.db.monitor.StatMonitor;
- //import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
- //import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
- //import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
- //import org.apache.iotdb.db.query.context.QueryContext;
- //import org.apache.iotdb.db.query.control.FileReaderManager;
- //import org.apache.iotdb.db.service.IService;
- //import org.apache.iotdb.db.service.ServiceType;
- //import org.apache.iotdb.db.utils.MemUtils;
- //import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
- //import org.apache.iotdb.db.writelog.node.WriteLogNode;
- //import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
- //import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- //import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
- //import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
- //import org.apache.iotdb.tsfile.read.common.Path;
- //import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
- //import org.apache.iotdb.tsfile.write.record.TSRecord;
- //import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
- //import org.slf4j.Logger;
- //import org.slf4j.LoggerFactory;
- //
- //public class FileNodeManager implements IStatistic, IService {
- //
- // private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeManagerV2.class);
- // private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- // private static final DirectoryManager DIRECTORY_MANAGER = DirectoryManager.getInstance();
- // /**
- // * a folder that persist FileNodeProcessorStore classes. Each stroage group will have a subfolder.
- // * by default, it is system/info
- // */
- // private final String baseDir;
- //
- // /**
- // * This map is used to manage all filenode processor,<br> the key is filenode name which is
- // * storage group seriesPath.
- // */
- // private ConcurrentHashMap<String, FileNodeProcessor> processorMap;
- // /**
- // * This set is used to store overflowed filenode name.<br> The overflowed filenode will be merge.
- // */
- // private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // // There is no need to add concurrently
- // private HashMap<String, AtomicLong> statParamsHashMap;
- //
- // ScheduledExecutorService closedProcessorCleaner = IoTDBThreadPoolFactory.newScheduledThreadPool(1,
- // "Closed FileNodeProcessors Cleaner");
- //
- // private FileNodeManager(String baseDir) {
- // processorMap = new ConcurrentHashMap<>();
- // statParamsHashMap = new HashMap<>();
- // //label: A
- // for (MonitorConstants.FileNodeManagerStatConstants fileNodeManagerStatConstant :
- // MonitorConstants.FileNodeManagerStatConstants.values()) {
- // statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
- // }
- //
- // String normalizedBaseDir = baseDir;
- // if (normalizedBaseDir.charAt(normalizedBaseDir.size() - 1) != File.separatorChar) {
- // normalizedBaseDir += Character.toString(File.separatorChar);
- // }
- // this.baseDir = normalizedBaseDir;
- // File dir = new File(normalizedBaseDir);
- // if (dir.mkdirs()) {
- // LOGGER.info("{} dir home doesn't exist, create it", dir.getPathString());
- // }
- // //TODO merge this with label A
- // if (TsFileDBConf.isEnableStatMonitor()) {
- // StatMonitor statMonitor = StatMonitor.getInstance();
- // registerStatMetadata();
- // statMonitor.registerStatistics(MonitorConstants.STAT_STORAGE_DELTA_NAME, this);
- // }
- //
- // closedProcessorCleaner.scheduleWithFixedDelay(()->{
- // int size = 0;
- // for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
- // size += fileNodeProcessor.getClosingBufferWriteProcessor().size();
- // }
- // if (size > 5) {
- // LOGGER.info("Current closing processor number is {}", size);
- // }
- //// for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
- //// fileNodeProcessor.checkAllClosingProcessors();
- //// }
- // }, 0, 30000, TimeUnit.MILLISECONDS);
- //
- // }
- //
- // public static FileNodeManager getInstance() {
- // return FileNodeManagerHolder.INSTANCE;
- // }
- //
- // private void updateStatHashMapWhenFail(TSRecord tsRecord) {
- // statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_FAIL.name())
- // .incrementAndGet();
- // statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_FAIL.name())
- // .addAndGet(tsRecord.dataPointList.size());
- // }
- //
- // /**
- // * get stats parameter hash map.
- // *
- // * @return the key represents the params' name, values is AtomicLong type
- // */
- // @Override
- // public Map<String, AtomicLong> getStatParamsHashMap() {
- // return statParamsHashMap;
- // }
- //
- // @Override
- // public List<String> getAllPathForStatistic() {
- // List<String> list = new ArrayList<>();
- // for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- // MonitorConstants.FileNodeManagerStatConstants.values()) {
- // list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- // + statConstant.name());
- // }
- // return list;
- // }
- //
- // @Override
- // public Map<String, TSRecord> getAllStatisticsValue() {
- // long curTime = System.currentTimeMillis();
- // TSRecord tsRecord = StatMonitor
- // .convertToTSRecord(getStatParamsHashMap(), MonitorConstants.STAT_STORAGE_DELTA_NAME,
- // curTime);
- // HashMap<String, TSRecord> ret = new HashMap<>();
- // ret.put(MonitorConstants.STAT_STORAGE_DELTA_NAME, tsRecord);
- // return ret;
- // }
- //
- // /**
- // * Init Stat MetaDta.
- // */
- // @Override
- // public void registerStatMetadata() {
- // Map<String, String> hashMap = new HashMap<>();
- // for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- // MonitorConstants.FileNodeManagerStatConstants.values()) {
- // hashMap
- // .put(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- // + statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
- // }
- // StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- // }
- //
- // /**
- // * This function is just for unit test.
- // */
- // public synchronized void resetFileNodeManager() {
- // for (String key : statParamsHashMap.keySet()) {
- // statParamsHashMap.put(key, new AtomicLong());
- // }
- // processorMap.clear();
- // }
- //
- // /**
- // * @param filenodeName storage name, e.g., root.a.b
- // */
- // private FileNodeProcessor constructNewProcessor(String filenodeName)
- // throws FileNodeManagerException {
- // try {
- // return new FileNodeProcessor(baseDir, filenodeName);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("Can't construct the FileNodeProcessor, the filenode is {}", filenodeName, e);
- // throw new FileNodeManagerException(e);
- // }
- // }
- //
- // private FileNodeProcessor getProcessor(String path, boolean isWriteLock)
- // throws FileNodeManagerException {
- // String filenodeName;
- // try {
- // // return the stroage name
- // filenodeName = MManager.getInstance().getFileNameByPath(path);
- // } catch (PathErrorException e) {
- // LOGGER.error("MManager get filenode name error, seriesPath is {}", path);
- // throw new FileNodeManagerException(e);
- // }
- // FileNodeProcessor processor;
- // processor = processorMap.get(filenodeName);
- // if (processor != null) {
- // processor.lock(isWriteLock);
- // } else {
- // filenodeName = filenodeName.intern();
- // // calculate the value with same key synchronously
- // synchronized (filenodeName) {
- // processor = processorMap.get(filenodeName);
- // if (processor != null) {
- // processor.lock(isWriteLock);
- // } else {
- // // calculate the value with the key monitor
- // LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
- // filenodeName, Thread.currentThread().getId());
- // processor = constructNewProcessor(filenodeName);
- // processor.lock(isWriteLock);
- // processorMap.put(filenodeName, processor);
- // }
- // }
- // }
- // return processor;
- // }
- //
- // /**
- // * recovery the filenode processor.
- // */
- // public void recovery() {
- // List<String> filenodeNames = null;
- // try {
- // filenodeNames = MManager.getInstance().getAllFileNames();
- // } catch (PathErrorException e) {
- // LOGGER.error("Restoring all FileNodes failed.", e);
- // return;
- // }
- // for (String filenodeName : filenodeNames) {
- // FileNodeProcessor fileNodeProcessor = null;
- // try {
- // // recover in initialization
- // fileNodeProcessor = getProcessor(filenodeName, true);
- // } catch (FileNodeManagerException e) {
- // LOGGER.error("Restoring fileNode {} failed.", filenodeName, e);
- // } finally {
- // if (fileNodeProcessor != null) {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- // // add index check sum
- // }
- // }
- //
- // /**
- // * insert TsRecord into storage group.
- // *
- // * @param tsRecord input Data
- // * @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
- // * be recorded. if false, the statParamsHashMap will be updated.
- // * @return an int value represents the insert type, 0: failed; 1: overflow; 2: bufferwrite
- // */
- // public int insert(TSRecord tsRecord, boolean isMonitor) throws FileNodeManagerException {
- // long timestamp = tsRecord.time;
- //
- // String deviceId = tsRecord.deviceId;
- // checkTimestamp(tsRecord);
- //// //if memory is dangerous, directly reject
- //// long memUsage = MemUtils.getRecordSize(tsRecord);
- //// BasicMemController.UsageLevel level = BasicMemController.getInstance()
- //// .acquireUsage(this, memUsage);
- //// if (level == UsageLevel.DANGEROUS) {
- //// return 0;
- //// }
- //
- // updateStat(isMonitor, tsRecord);
- //
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // int insertType;
- //
- // try {
- // long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
- // if (timestamp < lastUpdateTime) {
- //
- // long startOverflow = System.currentTimeMillis();
- //
- // insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
- //
- // startOverflow = System.currentTimeMillis() - startOverflow;
- // if (startOverflow > 1000) {
- // LOGGER.info("has overflow data, insert cost: {}", startOverflow);
- // }
- //
- // insertType = 1;
- // } else {
- // insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
- // insertType = 2;
- // }
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error(String.format("Encounter an error when closing the buffer insert processor %s.",
- // fileNodeProcessor.getProcessorName()), e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // // Modify the insert
- // if (!isMonitor) {
- // fileNodeProcessor.getStatParamsHashMap()
- // .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
- // .addAndGet(tsRecord.dataPointList.size());
- // fileNodeProcessor.getStatParamsHashMap()
- // .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
- // .incrementAndGet();
- // statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_SUCCESS.name())
- // .incrementAndGet();
- // statParamsHashMap
- // .get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
- // .addAndGet(tsRecord.dataPointList.size());
- // }
- // return insertType;
- // }
- //
- // private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
- // throws FileNodeManagerException {
- // try {
- // if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // String[] measurementList = new String[tsRecord.dataPointList.size()];
- // String[] insertValues = new String[tsRecord.dataPointList.size()];
- // int i=0;
- // for (DataPoint dp : tsRecord.dataPointList) {
- // measurementList[i] = dp.getMeasurementId();
- // insertValues[i] = dp.getValue().toString();
- // i++;
- // }
- // logNode.write(new InsertPlan(2, tsRecord.deviceId, tsRecord.time, measurementList,
- // insertValues));
- // }
- // } catch (IOException e) {
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // }
- // }
- //
- // private void checkTimestamp(TSRecord tsRecord) throws FileNodeManagerException {
- // if (tsRecord.time < 0) {
- // LOGGER.error("The insert time lt 0, {}.", tsRecord);
- // throw new FileNodeManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
- // }
- // }
- //
- // private void updateStat(boolean isMonitor, TSRecord tsRecord) {
- // if (!isMonitor) {
- // statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS.name())
- // .addAndGet(tsRecord.dataPointList.size());
- // }
- // }
- //
- // private void insertOverflow(FileNodeProcessor fileNodeProcessor, long timestamp,
- // TSRecord tsRecord, boolean isMonitor, String deviceId)
- // throws FileNodeManagerException {
- // // get overflow processor
- // OverflowProcessor overflowProcessor;
- // String filenodeName = fileNodeProcessor.getProcessorName();
- // try {
- // overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // } catch (ProcessorException e) {
- // LOGGER.error("Get the overflow processor failed, the filenode is {}, insert time is {}",
- // filenodeName, timestamp);
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // }
- // // insert wal
- // try {
- // writeLog(tsRecord, isMonitor, overflowProcessor.getLogNode());
- // } catch (IOException e) {
- // throw new FileNodeManagerException(e);
- // }
- // // insert overflow data
- // try {
- // overflowProcessor.insert(tsRecord);
- // fileNodeProcessor.changeTypeToChanged(deviceId, timestamp);
- // fileNodeProcessor.setOverflowed(true);
- // } catch (IOException e) {
- // LOGGER.error("Insert into overflow error, the reason is {}", e);
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // }
- // }
- //
- // private void insertBufferWrite(FileNodeProcessor fileNodeProcessor, long timestamp,
- // boolean isMonitor, TSRecord tsRecord, String deviceId)
- // throws FileNodeManagerException, FileNodeProcessorException {
- //
- // long start1 = System.currentTimeMillis();
- // // get bufferwrite processor
- // BufferWriteProcessor bufferWriteProcessor;
- // String filenodeName = fileNodeProcessor.getProcessorName();
- // try {
- // bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName, timestamp);
- //
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("Get the bufferwrite processor failed, the filenode is {}, insert time is {}",
- // filenodeName, timestamp);
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // } finally {
- // long start1_1 = System.currentTimeMillis() - start1;
- // if (start1_1 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step-1-1, cost: {}", start1_1);
- // }
- // }
- //
- // long start1_2 = System.currentTimeMillis();
- // // Add a new interval file to newfilelist
- // if (bufferWriteProcessor.isNewProcessor()) {
- // bufferWriteProcessor.setNewProcessor(false);
- // String bufferwriteBaseDir = bufferWriteProcessor.getBaseDir();
- // String bufferwriteRelativePath = bufferWriteProcessor.getFileRelativePath();
- // try {
- // bufferWriteProcessor.setCurrentTsFileResource(new TsFileResource(new File(new File(bufferwriteBaseDir), bufferwriteRelativePath), false));
- // fileNodeProcessor.addIntervalFileNode(bufferWriteProcessor.getCurrentTsFileResource());
- // } catch (Exception e) {
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // }
- // }
- // start1_2 = System.currentTimeMillis() - start1_2;
- // if (start1_2 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step-1-2, cost: {}", start1_2);
- // }
- //
- // start1 = System.currentTimeMillis() - start1;
- // if (start1 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step-1, cost: {}", start1);
- // }
- //
- // long start2 = System.currentTimeMillis();
- //
- // long start2_1 = start2;
- // // insert wal
- // try {
- // writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
- // } catch (IOException e) {
- // throw new FileNodeManagerException(e);
- // }
- // start2_1 = System.currentTimeMillis() - start2_1;
- // if (start2_1 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step2-1 cost: {}", start2_1);
- // }
- //
- // long start2_2 = System.currentTimeMillis();
- // // Write data
- // long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
- // long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- //
- // fileNodeProcessor.setIntervalFileNodeStartTime(deviceId);
- // fileNodeProcessor.setLastUpdateTime(deviceId, timestamp);
- //
- // start2_2 = System.currentTimeMillis() - start2_2;
- // if (start2_2 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step2-2 cost: {}", start2_2);
- // }
- // try {
- // long start2_3 = System.currentTimeMillis();
- //
- // // insert tsrecord and check flush
- // if (!bufferWriteProcessor.write(tsRecord)) {
- // start2_3 = System.currentTimeMillis() - start2_3;
- // if (start2_3 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step2-3 cost: {}", start2_3);
- // }
- //
- // long start2_4 = System.currentTimeMillis();
- // // undo time update
- // fileNodeProcessor.setIntervalFileNodeStartTime(deviceId, prevStartTime);
- // fileNodeProcessor.setLastUpdateTime(deviceId, prevUpdateTime);
- // start2_4 = System.currentTimeMillis() - start2_4;
- // if (start2_4 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step2-4 cost: {}", start2_4);
- // }
- // }
- // } catch (BufferWriteProcessorException e) {
- // if (!isMonitor) {
- // updateStatHashMapWhenFail(tsRecord);
- // }
- // throw new FileNodeManagerException(e);
- // }
- // start2 = System.currentTimeMillis() - start2;
- // if (start2 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step-2, cost: {}", start2);
- // }
- //
- // long start3 = System.currentTimeMillis();
- //
- // // check if the file should be closed
- // if (bufferWriteProcessor
- // .getFileSize() > IoTDBDescriptor.getInstance()
- // .getConfig().getBufferwriteFileSizeThreshold()) {
- // if (LOGGER.isInfoEnabled()) {
- // LOGGER.info(
- // "The filenode processor {} will close the bufferwrite processor, "
- // + "because the size[{}] of tsfile {} reaches the threshold {}",
- // filenodeName, MemUtils.bytesCntToStr(bufferWriteProcessor.getFileSize()),
- // bufferWriteProcessor.getInsertFilePath(), MemUtils.bytesCntToStr(
- // IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold()));
- // }
- //
- // fileNodeProcessor.closeBufferWrite();
- // start3 = System.currentTimeMillis() - start3;
- // if (start3 > 1000) {
- // LOGGER.info("FileNodeManagerV2.insertBufferWrite step-3, close buffer insert cost: {}", start3);
- // }
- // }
- // }
- //
- // /**
- // * update data.
- // */
- // public void update(String deviceId, String measurementId, long startTime, long endTime,
- // TSDataType type, String v)
- // throws FileNodeManagerException {
- //
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- //
- // long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- // if (startTime > lastUpdateTime) {
- // LOGGER.warn("The update range is error, startTime {} is great than lastUpdateTime {}",
- // startTime,
- // lastUpdateTime);
- // return;
- // }
- // long finalEndTime = endTime > lastUpdateTime ? lastUpdateTime : endTime;
- //
- // String filenodeName = fileNodeProcessor.getProcessorName();
- // // get overflow processor
- // OverflowProcessor overflowProcessor;
- // try {
- // overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // } catch (ProcessorException e) {
- // LOGGER.error(
- // "Get the overflow processor failed, the filenode is {}, "
- // + "insert time range is from {} to {}",
- // filenodeName, startTime, finalEndTime);
- // throw new FileNodeManagerException(e);
- // }
- // overflowProcessor.update(deviceId, measurementId, startTime, finalEndTime, type, v);
- // // change the type of tsfile to overflowed
- // fileNodeProcessor.changeTypeToChanged(deviceId, startTime, finalEndTime);
- // fileNodeProcessor.setOverflowed(true);
- //
- // // insert wal
- // try {
- // if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // overflowProcessor.getLogNode().write(
- // new UpdatePlan(startTime, finalEndTime, v, new Path(deviceId
- // + "." + measurementId)));
- // }
- // } catch (IOException e) {
- // throw new FileNodeManagerException(e);
- // }
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- //
- // /**
- // * delete data.
- // */
- // public void delete(String deviceId, String measurementId, long timestamp)
- // throws FileNodeManagerException {
- //
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- // long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- // // no tsfile data, the delete operation is invalid
- // if (lastUpdateTime == -1) {
- // LOGGER.warn("The last update time is -1, delete overflow is invalid, "
- // + "the filenode processor is {}",
- // fileNodeProcessor.getProcessorName());
- // } else {
- // // insert wal
- // if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // // get processors for wal
- // String filenodeName = fileNodeProcessor.getProcessorName();
- // OverflowProcessor overflowProcessor;
- // BufferWriteProcessor bufferWriteProcessor;
- // try {
- // overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // // in case that no BufferWriteProcessor is available, a new BufferWriteProcessor is
- // // needed to access LogNode.
- // // TODO this may make the time range of the next TsFile a little wider
- // bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName,
- // lastUpdateTime + 1);
- // } catch (ProcessorException e) {
- // LOGGER.error("Getting the processor failed, the filenode is {}, delete time is {}.",
- // filenodeName, timestamp);
- // throw new FileNodeManagerException(e);
- // }
- // try {
- // overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
- // new Path(deviceId + "." + measurementId)));
- // bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
- // new Path(deviceId + "." + measurementId)));
- // } catch (IOException e) {
- // throw new FileNodeManagerException(e);
- // }
- // }
- //
- // try {
- // fileNodeProcessor.delete(deviceId, measurementId, timestamp);
- // } catch (IOException e) {
- // throw new FileNodeManagerException(e);
- // }
- // // change the type of tsfile to overflowed
- // fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- // fileNodeProcessor.setOverflowed(true);
- //
- // }
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- //
- // private void delete(String processorName,
- // Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- // throws FileNodeManagerException {
- // if (!processorMap.containsKey(processorName)) {
- // //TODO do we need to call processorIterator.remove() ?
- // LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- // return;
- // }
- // LOGGER.info("Try to delete the filenode processor {}.", processorName);
- // FileNodeProcessor processor = processorMap.get(processorName);
- // if (!processor.tryWriteLock()) {
- // throw new FileNodeManagerException(String
- // .format("Can't delete the filenode processor %s because Can't get the insert lock.",
- // processorName));
- // }
- //
- // try {
- // if (!processor.canBeClosed()) {
- // LOGGER.warn("The filenode processor {} can't be deleted.", processorName);
- // return;
- // }
- //
- // try {
- // LOGGER.info("Delete the filenode processor {}.", processorName);
- // processor.delete();
- // processorIterator.remove();
- // } catch (ProcessorException e) {
- // LOGGER.error("Delete the filenode processor {} by iterator error.", processorName, e);
- // throw new FileNodeManagerException(e);
- // }
- // } finally {
- // processor.writeUnlock();
- // }
- // }
- //
- // /**
- // * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- // */
- // public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- // throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- // fileNodeProcessor.deleteBufferWrite(deviceId, measurementId, timestamp);
- // } catch (BufferWriteProcessorException | IOException e) {
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // // change the type of tsfile to overflowed
- // fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- // fileNodeProcessor.setOverflowed(true);
- // }
- //
- // /**
- // * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- // */
- // public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- // throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- // fileNodeProcessor.deleteOverflow(deviceId, measurementId, timestamp);
- // } catch (ProcessorException e) {
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // // change the type of tsfile to overflowed
- // fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- // fileNodeProcessor.setOverflowed(true);
- // }
- //
- // /**
- // * begin query.
- // *
- // * @param deviceId queried deviceId
- // * @return a query token for the device.
- // */
- // public int beginQuery(String deviceId) throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- // LOGGER.debug("Get the FileNodeProcessor: filenode is {}, begin query.",
- // fileNodeProcessor.getProcessorName());
- // return fileNodeProcessor.addMultiPassCount();
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- //
- // /**
- // * query data.
- // */
- // public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
- // throws FileNodeManagerException {
- // String deviceId = seriesExpression.getSeriesPath().getDevice();
- // String measurementId = seriesExpression.getSeriesPath().getMeasurement();
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, false);
- // LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
- // fileNodeProcessor.getProcessorName());
- // try {
- // QueryDataSource queryDataSource;
- // // query operation must have overflow processor
- // if (!fileNodeProcessor.hasOverflowProcessor()) {
- // try {
- // fileNodeProcessor.getOverflowProcessor(fileNodeProcessor.getProcessorName());
- // } catch (ProcessorException e) {
- // LOGGER.error("Get the overflow processor failed, the filenode is {}, query is {},{}",
- // fileNodeProcessor.getProcessorName(), deviceId, measurementId);
- // throw new FileNodeManagerException(e);
- // }
- // }
- // try {
- // queryDataSource = fileNodeProcessor.query(deviceId, measurementId, context);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("Query error: the deviceId {}, the measurementId {}", deviceId, measurementId,
- // e);
- // throw new FileNodeManagerException(e);
- // }
- // // return query structure
- // return queryDataSource;
- // } finally {
- // fileNodeProcessor.readUnlock();
- // }
- // }
- //
- // /**
- // * end query.
- // */
- // public void endQuery(String deviceId, int token) throws FileNodeManagerException {
- //
- // FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- // try {
- // LOGGER.debug("Get the FileNodeProcessor: {} end query.",
- // fileNodeProcessor.getProcessorName());
- // fileNodeProcessor.decreaseMultiPassCount(token);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("Failed to end query: the deviceId {}, token {}.", deviceId, token, e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- //
- // /**
- // * Append one specified tsfile to the storage group. <b>This method is only provided for
- // * transmission module</b>
- // *
- // * @param fileNodeName the seriesPath of storage group
- // * @param appendFile the appended tsfile information
- // */
- // public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
- // String appendFilePath) throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- // try {
- // // check append file
- // for (Map.Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- // if (fileNodeProcessor.getLastUpdateTime(entry.getKey()) >= entry.getValue()) {
- // return false;
- // }
- // }
- // // close bufferwrite file
- // fileNodeProcessor.closeBufferWrite();
- // // append file to storage group.
- // fileNodeProcessor.appendFile(appendFile, appendFilePath);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // return true;
- // }
- //
- // /**
- // * get all overlap tsfiles which are conflict with the appendFile.
- // *
- // * @param fileNodeName the seriesPath of storage group
- // * @param appendFile the appended tsfile information
- // */
- // public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
- // String uuid) throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- // List<String> overlapFiles;
- // try {
- // overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
- // } catch (FileNodeProcessorException e) {
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // return overlapFiles;
- // }
- //
- // /**
- // * merge all overflowed filenode.
- // *
- // * @throws FileNodeManagerException FileNodeManagerException
- // */
- // public void mergeAll() throws FileNodeManagerException {
- // if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- // LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
- // fileNodeManagerStatus);
- // return;
- // }
- //
- // fileNodeManagerStatus = FileNodeManagerStatus.MERGE;
- // LOGGER.info("Start to merge all overflowed filenode");
- // List<String> allFileNodeNames;
- // try {
- // allFileNodeNames = MManager.getInstance().getAllFileNames();
- // } catch (PathErrorException e) {
- // LOGGER.error("Get all storage group seriesPath error,", e);
- // throw new FileNodeManagerException(e);
- // }
- // List<Future<?>> futureTasks = new ArrayList<>();
- // for (String fileNodeName : allFileNodeNames) {
- // FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- // try {
- // Future<?> task = fileNodeProcessor.submitToMerge();
- // if (task != null) {
- // LOGGER.info("Submit the filenode {} to the merge pool", fileNodeName);
- // futureTasks.add(task);
- // }
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- // long totalTime = 0;
- // // loop waiting for merge to end, the longest waiting time is
- // // 60s.
- // int time = 2;
- // List<Exception> mergeException = new ArrayList<>();
- // for (Future<?> task : futureTasks) {
- // while (!task.isDone()) {
- // try {
- // LOGGER.info(
- // "Waiting for the end of merge, already waiting for {}s, "
- // + "continue to wait anothor {}s",
- // totalTime, time);
- // TimeUnit.SECONDS.sleep(time);
- // totalTime += time;
- // time = updateWaitTime(time);
- // } catch (InterruptedException e) {
- // LOGGER.error("Unexpected interruption {}", e);
- // Thread.currentThread().interrupt();
- // }
- // }
- // try {
- // task.get();
- // } catch (InterruptedException e) {
- // LOGGER.error("Unexpected interruption {}", e);
- // } catch (ExecutionException e) {
- // mergeException.add(e);
- // LOGGER.error("The exception for merge: {}", e);
- // }
- // }
- // if (!mergeException.isEmpty()) {
- // // just throw the first exception
- // throw new FileNodeManagerException(mergeException.get(0));
- // }
- // fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // LOGGER.info("End to merge all overflowed filenode");
- // }
- //
- // private int updateWaitTime(int time) {
- // return time < 32 ? time * 2 : 60;
- // }
- //
- // /**
- // * delete one filenode.
- // */
- // public void deleteOneFileNode(String processorName) throws FileNodeManagerException {
- // if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- // return;
- // }
- //
- // fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- // try {
- // if (processorMap.containsKey(processorName)) {
- // deleteFileNodeBlocked(processorName);
- // }
- // String fileNodePath = TsFileDBConf.getFileNodeDir();
- // fileNodePath = standardizeDir(fileNodePath) + processorName;
- // FileUtils.deleteDirectory(new File(fileNodePath));
- //
- // cleanBufferWrite(processorName);
- //
- // MultiFileLogNodeManager.getInstance()
- // .deleteNode(processorName + IoTDBConstant.BUFFERWRITE_LOG_NODE_SUFFIX);
- // MultiFileLogNodeManager.getInstance()
- // .deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
- // } catch (IOException e) {
- // LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // }
- // }
- //
- // private void cleanBufferWrite(String processorName) throws IOException {
- // List<String> bufferwritePathList = DIRECTORY_MANAGER.getAllTsFileFolders();
- // for (String bufferwritePath : bufferwritePathList) {
- // bufferwritePath = standardizeDir(bufferwritePath) + processorName;
- // File bufferDir = new File(bufferwritePath);
- // // free and close the streams under this bufferwrite directory
- // if (!bufferDir.exists()) {
- // continue;
- // }
- // File[] bufferFiles = bufferDir.listFiles();
- // if (bufferFiles != null) {
- // for (File bufferFile : bufferFiles) {
- // FileReaderManager.getInstance().closeFileAndRemoveReader(bufferFile.getPathString());
- // }
- // }
- // FileUtils.deleteDirectory(new File(bufferwritePath));
- // }
- // }
- //
- // private void deleteFileNodeBlocked(String processorName) throws FileNodeManagerException {
- // LOGGER.info("Forced to delete the filenode processor {}", processorName);
- // FileNodeProcessor processor = processorMap.get(processorName);
- // while (true) {
- // if (processor.tryWriteLock()) {
- // try {
- // if (processor.canBeClosed()) {
- // LOGGER.info("Delete the filenode processor {}.", processorName);
- // processor.delete();
- // processorMap.remove(processorName);
- // break;
- // } else {
- // LOGGER.info(
- // "Can't delete the filenode processor {}, "
- // + "because the filenode processor can't be closed."
- // + " Wait 100ms to retry");
- // }
- // } catch (ProcessorException e) {
- // LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // processor.writeUnlock();
- // }
- // } else {
- // LOGGER.info(
- // "Can't delete the filenode processor {}, because it can't get the insert lock."
- // + " Wait 100ms to retry", processorName);
- // }
- // try {
- // TimeUnit.MILLISECONDS.sleep(100);
- // } catch (InterruptedException e) {
- // LOGGER.error(e.getMessage());
- // Thread.currentThread().interrupt();
- // }
- // }
- // }
- //
- // private String standardizeDir(String originalPath) {
- // String res = originalPath;
- // if ((originalPath.size() > 0
- // && originalPath.charAt(originalPath.size() - 1) != File.separatorChar)
- // || originalPath.size() == 0) {
- // res = originalPath + File.separatorChar;
- // }
- // return res;
- // }
- //
- // /**
- // * add time series.
- // */
- // public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
- // CompressionType compressor,
- // Map<String, String> props) throws FileNodeManagerException {
- // FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
- // try {
- // fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
- // } finally {
- // fileNodeProcessor.writeUnlock();
- // }
- // }
- //
- //
- // /**
- // * Force to close the filenode processor.
- // */
- // public void closeOneFileNode(String processorName) throws FileNodeManagerException {
- // if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- // return;
- // }
- //
- // fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- // try {
- // LOGGER.info("Force to close the filenode processor {}.", processorName);
- // while (!closeOneProcessor(processorName)) {
- // try {
- // LOGGER.info("Can't force to close the filenode processor {}, wait 100ms to retry",
- // processorName);
- // TimeUnit.MILLISECONDS.sleep(100);
- // } catch (InterruptedException e) {
- // // ignore the interrupted exception
- // LOGGER.error("Unexpected interruption {}", e);
- // Thread.currentThread().interrupt();
- // }
- // }
- // } finally {
- // fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // }
- // }
- //
- //
- // /**
- // * try to close the filenode processor. The name of filenode processor is processorName
- // * notice: this method has the same function with close()
- // */
- // private boolean closeOneProcessor(String processorName) throws FileNodeManagerException {
- // if (!processorMap.containsKey(processorName)) {
- // return true;
- // }
- //
- // Processor processor = processorMap.get(processorName);
- // if (processor.tryWriteLock()) {
- // try {
- // if (processor.canBeClosed()) {
- // processor.close();
- // return true;
- // } else {
- // return false;
- // }
- // } catch (ProcessorException e) {
- // LOGGER.error("Close the filenode processor {} error.", processorName, e);
- // throw new FileNodeManagerException(e);
- // } finally {
- // processor.writeUnlock();
- // }
- // } else {
- // return false;
- // }
- // }
- //
- // /**
- // * try to close the filenode processor.
- // * notice: This method has the same function with closeOneProcessor()
- // */
- // private void close(String processorName) throws FileNodeManagerException {
- // if (!processorMap.containsKey(processorName)) {
- // LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- // return;
- // }
- // LOGGER.info("Try to close the filenode processor {}.", processorName);
- // FileNodeProcessor processor = processorMap.get(processorName);
- // if (!processor.tryWriteLock()) {
- // LOGGER.warn("Can't get the insert lock of the filenode processor {}.", processorName);
- // return;
- // }
- // try {
- // if (processor.canBeClosed()) {
- // try {
- // LOGGER.info("Close the filenode processor {}.", processorName);
- // processor.close();
- // } catch (ProcessorException e) {
- // LOGGER.error("Close the filenode processor {} error.", processorName, e);
- // throw new FileNodeManagerException(e);
- // }
- // } else {
- // LOGGER.warn("The filenode processor {} can't be closed.", processorName);
- // }
- // } finally {
- // processor.writeUnlock();
- // }
- // }
- //
- // /**
- // * delete all filenode.
- // */
- // public synchronized boolean deleteAll() throws FileNodeManagerException {
- // LOGGER.info("Start deleting all filenode");
- // if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- // LOGGER.info("Failed to delete all filenode processor because of merge operation");
- // return false;
- // }
- //
- // fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- // try {
- // Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator = processorMap.entrySet()
- // .iterator();
- // while (processorIterator.hasNext()) {
- // Map.Entry<String, FileNodeProcessor> processorEntry = processorIterator.next();
- // delete(processorEntry.getKey(), processorIterator);
- // }
- // return processorMap.isEmpty();
- // } finally {
- // LOGGER.info("Deleting all FileNodeProcessors ends");
- // fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // }
- // }
- //
- // /**
- // * Try to close All.
- // */
- // public void closeAll() throws FileNodeManagerException {
- // LOGGER.info("Start closing all filenode processor");
- // if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- // LOGGER.info("Failed to close all filenode processor because of merge operation");
- // return;
- // }
- // fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- // try {
- // for (Map.Entry<String, FileNodeProcessor> processorEntry : processorMap.entrySet()) {
- // close(processorEntry.getKey());
- // }
- // } finally {
- // LOGGER.info("Close all FileNodeProcessors ends");
- // fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // }
- // }
- //
- // /**
- // * force flush to control memory usage.
- // */
- // public void forceFlush(BasicMemController.UsageLevel level) {
- // // you may add some delicate process like below
- // // or you could provide multiple methods for different urgency
- // switch (level) {
- // // only select the most urgent (most active or biggest in size)
- // // processors to flush
- // // only select top 10% active memory user to flush
- // case WARNING:
- // try {
- // flushTop(0.1f);
- // } catch (IOException e) {
- // LOGGER.error("force flush memory data error: {}", e);
- // }
- // break;
- // // force all processors to flush
- // case DANGEROUS:
- // try {
- // flushAll();
- // } catch (IOException e) {
- // LOGGER.error("force flush memory data error: {}", e);
- // }
- // break;
- // // if the flush thread pool is not full ( or half full), start a new
- // // flush task
- // case SAFE:
- // if (FlushPoolManager.getInstance().getActiveCnt() < 0.5 * FlushPoolManager.getInstance()
- // .getThreadCnt()) {
- // try {
- // flushTop(0.01f);
- // } catch (IOException e) {
- // LOGGER.error("force flush memory data error: ", e);
- // }
- // }
- // break;
- // default:
- // }
- // }
- //
- // private void flushAll() throws IOException {
- // for (FileNodeProcessor processor : processorMap.values()) {
- // if (!processor.tryLock(true)) {
- // continue;
- // }
- // try {
- // boolean isMerge = processor.flush().isHasOverflowFlushTask();
- // if (isMerge) {
- // processor.submitToMerge();
- // }
- // } finally {
- // processor.unlock(true);
- // }
- // }
- // }
- //
- // private void flushTop(float percentage) throws IOException {
- // List<FileNodeProcessor> tempProcessors = new ArrayList<>(processorMap.values());
- // // sort the tempProcessors as descending order
- // tempProcessors.sort((o1, o2) -> (int) (o2.memoryUsage() - o1.memoryUsage()));
- // int flushNum =
- // (int) (tempProcessors.size() * percentage) > 1
- // ? (int) (tempProcessors.size() * percentage)
- // : 1;
- // for (int i = 0; i < flushNum && i < tempProcessors.size(); i++) {
- // FileNodeProcessor processor = tempProcessors.get(i);
- // // 64M
- // if (processor.memoryUsage() <= TSFileConfig.groupSizeInByte / 2) {
- // continue;
- // }
- // long start = System.currentTimeMillis();
- // processor.writeLock();
- // try {
- // boolean isMerge = processor.flush().isHasOverflowFlushTask();
- // if (isMerge) {
- // processor.submitToMerge();
- // }
- // } finally {
- // processor.writeUnlock();
- // }
- // start = System.currentTimeMillis() - start;
- // LOGGER.info("flush Top cost: {}", start);
- // }
- // }
- //
- // @Override
- // public void start() {
- // // do no thing
- // }
- //
- // @Override
- // public void stop() {
- // try {
- // closeAll();
- // } catch (FileNodeManagerException e) {
- // LOGGER.error("Failed to close file node manager because .", e);
- // }
- //
- // boolean notFinished = true;
- // while (notFinished) {
- // int size = 0;
- // for (FileNodeProcessor fileNodeProcessor : processorMap.values()) {
- // size += fileNodeProcessor.getClosingBufferWriteProcessor().size();
- // }
- // if (size == 0) {
- // notFinished = false;
- // } else {
- // try {
- // Thread.sleep(10);
- // } catch (InterruptedException e) {
- // LOGGER.error("File node Manager Stop process is interrupted", e);
- // }
- // }
- // }
- // closedProcessorCleaner.shutdownNow();
- // }
- //
- // @Override
- // public ServiceType getID() {
- // return ServiceType.FILE_NODE_SERVICE;
- // }
- //
- // private enum FileNodeManagerStatus {
- // NONE, MERGE, CLOSE
- // }
- //
- // private static class FileNodeManagerHolder {
- //
- // private FileNodeManagerHolder() {
- // }
- //
- // private static final FileNodeManager INSTANCE = new FileNodeManager(
- // TsFileDBConf.getFileNodeDir());
- // }
- //
- //}
- //
- //
- //
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.filenode;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.memcontrol.BasicMemController;
-import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
-import org.apache.iotdb.db.engine.pool.FlushManager;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.FileNodeManagerException;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
-import org.apache.iotdb.db.exception.MetadataErrorException;
-import org.apache.iotdb.db.exception.PathErrorException;
-import org.apache.iotdb.db.exception.ProcessorException;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.monitor.IStatistic;
-import org.apache.iotdb.db.monitor.MonitorConstants;
-import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.UpdatePlan;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.service.IService;
-import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
-import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileNodeManager implements IStatistic, IService {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeManager.class);
- private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- private static final Directories directories = Directories.getInstance();
- /**
- * a folder that persist FileNodeProcessorStore classes. Each stroage group will have a subfolder.
- * by default, it is system/info
- */
- private final String baseDir;
-
- /**
- * This map is used to manage all filenode processor,<br> the key is filenode name which is
- * storage group seriesPath.
- */
- private ConcurrentHashMap<String, FileNodeProcessor> processorMap;
- /**
- * This set is used to store overflowed filenode name.<br> The overflowed filenode will be merge.
- */
- private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- // There is no need to add concurrently
- private HashMap<String, AtomicLong> statParamsHashMap;
-
- private FileNodeManager(String baseDir) {
- processorMap = new ConcurrentHashMap<>();
- statParamsHashMap = new HashMap<>();
- //label: A
- for (MonitorConstants.FileNodeManagerStatConstants fileNodeManagerStatConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
- statParamsHashMap.put(fileNodeManagerStatConstant.name(), new AtomicLong(0));
- }
-
- String normalizedBaseDir = baseDir;
- if (normalizedBaseDir.charAt(normalizedBaseDir.length() - 1) != File.separatorChar) {
- normalizedBaseDir += Character.toString(File.separatorChar);
- }
- this.baseDir = normalizedBaseDir;
- File dir = new File(normalizedBaseDir);
- if (dir.mkdirs()) {
- LOGGER.info("{} dir home doesn't exist, create it", dir.getPath());
- }
- //TODO merge this with label A
- if (TsFileDBConf.isEnableStatMonitor()) {
- StatMonitor statMonitor = StatMonitor.getInstance();
- registerStatMetadata();
- statMonitor.registerStatistics(MonitorConstants.STAT_STORAGE_DELTA_NAME, this);
- }
- }
-
- public static FileNodeManager getInstance() {
- return FileNodeManagerHolder.INSTANCE;
- }
-
- private void updateStatHashMapWhenFail(TSRecord tsRecord) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_FAIL.name())
- .incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_FAIL.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
-
- /**
- * get stats parameter hash map.
- *
- * @return the key represents the params' name, values is AtomicLong type
- */
- @Override
- public Map<String, AtomicLong> getStatParamsHashMap() {
- return statParamsHashMap;
- }
-
- @Override
- public List<String> getAllPathForStatistic() {
- List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
- list.add(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- + statConstant.name());
- }
- return list;
- }
-
- @Override
- public Map<String, TSRecord> getAllStatisticsValue() {
- long curTime = System.currentTimeMillis();
- TSRecord tsRecord = StatMonitor
- .convertToTSRecord(getStatParamsHashMap(), MonitorConstants.STAT_STORAGE_DELTA_NAME,
- curTime);
- HashMap<String, TSRecord> ret = new HashMap<>();
- ret.put(MonitorConstants.STAT_STORAGE_DELTA_NAME, tsRecord);
- return ret;
- }
-
- /**
- * Init Stat MetaDta.
- */
- @Override
- public void registerStatMetadata() {
- Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeManagerStatConstants statConstant :
- MonitorConstants.FileNodeManagerStatConstants.values()) {
- hashMap
- .put(MonitorConstants.STAT_STORAGE_DELTA_NAME + MonitorConstants.MONITOR_PATH_SEPARATOR
- + statConstant.name(), MonitorConstants.DATA_TYPE_INT64);
- }
- StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- }
-
- /**
- * This function is just for unit test.
- */
- public synchronized void resetFileNodeManager() {
- for (String key : statParamsHashMap.keySet()) {
- statParamsHashMap.put(key, new AtomicLong());
- }
- processorMap.clear();
- }
-
- /**
- * @param filenodeName storage name, e.g., root.a.b
- */
- private FileNodeProcessor constructNewProcessor(String filenodeName)
- throws FileNodeManagerException {
- try {
- return new FileNodeProcessor(baseDir, filenodeName);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Can't construct the FileNodeProcessor, the filenode is {}", filenodeName, e);
- throw new FileNodeManagerException(e);
- }
- }
-
- private FileNodeProcessor getProcessor(String path, boolean isWriteLock)
- throws FileNodeManagerException {
- String filenodeName;
- try {
- // return the stroage name
- filenodeName = MManager.getInstance().getFileNameByPath(path);
- } catch (PathErrorException e) {
- LOGGER.error("MManager get filenode name error, seriesPath is {}", path);
- throw new FileNodeManagerException(e);
- }
- FileNodeProcessor processor;
- processor = processorMap.get(filenodeName);
- if (processor != null) {
- processor.lock(isWriteLock);
- } else {
- filenodeName = filenodeName.intern();
- // calculate the value with same key synchronously
- synchronized (filenodeName) {
- processor = processorMap.get(filenodeName);
- if (processor != null) {
- processor.lock(isWriteLock);
- } else {
- // calculate the value with the key monitor
- LOGGER.debug("construct a processor instance, the filenode is {}, Thread is {}",
- filenodeName, Thread.currentThread().getId());
- processor = constructNewProcessor(filenodeName);
- processor.lock(isWriteLock);
- processorMap.put(filenodeName, processor);
- }
- }
- }
- return processor;
- }
-
- /**
- * recovery the filenode processor.
- */
- public void recovery() {
- List<String> filenodeNames = null;
- try {
- filenodeNames = MManager.getInstance().getAllFileNames();
- } catch (MetadataErrorException e) {
- LOGGER.error("Restoring all FileNodes failed.", e);
- return;
- }
- for (String filenodeName : filenodeNames) {
- FileNodeProcessor fileNodeProcessor = null;
- try {
- fileNodeProcessor = getProcessor(filenodeName, true);
- if (fileNodeProcessor.shouldRecovery()) {
- LOGGER.info("Recovery the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- fileNodeProcessor.fileNodeRecovery();
- }
- } catch (FileNodeManagerException | FileNodeProcessorException e) {
- LOGGER.error("Restoring fileNode {} failed.", filenodeName, e);
- } finally {
- if (fileNodeProcessor != null) {
- fileNodeProcessor.writeUnlock();
- }
- }
- // add index check sum
- }
- }
-
- /**
- * insert TsRecord into storage group.
- *
- * @param tsRecord input Data
- * @param isMonitor if true, the insertion is done by StatMonitor and the statistic Info will not
- * be recorded. if false, the statParamsHashMap will be updated.
- * @return an int value represents the insert type
- */
- public int insert(TSRecord tsRecord, boolean isMonitor) throws FileNodeManagerException {
- long timestamp = tsRecord.time;
-
- String deviceId = tsRecord.deviceId;
- checkTimestamp(tsRecord);
- updateStat(isMonitor, tsRecord);
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- int insertType;
-
- try {
- long lastUpdateTime = fileNodeProcessor.getFlushLastUpdateTime(deviceId);
- if (timestamp < lastUpdateTime) {
- insertOverflow(fileNodeProcessor, timestamp, tsRecord, isMonitor, deviceId);
- insertType = 1;
- } else {
- insertBufferWrite(fileNodeProcessor, timestamp, isMonitor, tsRecord, deviceId);
- insertType = 2;
- }
- } catch (FileNodeProcessorException e) {
- LOGGER.error(String.format("Encounter an error when closing the buffer write processor %s.",
- fileNodeProcessor.getProcessorName()), e);
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // Modify the insert
- if (!isMonitor) {
- fileNodeProcessor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
- fileNodeProcessor.getStatParamsHashMap()
- .get(MonitorConstants.FileNodeProcessorStatConstants.TOTAL_REQ_SUCCESS.name())
- .incrementAndGet();
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_REQ_SUCCESS.name())
- .incrementAndGet();
- statParamsHashMap
- .get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS_SUCCESS.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
- return insertType;
- }
-
- private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
- throws FileNodeManagerException {
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- String[] measurementList = new String[tsRecord.dataPointList.size()];
- String[] insertValues = new String[tsRecord.dataPointList.size()];
- int i=0;
- for (DataPoint dp : tsRecord.dataPointList) {
- measurementList[i] = dp.getMeasurementId();
- insertValues[i] = dp.getValue().toString();
- i++;
- }
- logNode.write(new InsertPlan(2, tsRecord.deviceId, tsRecord.time, measurementList,
- insertValues));
- }
- } catch (IOException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- }
-
- private void checkTimestamp(TSRecord tsRecord) throws FileNodeManagerException {
- if (tsRecord.time < 0) {
- LOGGER.error("The insert time lt 0, {}.", tsRecord);
- throw new FileNodeManagerException("The insert time lt 0, the tsrecord is " + tsRecord);
- }
- }
-
- private void updateStat(boolean isMonitor, TSRecord tsRecord) {
- if (!isMonitor) {
- statParamsHashMap.get(MonitorConstants.FileNodeManagerStatConstants.TOTAL_POINTS.name())
- .addAndGet(tsRecord.dataPointList.size());
- }
- }
-
- private void insertOverflow(FileNodeProcessor fileNodeProcessor, long timestamp,
- TSRecord tsRecord, boolean isMonitor, String deviceId)
- throws FileNodeManagerException {
- // get overflow processor
- OverflowProcessor overflowProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- // write wal
- try {
- writeLog(tsRecord, isMonitor, overflowProcessor.getLogNode());
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- // write overflow data
- try {
- overflowProcessor.insert(tsRecord);
- fileNodeProcessor.changeTypeToChanged(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- } catch (IOException e) {
- LOGGER.error("Insert into overflow error, the reason is {}", e);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- }
-
- private void insertBufferWrite(FileNodeProcessor fileNodeProcessor, long timestamp,
- boolean isMonitor, TSRecord tsRecord, String deviceId)
- throws FileNodeManagerException, FileNodeProcessorException {
- // get bufferwrite processor
- BufferWriteProcessor bufferWriteProcessor;
- String filenodeName = fileNodeProcessor.getProcessorName();
- try {
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName, timestamp);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Get the bufferwrite processor failed, the filenode is {}, insert time is {}",
- filenodeName, timestamp);
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- // Add a new interval file to newfilelist
- if (bufferWriteProcessor.isNewProcessor()) {
- bufferWriteProcessor.setNewProcessor(false);
- String bufferwriteBaseDir = bufferWriteProcessor.getBaseDir();
- String bufferwriteRelativePath = bufferWriteProcessor.getFileRelativePath();
- try {
- fileNodeProcessor.addIntervalFileNode(new File(new File(bufferwriteBaseDir), bufferwriteRelativePath));
- } catch (Exception e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
- }
- // write wal
- try {
- writeLog(tsRecord, isMonitor, bufferWriteProcessor.getLogNode());
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- // Write data
- long prevStartTime = fileNodeProcessor.getIntervalFileNodeStartTime(deviceId);
- long prevUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
-
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId);
- fileNodeProcessor.setLastUpdateTime(deviceId, timestamp);
- try {
- if (!bufferWriteProcessor.write(tsRecord)) {
- // undo time update
- fileNodeProcessor.setIntervalFileNodeStartTime(deviceId, prevStartTime);
- fileNodeProcessor.setLastUpdateTime(deviceId, prevUpdateTime);
- }
- } catch (BufferWriteProcessorException e) {
- if (!isMonitor) {
- updateStatHashMapWhenFail(tsRecord);
- }
- throw new FileNodeManagerException(e);
- }
-
- if (bufferWriteProcessor
- .getFileSize() > IoTDBDescriptor.getInstance()
- .getConfig().getBufferwriteFileSizeThreshold()) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "The filenode processor {} will close the bufferwrite processor, "
- + "because the size[{}] of tsfile {} reaches the threshold {}",
- filenodeName, MemUtils.bytesCntToStr(bufferWriteProcessor.getFileSize()),
- bufferWriteProcessor.getInsertFilePath(), MemUtils.bytesCntToStr(
- IoTDBDescriptor.getInstance().getConfig().getBufferwriteFileSizeThreshold()));
- }
-
- fileNodeProcessor.closeBufferWrite();
- }
- }
-
- /**
- * update data.
- */
- public void update(String deviceId, String measurementId, long startTime, long endTime,
- TSDataType type, String v)
- throws FileNodeManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
-
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- if (startTime > lastUpdateTime) {
- LOGGER.warn("The update range is error, startTime {} is great than lastUpdateTime {}",
- startTime,
- lastUpdateTime);
- return;
- }
- long finalEndTime = endTime > lastUpdateTime ? lastUpdateTime : endTime;
-
- String filenodeName = fileNodeProcessor.getProcessorName();
- // get overflow processor
- OverflowProcessor overflowProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- } catch (IOException e) {
- LOGGER.error(
- "Get the overflow processor failed, the filenode is {}, "
- + "insert time range is from {} to {}",
- filenodeName, startTime, finalEndTime);
- throw new FileNodeManagerException(e);
- }
- overflowProcessor.update(deviceId, measurementId, startTime, finalEndTime, type, v);
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChanged(deviceId, startTime, finalEndTime);
- fileNodeProcessor.setOverflowed(true);
-
- // write wal
- try {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- overflowProcessor.getLogNode().write(
- new UpdatePlan(startTime, finalEndTime, v, new Path(deviceId
- + "." + measurementId)));
- }
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * delete data.
- */
- public void delete(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- long lastUpdateTime = fileNodeProcessor.getLastUpdateTime(deviceId);
- // no tsfile data, the delete operation is invalid
- if (lastUpdateTime == -1) {
- LOGGER.warn("The last update time is -1, delete overflow is invalid, "
- + "the filenode processor is {}",
- fileNodeProcessor.getProcessorName());
- } else {
- // write wal
- if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- // get processors for wal
- String filenodeName = fileNodeProcessor.getProcessorName();
- OverflowProcessor overflowProcessor;
- BufferWriteProcessor bufferWriteProcessor;
- try {
- overflowProcessor = fileNodeProcessor.getOverflowProcessor(filenodeName);
- // in case that no BufferWriteProcessor is available, a new BufferWriteProcessor is
- // needed to access LogNode.
- // TODO this may make the time range of the next TsFile a little wider
- bufferWriteProcessor = fileNodeProcessor.getBufferWriteProcessor(filenodeName,
- lastUpdateTime + 1);
- } catch (IOException | FileNodeProcessorException e) {
- LOGGER.error("Getting the processor failed, the filenode is {}, delete time is {}.",
- filenodeName, timestamp);
- throw new FileNodeManagerException(e);
- }
- try {
- overflowProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- bufferWriteProcessor.getLogNode().write(new DeletePlan(timestamp,
- new Path(deviceId + "." + measurementId)));
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- }
-
- try {
- fileNodeProcessor.delete(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
-
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- private void delete(String processorName,
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator)
- throws FileNodeManagerException {
- if (!processorMap.containsKey(processorName)) {
- //TODO do we need to call processorIterator.remove() ?
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- return;
- }
- LOGGER.info("Try to delete the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- if (!processor.tryWriteLock()) {
- throw new FileNodeManagerException(String
- .format("Can't delete the filenode processor %s because Can't get the write lock.",
- processorName));
- }
-
- try {
- if (!processor.canBeClosed()) {
- LOGGER.warn("The filenode processor {} can't be deleted.", processorName);
- return;
- }
-
- try {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
- processorIterator.remove();
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} by iterator error.", processorName, e);
- throw new FileNodeManagerException(e);
- }
- } finally {
- processor.writeUnlock();
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- */
- public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- fileNodeProcessor.deleteBufferWrite(deviceId, measurementId, timestamp);
- } catch (BufferWriteProcessorException | IOException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- }
-
- /**
- * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- */
- public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- fileNodeProcessor.deleteOverflow(deviceId, measurementId, timestamp);
- } catch (IOException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- // change the type of tsfile to overflowed
- fileNodeProcessor.changeTypeToChangedForDelete(deviceId, timestamp);
- fileNodeProcessor.setOverflowed(true);
- }
-
- /**
- * begin query.
- *
- * @param deviceId queried deviceId
- * @return a query token for the device.
- */
- public int beginQuery(String deviceId) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, begin query.",
- fileNodeProcessor.getProcessorName());
- return fileNodeProcessor.addMultiPassCount();
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * query data.
- */
- public QueryDataSource query(SingleSeriesExpression seriesExpression, QueryContext context)
- throws FileNodeManagerException {
- String deviceId = seriesExpression.getSeriesPath().getDevice();
- String measurementId = seriesExpression.getSeriesPath().getMeasurement();
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, false);
- LOGGER.debug("Get the FileNodeProcessor: filenode is {}, query.",
- fileNodeProcessor.getProcessorName());
- try {
- QueryDataSource queryDataSource;
- // query operation must have overflow processor
- if (!fileNodeProcessor.hasOverflowProcessor()) {
- try {
- fileNodeProcessor.getOverflowProcessor(fileNodeProcessor.getProcessorName());
- } catch (IOException e) {
- LOGGER.error("Get the overflow processor failed, the filenode is {}, query is {},{}",
- fileNodeProcessor.getProcessorName(), deviceId, measurementId);
- throw new FileNodeManagerException(e);
- }
- }
- try {
- queryDataSource = fileNodeProcessor.query(deviceId, measurementId, context);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Query error: the deviceId {}, the measurementId {}", deviceId, measurementId,
- e);
- throw new FileNodeManagerException(e);
- }
- // return query structure
- return queryDataSource;
- } finally {
- fileNodeProcessor.readUnlock();
- }
- }
-
- /**
- * end query.
- */
- public void endQuery(String deviceId, int token) throws FileNodeManagerException {
-
- FileNodeProcessor fileNodeProcessor = getProcessor(deviceId, true);
- try {
- LOGGER.debug("Get the FileNodeProcessor: {} end query.",
- fileNodeProcessor.getProcessorName());
- fileNodeProcessor.decreaseMultiPassCount(token);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Failed to end query: the deviceId {}, token {}.", deviceId, token, e);
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- /**
- * Append one specified tsfile to the storage group. <b>This method is only provided for
- * transmission module</b>
- *
- * @param fileNodeName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
- public boolean appendFileToFileNode(String fileNodeName, TsFileResource appendFile,
- String appendFilePath) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- try {
- // check append file
- for (Map.Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- if (fileNodeProcessor.getLastUpdateTime(entry.getKey()) >= entry.getValue()) {
- return false;
- }
- }
- // close bufferwrite file
- fileNodeProcessor.closeBufferWrite();
- // append file to storage group.
- fileNodeProcessor.appendFile(appendFile, appendFilePath);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("Cannot append the file {} to {}", appendFile.getFile().getAbsolutePath(), fileNodeName, e);
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- return true;
- }
-
- /**
- * get all overlap tsfiles which are conflict with the appendFile.
- *
- * @param fileNodeName the seriesPath of storage group
- * @param appendFile the appended tsfile information
- */
- public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResource appendFile,
- String uuid) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- List<String> overlapFiles;
- try {
- overlapFiles = fileNodeProcessor.getOverlapFiles(appendFile, uuid);
- } catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- return overlapFiles;
- }
-
- /**
- * merge all overflowed filenode.
- *
- * @throws FileNodeManagerException FileNodeManagerException
- */
- public void mergeAll() throws FileNodeManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.warn("Failed to merge all overflowed filenode, because filenode manager status is {}",
- fileNodeManagerStatus);
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.MERGE;
- LOGGER.info("Start to merge all overflowed filenode");
- List<String> allFileNodeNames;
- try {
- allFileNodeNames = MManager.getInstance().getAllFileNames();
- } catch (MetadataErrorException e) {
- LOGGER.error("Get all storage group seriesPath error,", e);
- throw new FileNodeManagerException(e);
- }
- List<Future<?>> futureTasks = new ArrayList<>();
- for (String fileNodeName : allFileNodeNames) {
- FileNodeProcessor fileNodeProcessor = getProcessor(fileNodeName, true);
- try {
- Future<?> task = fileNodeProcessor.submitToMerge();
- if (task != null) {
- LOGGER.info("Submit the filenode {} to the merge pool", fileNodeName);
- futureTasks.add(task);
- }
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
- long totalTime = 0;
- // loop waiting for merge to end, the longest waiting time is
- // 60s.
- int time = 2;
- List<Exception> mergeException = new ArrayList<>();
- for (Future<?> task : futureTasks) {
- while (!task.isDone()) {
- try {
- LOGGER.info(
- "Waiting for the end of merge, already waiting for {}s, "
- + "continue to wait anothor {}s",
- totalTime, time);
- TimeUnit.SECONDS.sleep(time);
- totalTime += time;
- time = updateWaitTime(time);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
- Thread.currentThread().interrupt();
- }
- }
- try {
- task.get();
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption {}", e);
- } catch (ExecutionException e) {
- mergeException.add(e);
- LOGGER.error("The exception for merge: {}", e);
- }
- }
- if (!mergeException.isEmpty()) {
- // just throw the first exception
- throw new FileNodeManagerException(mergeException.get(0));
- }
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- LOGGER.info("End to merge all overflowed filenode");
- }
-
- private int updateWaitTime(int time) {
- return time < 32 ? time * 2 : 60;
- }
-
- /**
- * try to close the filenode processor. The name of filenode processor is processorName
- */
- private boolean closeOneProcessor(String processorName) throws FileNodeManagerException {
- if (!processorMap.containsKey(processorName)) {
- return true;
- }
-
- Processor processor = processorMap.get(processorName);
- if (processor.tryWriteLock()) {
- try {
- if (processor.canBeClosed()) {
- processor.close();
- return true;
- } else {
- return false;
- }
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
- } finally {
- processor.writeUnlock();
- }
- } else {
- return false;
- }
- }
-
- /**
- * delete one filenode.
- */
- public void deleteOneFileNode(String processorName) throws FileNodeManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- if (processorMap.containsKey(processorName)) {
- deleteFileNodeBlocked(processorName);
- }
- String fileNodePath = TsFileDBConf.getFileNodeDir();
- fileNodePath = standardizeDir(fileNodePath) + processorName;
- FileUtils.deleteDirectory(new File(fileNodePath));
-
- cleanBufferWrite(processorName);
- cleanOverflow(processorName);
-
- MultiFileLogNodeManager.getInstance()
- .deleteNode(processorName + IoTDBConstant.BUFFERWRITE_LOG_NODE_SUFFIX);
- MultiFileLogNodeManager.getInstance()
- .deleteNode(processorName + IoTDBConstant.OVERFLOW_LOG_NODE_SUFFIX);
- } catch (IOException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- private void cleanOverflow(String processorName) throws IOException {
- String overflowDirPath = TsFileDBConf.getOverflowDataDir();
- String overflowPath = standardizeDir(overflowDirPath) + processorName;
- File overflowDir = new File(overflowPath);
- // free and close the streams under this overflow directory
- if (!overflowDir.exists()) {
- return;
- }
- File[] overflowSubDirs = overflowDir.listFiles();
- if (overflowSubDirs == null) {
- return;
- }
- for (File overflowSubDir : overflowSubDirs) {
- closeAndRemoveReader(overflowSubDir);
- }
- FileUtils.deleteDirectory(overflowDir);
- }
-
- private void cleanBufferWrite(String processorName) throws IOException {
- List<String> bufferwritePathList = directories.getAllTsFileFolders();
- for (String bufferwritePath : bufferwritePathList) {
- bufferwritePath = standardizeDir(bufferwritePath) + processorName;
- File bufferDir = new File(bufferwritePath);
- // free and close the streams under this bufferwrite directory
- if (!bufferDir.exists()) {
- continue;
- }
- closeAndRemoveReader(bufferDir);
- FileUtils.deleteDirectory(new File(bufferwritePath));
- }
- }
-
- private void closeAndRemoveReader(File folder) throws IOException {
- File[] files = folder.listFiles();
- if (files != null) {
- for (File file : files) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(file.getPath());
- }
- }
- }
-
- private void deleteFileNodeBlocked(String processorName) throws FileNodeManagerException {
- LOGGER.info("Forced to delete the filenode processor {}", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- while (true) {
- if (processor.tryWriteLock()) {
- try {
- if (processor.canBeClosed()) {
- LOGGER.info("Delete the filenode processor {}.", processorName);
- processor.delete();
- processorMap.remove(processorName);
- break;
- } else {
- LOGGER.info(
- "Can't delete the filenode processor {}, "
- + "because the filenode processor can't be closed."
- + " Wait 100ms to retry");
- }
- } catch (ProcessorException e) {
- LOGGER.error("Delete the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
- } finally {
- processor.writeUnlock();
- }
- } else {
- LOGGER.info(
- "Can't delete the filenode processor {}, because it can't get the write lock."
- + " Wait 100ms to retry", processorName);
- }
- try {
- TimeUnit.MILLISECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error(e.getMessage());
- Thread.currentThread().interrupt();
- }
- }
- }
-
- private String standardizeDir(String originalPath) {
- String res = originalPath;
- if ((originalPath.length() > 0
- && originalPath.charAt(originalPath.length() - 1) != File.separatorChar)
- || originalPath.length() == 0) {
- res = originalPath + File.separatorChar;
- }
- return res;
- }
-
- /**
- * add time series.
- */
- public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
- CompressionType compressor,
- Map<String, String> props) throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(path.getFullPath(), true);
- try {
- fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
-
- /**
- * Force to close the filenode processor.
- */
- public void closeOneFileNode(String processorName) throws FileNodeManagerException {
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- return;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- LOGGER.info("Force to close the filenode processor {}.", processorName);
- while (!closeOneProcessor(processorName)) {
- try {
- LOGGER.info("Can't force to close the filenode processor {}, wait 100ms to retry",
- processorName);
- TimeUnit.MILLISECONDS.sleep(100);
- } catch (InterruptedException e) {
- // ignore the interrupted exception
- LOGGER.error("Unexpected interruption {}", e);
- Thread.currentThread().interrupt();
- }
- }
- } finally {
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * try to close the filenode processor.
- */
- private void close(String processorName) throws FileNodeManagerException {
- if (!processorMap.containsKey(processorName)) {
- LOGGER.warn("The processorMap doesn't contain the filenode processor {}.", processorName);
- return;
- }
- LOGGER.info("Try to close the filenode processor {}.", processorName);
- FileNodeProcessor processor = processorMap.get(processorName);
- if (!processor.tryWriteLock()) {
- LOGGER.warn("Can't get the write lock of the filenode processor {}.", processorName);
- return;
- }
- try {
- if (processor.canBeClosed()) {
- try {
- LOGGER.info("Close the filenode processor {}.", processorName);
- processor.close();
- } catch (ProcessorException e) {
- LOGGER.error("Close the filenode processor {} error.", processorName, e);
- throw new FileNodeManagerException(e);
- }
- } else {
- LOGGER.warn("The filenode processor {} can't be closed.", processorName);
- }
- } finally {
- processor.writeUnlock();
- }
- }
-
- /**
- * delete all filenode.
- */
- public synchronized boolean deleteAll() throws FileNodeManagerException {
- LOGGER.info("Start deleting all filenode");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to delete all filenode processor because of merge operation");
- return false;
- }
-
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- Iterator<Map.Entry<String, FileNodeProcessor>> processorIterator = processorMap.entrySet()
- .iterator();
- while (processorIterator.hasNext()) {
- Map.Entry<String, FileNodeProcessor> processorEntry = processorIterator.next();
- delete(processorEntry.getKey(), processorIterator);
- }
- return processorMap.isEmpty();
- } finally {
- LOGGER.info("Deleting all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * Try to close All.
- */
- public void closeAll() throws FileNodeManagerException {
- LOGGER.info("Start closing all filenode processor");
- if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
- LOGGER.info("Failed to close all filenode processor because of merge operation");
- return;
- }
- fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
- try {
- for (Map.Entry<String, FileNodeProcessor> processorEntry : processorMap.entrySet()) {
- close(processorEntry.getKey());
- }
- } finally {
- LOGGER.info("Close all FileNodeProcessors ends");
- fileNodeManagerStatus = FileNodeManagerStatus.NONE;
- }
- }
-
- /**
- * force flush to control memory usage.
- */
- public void forceFlush(BasicMemController.UsageLevel level) {
- // you may add some delicate process like below
- // or you could provide multiple methods for different urgency
- switch (level) {
- // only select the most urgent (most active or biggest in size)
- // processors to flush
- // only select top 10% active memory user to flush
- case WARNING:
- try {
- flushTop(0.1f);
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
- }
- break;
- // force all processors to flush
- case DANGEROUS:
- try {
- flushAll();
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: {}", e);
- }
- break;
- // if the flush thread pool is not full ( or half full), start a new
- // flush task
- case SAFE:
- if (FlushManager.getInstance().getActiveCnt() < 0.5 * FlushManager.getInstance()
- .getThreadCnt()) {
- try {
- flushTop(0.01f);
- } catch (IOException e) {
- LOGGER.error("force flush memory data error: ", e);
- }
- }
- break;
- default:
- }
- }
-
- private void flushAll() throws IOException {
- for (FileNodeProcessor processor : processorMap.values()) {
- if (!processor.tryLock(true)) {
- continue;
- }
- try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
- }
- } finally {
- processor.unlock(true);
- }
- }
- }
-
- private void flushTop(float percentage) throws IOException {
- List<FileNodeProcessor> tempProcessors = new ArrayList<>(processorMap.values());
- // sort the tempProcessors as descending order
- tempProcessors.sort((o1, o2) -> (int) (o2.memoryUsage() - o1.memoryUsage()));
- int flushNum =
- (int) (tempProcessors.size() * percentage) > 1
- ? (int) (tempProcessors.size() * percentage)
- : 1;
- for (int i = 0; i < flushNum && i < tempProcessors.size(); i++) {
- FileNodeProcessor processor = tempProcessors.get(i);
- // 64M
- if (processor.memoryUsage() <= TSFileConfig.groupSizeInByte / 2) {
- continue;
- }
- processor.writeLock();
- try {
- boolean isMerge = processor.flush().isHasOverflowFlushTask();
- if (isMerge) {
- processor.submitToMerge();
- }
- } finally {
- processor.writeUnlock();
- }
- }
- }
-
- @Override
- public void start() {
- // do no thing
- }
-
- @Override
- public void stop() {
- try {
- closeAll();
- } catch (FileNodeManagerException e) {
- LOGGER.error("Failed to close file node manager because .", e);
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.FILE_NODE_SERVICE;
- }
-
- /**
- * get restore file path.
- */
- public String getRestoreFilePath(String processorName) {
- FileNodeProcessor fileNodeProcessor = processorMap.get(processorName);
- if (fileNodeProcessor != null) {
- return fileNodeProcessor.getFileNodeRestoreFilePath();
- } else {
- return null;
- }
- }
-
- /**
- * recover filenode.
- */
- public void recoverFileNode(String filenodeName)
- throws FileNodeManagerException {
- FileNodeProcessor fileNodeProcessor = getProcessor(filenodeName, true);
- LOGGER.info("Recover the filenode processor, the filenode is {}, the status is {}",
- filenodeName, fileNodeProcessor.getFileNodeProcessorStatus());
- try {
- fileNodeProcessor.fileNodeRecovery();
- } catch (FileNodeProcessorException e) {
- throw new FileNodeManagerException(e);
- } finally {
- fileNodeProcessor.writeUnlock();
- }
- }
-
- private enum FileNodeManagerStatus {
- NONE, MERGE, CLOSE
- }
-
- private static class FileNodeManagerHolder {
-
- private FileNodeManagerHolder() {
- }
-
- private static final FileNodeManager INSTANCE = new FileNodeManager(
- TsFileDBConf.getFileNodeDir());
- }
-
-}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
index 2d1144e,786d4f3..0000000
deleted file mode 100644,100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenode/FileNodeProcessor.java
+++ /dev/null
@@@ -1,2130 -1,2066 +1,0 @@@
- ///**
- // * Licensed to the Apache Software Foundation (ASF) under one
- // * or more contributor license agreements. See the NOTICE file
- // * distributed with this work for additional information
- // * regarding copyright ownership. The ASF licenses this file
- // * to you under the Apache License, Version 2.0 (the
- // * "License"); you may not use this file except in compliance
- // * with the License. You may obtain a copy of the License at
- // *
- // * http://www.apache.org/licenses/LICENSE-2.0
- // *
- // * Unless required by applicable law or agreed to in writing,
- // * software distributed under the License is distributed on an
- // * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // * KIND, either express or implied. See the License for the
- // * specific language governing permissions and limitations
- // * under the License.
- // */
- //package org.apache.iotdb.db.engine.filenode;
- //
- //import static java.time.ZonedDateTime.ofInstant;
- //
- //import java.io.File;
- //import java.io.FileInputStream;
- //import java.io.FileOutputStream;
- //import java.io.IOException;
- //import java.nio.file.FileSystems;
- //import java.nio.file.Files;
- //import java.time.Instant;
- //import java.time.ZoneId;
- //import java.time.ZonedDateTime;
- //import java.util.ArrayList;
- //import java.util.HashMap;
- //import java.util.HashSet;
- //import java.util.Iterator;
- //import java.util.List;
- //import java.util.Map;
- //import java.util.Map.Entry;
- //import java.util.Objects;
- //import java.util.Set;
- //import java.util.concurrent.CountDownLatch;
- //import java.util.concurrent.ExecutionException;
- //import java.util.concurrent.Future;
- //import java.util.concurrent.TimeUnit;
- //import java.util.concurrent.TimeoutException;
- //import java.util.concurrent.atomic.AtomicInteger;
- //import java.util.concurrent.atomic.AtomicLong;
- //import java.util.concurrent.locks.ReentrantLock;
- //import java.util.function.Consumer;
- //import org.apache.iotdb.db.conf.IoTDBConfig;
- //import org.apache.iotdb.db.conf.IoTDBConstant;
- //import org.apache.iotdb.db.conf.IoTDBDescriptor;
- //import org.apache.iotdb.db.conf.directories.DirectoryManager;
- //import org.apache.iotdb.db.engine.Processor;
- //import org.apache.iotdb.db.engine.bufferwrite.Action;
- //import org.apache.iotdb.db.engine.bufferwrite.ActionException;
- //import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
- //import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
- //import org.apache.iotdb.db.engine.modification.Deletion;
- //import org.apache.iotdb.db.engine.modification.Modification;
- //import org.apache.iotdb.db.engine.modification.ModificationFile;
- //import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
- //import org.apache.iotdb.db.engine.pool.MergePoolManager;
- //import org.apache.iotdb.db.engine.querycontext.GlobalSortedSeriesDataSource;
- //import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
- //import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
- //import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
- //import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
- //import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
- //import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
- //import org.apache.iotdb.db.engine.version.VersionController;
- //import org.apache.iotdb.db.exception.BufferWriteProcessorException;
- //import org.apache.iotdb.db.exception.ErrorDebugException;
- //import org.apache.iotdb.db.exception.FileNodeProcessorException;
- //import org.apache.iotdb.db.exception.OverflowProcessorException;
- //import org.apache.iotdb.db.exception.PathErrorException;
- //import org.apache.iotdb.db.exception.ProcessorException;
- //import org.apache.iotdb.db.metadata.MManager;
- //import org.apache.iotdb.db.monitor.IStatistic;
- //import org.apache.iotdb.db.monitor.MonitorConstants;
- //import org.apache.iotdb.db.monitor.StatMonitor;
- //import org.apache.iotdb.db.query.context.QueryContext;
- //import org.apache.iotdb.db.query.control.FileReaderManager;
- //import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
- //import org.apache.iotdb.db.query.reader.IReader;
- //import org.apache.iotdb.db.sync.conf.Constans;
- //import org.apache.iotdb.db.utils.ImmediateFuture;
- //import org.apache.iotdb.db.utils.MemUtils;
- //import org.apache.iotdb.db.utils.QueryUtils;
- //import org.apache.iotdb.db.utils.TimeValuePair;
- //import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
- //import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
- //import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
- //import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
- //import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
- //import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- //import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
- //import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
- //import org.apache.iotdb.tsfile.read.common.Path;
- //import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
- //import org.apache.iotdb.tsfile.read.filter.TimeFilter;
- //import org.apache.iotdb.tsfile.read.filter.basic.Filter;
- //import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
- //import org.apache.iotdb.tsfile.utils.Pair;
- //import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
- //import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
- //import org.apache.iotdb.tsfile.write.record.TSRecord;
- //import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
- //import org.apache.iotdb.tsfile.write.schema.FileSchema;
- //import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
- //import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
- //import org.slf4j.Logger;
- //import org.slf4j.LoggerFactory;
- //
- //public class FileNodeProcessor extends Processor implements IStatistic {
- //
- // private static final String WARN_NO_SUCH_OVERFLOWED_FILE = "Can not find any tsfile which"
- // + " will be overflowed in the filenode processor {}, ";
- // public static final String RESTORE_FILE_SUFFIX = ".restore";
- // private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeProcessor.class);
- // private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- // private static final MManager mManager = MManager.getInstance();
- // private static final DirectoryManager DIRECTORY_MANAGER = DirectoryManager.getInstance();
- // private final String statStorageDeltaName;
- // private final HashMap<String, AtomicLong> statParamsHashMap = new HashMap<>();
- // /**
- // * Used to keep the oldest timestamp for each deviceId. The key is deviceId.
- // */
- // private volatile boolean isOverflowed;
- // private Map<String, Long> lastUpdateTimeMap;
- // private Map<String, Long> flushLastUpdateTimeMap;
- // private Map<String, List<TsFileResource>> invertedIndexOfFiles;
- // private TsFileResource emptyTsFileResource;
- //// private TsFileResourceV2 currentTsFileResource;
- // private List<TsFileResource> newFileNodes;
- // private FileNodeProcessorStatus isMerging;
- //
- // /**
- // * this is used when work->merge operation
- // */
- // private int numOfMergeFile;
- // private FileNodeProcessorStore fileNodeProcessorStore;
- // private String fileNodeRestoreFilePath;
- // private final Object fileNodeRestoreLock = new Object();
- //
- // /**
- // * last merge time
- // */
- // private long lastMergeTime = -1;
- // private BufferWriteProcessor bufferWriteProcessor = null;
- //
- // //the bufferwrite Processors that are closing. (Because they are not closed well,
- // // their memtable are not released and we have to query data from them.
- // //private ConcurrentSkipListSet<BufferWriteProcessor> closingBufferWriteProcessor = new ConcurrentSkipListSet<>();
- // private CopyOnReadLinkedList<BufferWriteProcessor> closingBufferWriteProcessor = new CopyOnReadLinkedList<>();
- //
- // private OverflowProcessor overflowProcessor = null;
- // private Set<Integer> oldMultiPassTokenSet = null;
- // private Set<Integer> newMultiPassTokenSet = new HashSet<>();
- //
- // /**
- // * Represent the number of old queries that have not ended.
- // * This parameter only decreases but not increase.
- // */
- // private CountDownLatch oldMultiPassCount = null;
- //
- // /**
- // * Represent the number of new queries that have not ended.
- // */
- // private AtomicInteger newMultiPassCount = new AtomicInteger(0);
- //
- // /**
- // * statistic monitor parameters
- // */
- // private Map<String, Action> parameters;
- // private FileSchema fileSchema;
- //
- // private Action fileNodeFlushAction = () -> {
- // synchronized (fileNodeProcessorStore) {
- // try {
- // writeStoreToDisk(fileNodeProcessorStore);
- // } catch (FileNodeProcessorException e) {
- // throw new ActionException(e);
- // }
- // }
- // };
- //
- // private Action bufferwriteFlushAction = () -> {
- // // update the lastUpdateTime Notice: Thread safe
- // synchronized (fileNodeProcessorStore) {
- // // deep copy
- // Map<String, Long> tempLastUpdateMap = new HashMap<>(lastUpdateTimeMap);
- // // update flushLastUpdateTimeMap
- // for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- // flushLastUpdateTimeMap.put(entry.getKey(), entry.getValue() + 1);
- // }
- // fileNodeProcessorStore.setLastUpdateTimeMap(tempLastUpdateMap);
- // }
- // };
- //
- //// private Action bufferwriteCloseAction = new Action() {
- ////
- //// @Override
- //// public void act() {
- //// synchronized (fileNodeProcessorStore) {
- //// fileNodeProcessorStore.setLatestFlushTimeForEachDevice(lastUpdateTimeMap);
- //// addLastTimeToIntervalFile();
- //// fileNodeProcessorStore.setSequenceFileList(newFileNodes);
- //// }
- //// }
- ////
- //// private void addLastTimeToIntervalFile() {
- ////
- //// if (!newFileNodes.isEmpty()) {
- //// // end time with one start time
- //// Map<String, Long> endTimeMap = new HashMap<>();
- //// for (Entry<String, Long> startTime : currentTsFileResource.getStartTimeMap().entrySet()) {
- //// String deviceId = startTime.getKey();
- //// endTimeMap.put(deviceId, lastUpdateTimeMap.get(deviceId));
- //// }
- //// currentTsFileResource.setEndTimeMap(endTimeMap);
- //// }
- //// }
- //// };
- //
- // private Consumer<BufferWriteProcessor> bufferwriteCloseConsumer = (bwProcessor) -> {
- // synchronized (fileNodeProcessorStore) {
- // fileNodeProcessorStore.setLastUpdateTimeMap(lastUpdateTimeMap);
- //
- // if (!newFileNodes.isEmpty()) {
- // // end time with one start time
- // Map<String, Long> endTimeMap = new HashMap<>();
- // TsFileResource resource = bwProcessor.getCurrentTsFileResource();
- // for (Entry<String, Long> startTime : resource.getStartTimeMap().entrySet()) {
- // String deviceId = startTime.getKey();
- // endTimeMap.put(deviceId, lastUpdateTimeMap.get(deviceId));
- // }
- // resource.setEndTimeMap(endTimeMap);
- // }
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // }
- // };
- //
- //
- // private Action overflowFlushAction = () -> {
- //
- // // update the new TsFileResourceV2 List and emptyIntervalFile.
- // // Notice: thread safe
- // synchronized (fileNodeProcessorStore) {
- // fileNodeProcessorStore.setOverflowed(isOverflowed);
- // fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // }
- // };
- // // Token for query which used to
- // private int multiPassLockToken = 0;
- // private VersionController versionController;
- // private ReentrantLock mergeDeleteLock = new ReentrantLock();
- //
- // /**
- // * This is the modification file of the result of the current merge.
- // */
- // private ModificationFile mergingModification;
- //
- // private TsFileIOWriter mergeFileWriter = null;
- // private String mergeOutputPath = null;
- // private String mergeBaseDir = null;
- // private String mergeFileName = null;
- // private boolean mergeIsChunkGroupHasData = false;
- // private long mergeStartPos;
- //
- // /**
- // * constructor of FileNodeProcessor.
- // */
- // FileNodeProcessor(String fileNodeDirPath, String processorName)
- // throws FileNodeProcessorException {
- // super(processorName);
- // for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- // MonitorConstants.FileNodeProcessorStatConstants.values()) {
- // statParamsHashMap.put(statConstant.name(), new AtomicLong(0));
- // }
- // statStorageDeltaName =
- // MonitorConstants.STAT_STORAGE_GROUP_PREFIX + MonitorConstants.MONITOR_PATH_SEPARATOR
- // + MonitorConstants.FILE_NODE_PATH + MonitorConstants.MONITOR_PATH_SEPARATOR
- // + processorName.replaceAll("\\.", "_");
- //
- // this.parameters = new HashMap<>();
- // String dirPath = fileNodeDirPath;
- // if (dirPath.size() > 0
- // && dirPath.charAt(dirPath.size() - 1) != File.separatorChar) {
- // dirPath = dirPath + File.separatorChar;
- // }
- //
- // File restoreFolder = new File(dirPath + processorName);
- // if (!restoreFolder.exists()) {
- // restoreFolder.mkdirs();
- // LOGGER.info(
- // "The restore directory of the filenode processor {} doesn't exist. Create new " +
- // "directory {}",
- // getProcessorName(), restoreFolder.getAbsolutePath());
- // }
- // fileNodeRestoreFilePath = new File(restoreFolder, processorName + RESTORE_FILE_SUFFIX)
- // .getPathString();
- // try {
- // fileNodeProcessorStore = readStoreFromDisk();
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error(
- // "The fileNode processor {} encountered an error when recoverying restore " +
- // "information.", processorName);
- // throw new FileNodeProcessorException(e);
- // }
- // // TODO deep clone the lastupdate time
- // emptyTsFileResource = fileNodeProcessorStore.getEmptyTsFileResource();
- // newFileNodes = fileNodeProcessorStore.getNewFileNodes();
- // isMerging = fileNodeProcessorStore.getFileNodeProcessorStatus();
- // numOfMergeFile = fileNodeProcessorStore.getNumOfMergeFile();
- // invertedIndexOfFiles = new HashMap<>();
- //
- // // construct the fileschema
- // try {
- // this.fileSchema = constructFileSchema(processorName);
- // } catch (WriteProcessException e) {
- // throw new FileNodeProcessorException(e);
- // }
- //
- // recover();
- //
- // // RegistStatService
- // if (TsFileDBConf.isEnableStatMonitor()) {
- // StatMonitor statMonitor = StatMonitor.getInstance();
- // registerStatMetadata();
- // statMonitor.registerStatistics(statStorageDeltaName, this);
- // }
- // try {
- // versionController = new SimpleFileVersionController(restoreFolder.getPathString());
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- //
- // @Override
- // public Map<String, AtomicLong> getStatParamsHashMap() {
- // return statParamsHashMap;
- // }
- //
- // @Override
- // public void registerStatMetadata() {
- // Map<String, String> hashMap = new HashMap<>();
- // for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- // MonitorConstants.FileNodeProcessorStatConstants.values()) {
- // hashMap
- // .put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
- // MonitorConstants.DATA_TYPE_INT64);
- // }
- // StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- // }
- //
- // @Override
- // public List<String> getAllPathForStatistic() {
- // List<String> list = new ArrayList<>();
- // for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- // MonitorConstants.FileNodeProcessorStatConstants.values()) {
- // list.add(
- // statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
- // }
- // return list;
- // }
- //
- // @Override
- // public Map<String, TSRecord> getAllStatisticsValue() {
- // Long curTime = System.currentTimeMillis();
- // HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
- // TSRecord tsRecord = new TSRecord(curTime, statStorageDeltaName);
- //
- // Map<String, AtomicLong> hashMap = getStatParamsHashMap();
- // tsRecord.dataPointList = new ArrayList<>();
- // for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
- // tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(), entry.getValue().get()));
- // }
- //
- // tsRecordHashMap.put(statStorageDeltaName, tsRecord);
- // return tsRecordHashMap;
- // }
- //
- // /**
- // * add interval FileNode.
- // */
- // void addIntervalFileNode(TsFileResource tsFileResource) throws ActionException {
- // newFileNodes.add(tsFileResource);
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // fileNodeFlushAction.act();
- // }
- //
- // /**
- // * set interval filenode start time.
- // *
- // * @param deviceId device ID
- // */
- // void setIntervalFileNodeStartTime(String deviceId) {
- // if (getBufferWriteProcessor().getCurrentTsFileResource().getStartTime(deviceId) == -1) {
- // getBufferWriteProcessor().getCurrentTsFileResource().setStartTime(deviceId,
- // flushLastUpdateTimeMap.get(deviceId));
- // if (!invertedIndexOfFiles.containsKey(deviceId)) {
- // invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- // }
- // invertedIndexOfFiles.get(deviceId).add(getBufferWriteProcessor().getCurrentTsFileResource());
- // }
- // }
- //
- // void setIntervalFileNodeStartTime(String deviceId, long time) {
- // if (time != -1) {
- // getBufferWriteProcessor().getCurrentTsFileResource().setStartTime(deviceId, time);
- // } else {
- // getBufferWriteProcessor().getCurrentTsFileResource().removeTime(deviceId);
- // invertedIndexOfFiles.get(deviceId).remove(getBufferWriteProcessor().getCurrentTsFileResource());
- // }
- // }
- //
- // long getIntervalFileNodeStartTime(String deviceId) {
- // return getBufferWriteProcessor().getCurrentTsFileResource().getStartTime(deviceId);
- // }
- //
- // private void addAllFileIntoIndex(List<TsFileResource> fileList) {
- // // clear map
- // invertedIndexOfFiles.clear();
- // // add all file to index
- // for (TsFileResource fileNode : fileList) {
- // if (fileNode.getStartTimeMap().isEmpty()) {
- // continue;
- // }
- // for (String deviceId : fileNode.getStartTimeMap().keySet()) {
- // if (!invertedIndexOfFiles.containsKey(deviceId)) {
- // invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- // }
- // invertedIndexOfFiles.get(deviceId).add(fileNode);
- // }
- // }
- // }
- //
- // public boolean isOverflowed() {
- // return isOverflowed;
- // }
- //
- // /**
- // * if overflow insert, update and delete insert into this filenode processor, set
- // * <code>isOverflowed</code> to true.
- // */
- // public void setOverflowed(boolean isOverflowed) {
- // if (this.isOverflowed != isOverflowed) {
- // this.isOverflowed = isOverflowed;
- // }
- // }
- //
- // public FileNodeProcessorStatus getFileNodeProcessorStatus() {
- // return isMerging;
- // }
- //
- // /**
- // * execute filenode recovery.
- // */
- // public void recover() throws FileNodeProcessorException {
- // // restore sequential files
- // parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- // //parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- // parameters
- // .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fileNodeFlushAction);
- // parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
- // parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fileNodeFlushAction);
- //
- // for (int i = 0; i < newFileNodes.size(); i++) {
- // TsFileResource tsFile = newFileNodes.get(i);
- //// try {
- //// String filePath = tsFile.getFilePath();
- //// String logNodePrefix = BufferWriteProcessor.logNodePrefix(processorName);
- //// TsFileRecoverPerformer recoverPerformer =
- //// new TsFileRecoverPerformer(logNodePrefix,
- //// fileSchema, versionController, tsFile);
- //// recoverPerformer.recover();
- //// } catch (ProcessorException e) {
- //// LOGGER.error(
- //// "The filenode processor {} failed to recover the bufferwrite processor, "
- //// + "the last bufferwrite file is {}.",
- //// getProcessorName(), tsFile.getFile().getName());
- //// throw new FileNodeProcessorException(e);
- //// }
- // }
- // recoverUpdateTimeMap();
- //
- // // restore the overflow processor
- // LOGGER.info("The filenode processor {} will recover the overflow processor.",
- // getProcessorName());
- //
- // try {
- // overflowProcessor = new OverflowProcessor(getProcessorName(), parameters, fileSchema,
- // versionController);
- // } catch (ProcessorException e) {
- // LOGGER.error("The filenode processor {} failed to recovery the overflow processor.",
- // getProcessorName());
- // throw new FileNodeProcessorException(e);
- // }
- //
- // if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- // // re-merge all file
- // // if bufferwrite processor is not null, and close
- // LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- // getProcessorName(), isMerging);
- // merge();
- // } else if (isMerging == FileNodeProcessorStatus.WAITING) {
- // LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- // getProcessorName(), isMerging);
- // switchWaitingToWorking();
- // }
- // // add file into index of file
- // addAllFileIntoIndex(newFileNodes);
- // }
- //
- // private void recoverUpdateTimeMap() {
- // lastUpdateTimeMap = new HashMap<>();
- // flushLastUpdateTimeMap = new HashMap<>();
- // for (TsFileResource tsFileResource : newFileNodes) {
- // Map<String, Long> endTimeMap = tsFileResource.getEndTimeMap();
- // endTimeMap.forEach((key, value) -> {
- // Long lastTime = lastUpdateTimeMap.get(key);
- // if (lastTime == null || lastTime < value) {
- // lastUpdateTimeMap.put(key, value);
- // flushLastUpdateTimeMap.put(key, value);
- // }
- // });
- // }
- // }
- //
- // //when calling this method, the bufferWriteProcessor must not be null
- // private BufferWriteProcessor getBufferWriteProcessor() {
- // return bufferWriteProcessor;
- // }
- //
- // /**
- // * get buffer insert processor by processor name and insert time.
- // */
- // public BufferWriteProcessor getBufferWriteProcessor(String processorName, long insertTime)
- // throws FileNodeProcessorException {
- // if (bufferWriteProcessor == null) {
- // Map<String, Action> params = new HashMap<>();
- // params.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- // //params.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- // params
- // .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fileNodeFlushAction);
- // String baseDir = DIRECTORY_MANAGER.getNextFolderForSequenceFile();
- // LOGGER.info("Allocate folder {} for the new bufferwrite processor.", baseDir);
- // // construct processor or restore
- // try {
- // bufferWriteProcessor = new BufferWriteProcessor(baseDir, processorName,
- // insertTime + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
- // + System.currentTimeMillis(),
- // params, bufferwriteCloseConsumer, versionController, fileSchema);
- // } catch (BufferWriteProcessorException e) {
- // throw new FileNodeProcessorException(String
- // .format("The filenode processor %s failed to get the bufferwrite processor.",
- // processorName), e);
- // }
- // }
- // return bufferWriteProcessor;
- // }
- //
- // /**
- // * get overflow processor by processor name.
- // */
- // public OverflowProcessor getOverflowProcessor(String processorName) throws ProcessorException {
- // if (overflowProcessor == null) {
- // Map<String, Action> params = new HashMap<>();
- // // construct processor or restore
- // params.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
- // params
- // .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, fileNodeFlushAction);
- // overflowProcessor = new OverflowProcessor(processorName, params, fileSchema,
- // versionController);
- // } else if (overflowProcessor.isClosed()) {
- // overflowProcessor.reopen();
- // }
- // return overflowProcessor;
- // }
- //
- // /**
- // * get overflow processor.
- // */
- // public OverflowProcessor getOverflowProcessor() {
- // if (overflowProcessor == null || overflowProcessor.isClosed()) {
- // LOGGER.error("The overflow processor is null when getting the overflowProcessor");
- // }
- // return overflowProcessor;
- // }
- //
- // public boolean hasOverflowProcessor() {
- // return overflowProcessor != null && !overflowProcessor.isClosed();
- // }
- //
- // public void setBufferwriteProcessroToClosed() {
- //
- // bufferWriteProcessor = null;
- // }
- //
- // public boolean hasBufferwriteProcessor() {
- //
- // return bufferWriteProcessor != null;
- // }
- //
- // /**
- // * set last update time.
- // */
- // public void setLastUpdateTime(String deviceId, long timestamp) {
- // if (!lastUpdateTimeMap.containsKey(deviceId) || lastUpdateTimeMap.get(deviceId) < timestamp) {
- // lastUpdateTimeMap.put(deviceId, timestamp);
- // }
- // if (timestamp == -1) {
- // lastUpdateTimeMap.remove(deviceId);
- // }
- // }
- //
- // /**
- // * get last update time.
- // */
- // public long getLastUpdateTime(String deviceId) {
- //
- // if (lastUpdateTimeMap.containsKey(deviceId)) {
- // return lastUpdateTimeMap.get(deviceId);
- // } else {
- // return -1;
- // }
- // }
- //
- // /**
- // * get flush last update time.
- // */
- // public long getFlushLastUpdateTime(String deviceId) {
- // if (!flushLastUpdateTimeMap.containsKey(deviceId)) {
- // flushLastUpdateTimeMap.put(deviceId, 0L);
- // }
- // return flushLastUpdateTimeMap.get(deviceId);
- // }
- //
- // public Map<String, Long> getLastUpdateTimeMap() {
- // return lastUpdateTimeMap;
- // }
- //
- // /**
- // * For insert overflow.
- // */
- // public void changeTypeToChanged(String deviceId, long timestamp) {
- // if (!invertedIndexOfFiles.containsKey(deviceId)) {
- // LOGGER.warn(
- // WARN_NO_SUCH_OVERFLOWED_FILE
- // + "the data is [device:{},time:{}]",
- // getProcessorName(), deviceId, timestamp);
- // emptyTsFileResource.setStartTime(deviceId, 0L);
- // emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- // emptyTsFileResource.changeTypeToChanged(isMerging);
- // } else {
- // List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- // int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- // changeTypeToChanged(temp.get(index), deviceId);
- // }
- // }
- //
- // private void changeTypeToChanged(TsFileResource fileNode, String deviceId) {
- // fileNode.changeTypeToChanged(isMerging);
- // if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- // fileNode.addMergeChanged(deviceId);
- // }
- // }
- //
- // /**
- // * For update overflow.
- // */
- // public void changeTypeToChanged(String deviceId, long startTime, long endTime) {
- // if (!invertedIndexOfFiles.containsKey(deviceId)) {
- // LOGGER.warn(
- // WARN_NO_SUCH_OVERFLOWED_FILE
- // + "the data is [device:{}, start time:{}, end time:{}]",
- // getProcessorName(), deviceId, startTime, endTime);
- // emptyTsFileResource.setStartTime(deviceId, 0L);
- // emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- // emptyTsFileResource.changeTypeToChanged(isMerging);
- // } else {
- // List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- // int left = searchIndexNodeByTimestamp(deviceId, startTime, temp);
- // int right = searchIndexNodeByTimestamp(deviceId, endTime, temp);
- // for (int i = left; i <= right; i++) {
- // changeTypeToChanged(temp.get(i), deviceId);
- // }
- // }
- // }
- //
- // /**
- // * For delete overflow.
- // */
- // public void changeTypeToChangedForDelete(String deviceId, long timestamp) {
- // if (!invertedIndexOfFiles.containsKey(deviceId)) {
- // LOGGER.warn(
- // WARN_NO_SUCH_OVERFLOWED_FILE
- // + "the data is [device:{}, delete time:{}]",
- // getProcessorName(), deviceId, timestamp);
- // emptyTsFileResource.setStartTime(deviceId, 0L);
- // emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- // emptyTsFileResource.changeTypeToChanged(isMerging);
- // } else {
- // List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- // int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- // for (int i = 0; i <= index; i++) {
- // temp.get(i).changeTypeToChanged(isMerging);
- // if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- // temp.get(i).addMergeChanged(deviceId);
- // }
- // }
- // }
- // }
- //
- // /**
- // * Search the index of the interval by the timestamp.
- // *
- // * @return index of interval
- // */
- // private int searchIndexNodeByTimestamp(String deviceId, long timestamp,
- // List<TsFileResource> fileList) {
- // int index = 1;
- // while (index < fileList.size()) {
- // if (timestamp < fileList.get(index).getStartTime(deviceId)) {
- // break;
- // } else {
- // index++;
- // }
- // }
- // return index - 1;
- // }
- //
- // /**
- // * add multiple pass lock.
- // */
- // public int addMultiPassCount() {
- // LOGGER.debug("Add MultiPassCount: read lock newMultiPassCount.");
- // newMultiPassCount.incrementAndGet();
- // while (newMultiPassTokenSet.contains(multiPassLockToken)) {
- // multiPassLockToken++;
- // }
- // newMultiPassTokenSet.add(multiPassLockToken);
- // LOGGER.debug("Add multi token:{}, nsPath:{}.", multiPassLockToken, getProcessorName());
- // return multiPassLockToken;
- // }
- //
- // /**
- // * decrease multiple pass count. TODO: use the return value or remove it.
- // */
- // public boolean decreaseMultiPassCount(int token) throws FileNodeProcessorException {
- // if (newMultiPassTokenSet.contains(token)) {
- // int newMultiPassCountValue = newMultiPassCount.decrementAndGet();
- // if (newMultiPassCountValue < 0) {
- // throw new FileNodeProcessorException(String
- // .format("Remove MultiPassCount error, newMultiPassCount:%d", newMultiPassCountValue));
- // }
- // newMultiPassTokenSet.remove(token);
- // LOGGER.debug("Remove multi token:{}, nspath:{}, new set:{}, count:{}", token,
- // getProcessorName(),
- // newMultiPassTokenSet, newMultiPassCount);
- // return true;
- // } else if (oldMultiPassTokenSet != null && oldMultiPassTokenSet.contains(token)) {
- // // remove token first, then unlock
- // oldMultiPassTokenSet.remove(token);
- // oldMultiPassCount.countDown();
- // long oldMultiPassCountValue = oldMultiPassCount.getCount();
- // if (oldMultiPassCountValue < 0) {
- // throw new FileNodeProcessorException(String
- // .format("Remove MultiPassCount error, oldMultiPassCount:%d", oldMultiPassCountValue));
- // }
- // LOGGER.debug("Remove multi token:{}, old set:{}, count:{}", token, oldMultiPassTokenSet,
- // oldMultiPassCount.getCount());
- // return true;
- // } else {
- // LOGGER.error("remove token error:{},new set:{}, old set:{}", token, newMultiPassTokenSet,
- // oldMultiPassTokenSet);
- // // should add throw exception
- // return false;
- // }
- // }
- //
- // /**
- // * query data.
- // */
- // public <T extends Comparable<T>> QueryDataSource query(String deviceId, String measurementId,
- // QueryContext context) throws FileNodeProcessorException {
- // // query overflow data
- // MeasurementSchema mSchema;
- // TSDataType dataType;
- //
- // //mSchema = mManager.getSchemaForOnePath(deviceId + "." + measurementId);
- // mSchema = fileSchema.getMeasurementSchema(measurementId);
- // dataType = mSchema.getType();
- //
- // OverflowSeriesDataSource overflowSeriesDataSource;
- // try {
- // overflowSeriesDataSource = overflowProcessor.query(deviceId, measurementId, dataType,
- // mSchema.getProps(), context);
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // // tsfile dataØØ
- // List<TsFileResource> bufferwriteDataInFiles = new ArrayList<>();
- // for (TsFileResource tsFileResource : newFileNodes) {
- // // add the same tsFileResource, but not the same reference
- // if (tsFileResource.isClosed()) {
- // bufferwriteDataInFiles.add(tsFileResource.backUp());
- // }
- // }
- // Pair<ReadOnlyMemChunk, List<ChunkMetaData>> bufferwritedata = new Pair<>(null, null);
- // // bufferwrite data
- // UnsealedTsFile unsealedTsFile = null;
- //
- // if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()
- // && !newFileNodes.get(newFileNodes.size() - 1).getStartTimeMap().isEmpty()) {
- // unsealedTsFile = new UnsealedTsFile();
- // unsealedTsFile.setFilePath(newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath());
- // if (bufferWriteProcessor == null) {
- // throw new FileNodeProcessorException(String.format(
- // "The last of tsfile %s in filenode processor %s is not closed, "
- // + "but the bufferwrite processor is null.",
- // newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath(), getProcessorName()));
- // }
- // bufferwritedata = bufferWriteProcessor
- // .queryBufferWriteData(deviceId, measurementId, dataType, mSchema.getProps());
- //
- // try {
- // List<Modification> pathModifications = context.getPathModifications(
- // bufferWriteProcessor.getCurrentTsFileResource().getModFile(), deviceId
- // + IoTDBConstant.PATH_SEPARATOR + measurementId
- // );
- // if (!pathModifications.isEmpty()) {
- // QueryUtils.modifyChunkMetaData(bufferwritedata.right, pathModifications);
- // }
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- //
- // unsealedTsFile.setTimeSeriesChunkMetaDatas(bufferwritedata.right);
- // }
- // GlobalSortedSeriesDataSource globalSortedSeriesDataSource = new GlobalSortedSeriesDataSource(
- // new Path(deviceId + "." + measurementId), bufferwriteDataInFiles, unsealedTsFile,
- // bufferwritedata.left);
- // return new QueryDataSource(globalSortedSeriesDataSource, overflowSeriesDataSource);
- //
- // }
- //
- // /**
- // * append one specified tsfile to this filenode processor.
- // *
- // * @param appendFile the appended tsfile information
- // * @param appendFilePath the seriesPath of appended file
- // */
- // public void appendFile(TsFileResource appendFile, String appendFilePath)
- // throws FileNodeProcessorException {
- // try {
- // if (!appendFile.getFile().getParentFile().exists()) {
- // appendFile.getFile().getParentFile().mkdirs();
- // }
- // // move file
- // File originFile = new File(appendFilePath);
- // File targetFile = appendFile.getFile();
- // if (!originFile.exists()) {
- // throw new FileNodeProcessorException(
- // String.format("The appended file %s does not exist.", appendFilePath));
- // }
- // if (targetFile.exists()) {
- // throw new FileNodeProcessorException(
- // String.format("The appended target file %s already exists.",
- // appendFile.getFile().getAbsolutePath()));
- // }
- // if (!originFile.renameTo(targetFile)) {
- // LOGGER.warn("File renaming failed when appending new file. Origin: {}, Target: {}",
- // originFile.getPathString(), targetFile.getPathString());
- // }
- // // append the new tsfile
- // this.newFileNodes.add(appendFile);
- // // update the lastUpdateTime
- // for (Entry<String, Long> entry : appendFile.getEndTimeMap().entrySet()) {
- // lastUpdateTimeMap.put(entry.getKey(), entry.getValue());
- // }
- // bufferwriteFlushAction.act();
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // // reconstruct the inverted index of the newFileNodes
- // fileNodeFlushAction.act();
- // addAllFileIntoIndex(newFileNodes);
- // } catch (Exception e) {
- // LOGGER.error("Failed to append the tsfile {} to filenode processor {}.", appendFile,
- // getProcessorName());
- // throw new FileNodeProcessorException(e);
- // }
- // }
- //
- // /**
- // * get overlap tsfiles which are conflict with the appendFile.
- // *
- // * @param appendFile the appended tsfile information
- // */
- // public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
- // throws FileNodeProcessorException {
- // List<String> overlapFiles = new ArrayList<>();
- // try {
- // for (TsFileResource tsFileResource : newFileNodes) {
- // getOverlapFiles(appendFile, tsFileResource, uuid, overlapFiles);
- // }
- // } catch (IOException e) {
- // LOGGER.error("Failed to get overlap tsfiles which conflict with the appendFile.");
- // throw new FileNodeProcessorException(e);
- // }
- // return overlapFiles;
- // }
- //
- // private void getOverlapFiles(TsFileResource appendFile, TsFileResource tsFileResource,
- // String uuid, List<String> overlapFiles) throws IOException {
- // for (Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- // if (tsFileResource.getStartTimeMap().containsKey(entry.getKey()) &&
- // tsFileResource.getEndTime(entry.getKey()) >= entry.getValue()
- // && tsFileResource.getStartTime(entry.getKey()) <= appendFile
- // .getEndTime(entry.getKey())) {
- // String relativeFilePath =
- // Constans.SYNC_SERVER + File.separatorChar + uuid + File.separatorChar
- // + Constans.BACK_UP_DIRECTORY_NAME
- // + File.separatorChar + tsFileResource.getRelativePath();
- // File newFile = new File(
- // DirectoryManager.getInstance().getTsFileFolder(tsFileResource.getBaseDirIndex()),
- // relativeFilePath);
- // if (!newFile.getParentFile().exists()) {
- // newFile.getParentFile().mkdirs();
- // }
- // java.nio.file.Path link = FileSystems.getDefault().getPathString(newFile.getPathString());
- // java.nio.file.Path target = FileSystems.getDefault()
- // .getPathString(tsFileResource.getFile().getAbsolutePath());
- // Files.createLink(link, target);
- // overlapFiles.add(newFile.getPathString());
- // break;
- // }
- // }
- // }
- //
- // /**
- // * add time series.
- // */
- // public void addTimeSeries(String measurementId, TSDataType dataType, TSEncoding encoding,
- // CompressionType compressor, Map<String, String> props) {
- // fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
- // compressor, props));
- // }
- //
- // /**
- // * submit the merge task to the <code>MergePool</code>.
- // *
- // * @return null -can't submit the merge task, because this filenode is not overflowed or it is
- // * merging now. Future - submit the merge task successfully.
- // */
- // Future submitToMerge() {
- // ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- // if (lastMergeTime > 0) {
- // long thisMergeTime = System.currentTimeMillis();
- // long mergeTimeInterval = thisMergeTime - lastMergeTime;
- // ZonedDateTime lastDateTime = ofInstant(Instant.ofEpochMilli(lastMergeTime),
- // zoneId);
- // ZonedDateTime thisDateTime = ofInstant(Instant.ofEpochMilli(thisMergeTime),
- // zoneId);
- // LOGGER.info(
- // "The filenode {} last merge time is {}, this merge time is {}, "
- // + "merge time interval is {}s",
- // getProcessorName(), lastDateTime, thisDateTime, mergeTimeInterval / 1000);
- // }
- // lastMergeTime = System.currentTimeMillis();
- //
- // if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- // if (overflowProcessor.getFileSize() < IoTDBDescriptor.getInstance()
- // .getConfig().getOverflowFileSizeThreshold()) {
- // if (LOGGER.isInfoEnabled()) {
- // LOGGER.info(
- // "Skip this merge taks submission, because the size{} of overflow processor {} "
- // + "does not reaches the threshold {}.",
- // MemUtils.bytesCntToStr(overflowProcessor.getFileSize()), getProcessorName(),
- // MemUtils.bytesCntToStr(
- // IoTDBDescriptor.getInstance().getConfig().getOverflowFileSizeThreshold()));
- // }
- // return null;
- // }
- // } else {
- // LOGGER.info(
- // "Skip this merge taks submission, because the filenode processor {} "
- // + "has no overflow processor.",
- // getProcessorName());
- // return null;
- // }
- // if (isOverflowed && isMerging == FileNodeProcessorStatus.NONE) {
- // Runnable mergeThread;
- // mergeThread = new MergeRunnale();
- // LOGGER.info("Submit the merge task, the merge filenode is {}", getProcessorName());
- // return MergePoolManager.getInstance().submit(mergeThread);
- // } else {
- // if (!isOverflowed) {
- // LOGGER.info(
- // "Skip this merge taks submission, because the filenode processor {} is not " +
- // "overflowed.",
- // getProcessorName());
- // } else {
- // LOGGER.warn(
- // "Skip this merge task submission, because last merge task is not over yet, "
- // + "the merge filenode processor is {}",
- // getProcessorName());
- // }
- // }
- // return null;
- // }
- //
- // /**
- // * Prepare for merge, close the bufferwrite and overflow.
- // */
- // private void prepareForMerge() {
- // try {
- // LOGGER.info("The filenode processor {} prepares for merge, closes the bufferwrite processor",
- // getProcessorName());
- // Future<Boolean> future = closeBufferWrite();
- // future.get();
- // LOGGER.info("The bufferwrite processor {} is closed successfully",
- // getProcessorName());
- // // try to get overflow processor
- // getOverflowProcessor(getProcessorName());
- // // must close the overflow processor
- // while (!getOverflowProcessor().canBeClosed()) {
- // waitForClosing();
- // }
- // LOGGER.info("The filenode processor {} prepares for merge, closes the overflow processor",
- // getProcessorName());
- // getOverflowProcessor().close();
- // } catch (ProcessorException | InterruptedException | ExecutionException e) {
- // LOGGER.error("The filenode processor {} prepares for merge error.", getProcessorName());
- // writeUnlock();
- // throw new ErrorDebugException(e);
- // }
- // }
- //
- // private void waitForClosing() {
- // try {
- // LOGGER.info(
- // "The filenode processor {} prepares for merge, the overflow {} can't be closed, "
- // + "wait 100ms,",
- // getProcessorName(), getProcessorName());
- // TimeUnit.MICROSECONDS.sleep(100);
- // } catch (InterruptedException e) {
- // Thread.currentThread().interrupt();
- // }
- // }
- //
- // /**
- // * Merge this storage group, merge the tsfile data with overflow data.
- // */
- // public void merge() throws FileNodeProcessorException {
- // // close bufferwrite and overflow, prepare for merge
- // LOGGER.info("The filenode processor {} begins to merge.", getProcessorName());
- // writeLock();
- // prepareForMerge();
- // // change status from overflowed to no overflowed
- // isOverflowed = false;
- // // change status from work to merge
- // isMerging = FileNodeProcessorStatus.MERGING_WRITE;
- // // check the empty file
- // Map<String, Long> startTimeMap = emptyTsFileResource.getStartTimeMap();
- // mergeCheckEmptyFile(startTimeMap);
- //
- // for (TsFileResource tsFileResource : newFileNodes) {
- // if (tsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // tsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- // }
- // }
- //
- // addAllFileIntoIndex(newFileNodes);
- // synchronized (fileNodeProcessorStore) {
- // fileNodeProcessorStore.setOverflowed(isOverflowed);
- // fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // // flush this filenode information
- // try {
- // writeStoreToDisk(fileNodeProcessorStore);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("The filenode processor {} writes restore information error when merging.",
- // getProcessorName(), e);
- // writeUnlock();
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // // add numOfMergeFile to control the number of the merge file
- // List<TsFileResource> backupIntervalFiles;
- //
- // backupIntervalFiles = switchFileNodeToMerge();
- // //
- // // clear empty file
- // //
- // boolean needEmtpy = false;
- // if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // needEmtpy = true;
- // }
- // emptyTsFileResource.clear();
- // // attention
- // try {
- // if (overflowProcessor.isClosed()) {
- // overflowProcessor.reopen();
- // }
- // overflowProcessor.switchWorkToMerge();
- // } catch (ProcessorException | IOException e) {
- // LOGGER.error("The filenode processor {} can't switch overflow processor from work to merge.",
- // getProcessorName(), e);
- // writeUnlock();
- // throw new FileNodeProcessorException(e);
- // }
- // LOGGER.info("The filenode processor {} switches from {} to {}.", getProcessorName(),
- // FileNodeProcessorStatus.NONE, FileNodeProcessorStatus.MERGING_WRITE);
- // writeUnlock();
- //
- // // query tsfile data and overflow data, and merge them
- // int numOfMergeFiles = 0;
- // int allNeedMergeFiles = backupIntervalFiles.size();
- // for (TsFileResource backupIntervalFile : backupIntervalFiles) {
- // numOfMergeFiles++;
- // if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.CHANGED) {
- // // query data and merge
- // String filePathBeforeMerge = backupIntervalFile.getRelativePath();
- // try {
- // LOGGER.info(
- // "The filenode processor {} begins merging the {}/{} tsfile[{}] with "
- // + "overflow file, the process is {}%",
- // getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- // (int) (((numOfMergeFiles - 1) / (float) allNeedMergeFiles) * 100));
- // long startTime = System.currentTimeMillis();
- // String newFile = queryAndWriteDataForMerge(backupIntervalFile);
- // long endTime = System.currentTimeMillis();
- // long timeConsume = endTime - startTime;
- // ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- // LOGGER.info(
- // "The fileNode processor {} has merged the {}/{} tsfile[{}->{}] over, "
- // + "start time of merge is {}, end time of merge is {}, "
- // + "time consumption is {}ms,"
- // + " the process is {}%",
- // getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- // newFile, ofInstant(Instant.ofEpochMilli(startTime),
- // zoneId), ofInstant(Instant.ofEpochMilli(endTime), zoneId), timeConsume,
- // numOfMergeFiles / (float) allNeedMergeFiles * 100);
- // } catch (IOException | PathErrorException e) {
- // LOGGER.error("Merge: query and insert data error.", e);
- // throw new FileNodeProcessorException(e);
- // }
- // } else if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- // LOGGER.error("The overflowChangeType of backupIntervalFile must not be {}",
- // OverflowChangeType.MERGING_CHANGE);
- // // handle this error, throw one runtime exception
- // throw new FileNodeProcessorException(
- // "The overflowChangeType of backupIntervalFile must not be "
- // + OverflowChangeType.MERGING_CHANGE);
- // } else {
- // LOGGER.debug(
- // "The filenode processor {} is merging, the interval file {} doesn't "
- // + "need to be merged.",
- // getProcessorName(), backupIntervalFile.getRelativePath());
- // }
- // }
- //
- // // change status from merge to wait
- // switchMergeToWaiting(backupIntervalFiles, needEmtpy);
- //
- // // change status from wait to work
- // switchWaitingToWorking();
- // }
- //
- // private void mergeCheckEmptyFile(Map<String, Long> startTimeMap) {
- // if (emptyTsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- // return;
- // }
- // Iterator<Entry<String, Long>> iterator = emptyTsFileResource.getEndTimeMap().entrySet()
- // .iterator();
- // while (iterator.hasNext()) {
- // Entry<String, Long> entry = iterator.next();
- // String deviceId = entry.getKey();
- // if (invertedIndexOfFiles.containsKey(deviceId)) {
- // invertedIndexOfFiles.get(deviceId).get(0).setOverflowChangeType(OverflowChangeType.CHANGED);
- // startTimeMap.remove(deviceId);
- // iterator.remove();
- // }
- // }
- // if (emptyTsFileResource.checkEmpty()) {
- // emptyTsFileResource.clear();
- // } else {
- // if (!newFileNodes.isEmpty()) {
- // TsFileResource first = newFileNodes.get(0);
- // for (String deviceId : emptyTsFileResource.getStartTimeMap().keySet()) {
- // first.setStartTime(deviceId, emptyTsFileResource.getStartTime(deviceId));
- // first.setEndTime(deviceId, emptyTsFileResource.getEndTime(deviceId));
- // first.setOverflowChangeType(OverflowChangeType.CHANGED);
- // }
- // emptyTsFileResource.clear();
- // } else {
- // emptyTsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- // }
- // }
- // }
- //
- // private List<TsFileResource> switchFileNodeToMerge() throws FileNodeProcessorException {
- // List<TsFileResource> result = new ArrayList<>();
- // if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // // add empty
- // result.add(emptyTsFileResource.backUp());
- // if (!newFileNodes.isEmpty()) {
- // throw new FileNodeProcessorException(
- // String.format("The status of empty file is %s, but the new file list is not empty",
- // emptyTsFileResource.getOverflowChangeType()));
- // }
- // return result;
- // }
- // if (newFileNodes.isEmpty()) {
- // LOGGER.error("No file was changed when merging, the filenode is {}", getProcessorName());
- // throw new FileNodeProcessorException(
- // "No file was changed when merging, the filenode is " + getProcessorName());
- // }
- // for (TsFileResource tsFileResource : newFileNodes) {
- // updateFileNode(tsFileResource, result);
- // }
- // return result;
- // }
- //
- // private void updateFileNode(TsFileResource tsFileResource, List<TsFileResource> result) {
- // if (tsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- // result.add(tsFileResource.backUp());
- // } else {
- // Map<String, Long> startTimeMap = new HashMap<>();
- // Map<String, Long> endTimeMap = new HashMap<>();
- // for (String deviceId : tsFileResource.getEndTimeMap().keySet()) {
- // List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- // int index = temp.indexOf(tsFileResource);
- // int size = temp.size();
- // // start time
- // if (index == 0) {
- // startTimeMap.put(deviceId, 0L);
- // } else {
- // startTimeMap.put(deviceId, tsFileResource.getStartTime(deviceId));
- // }
- // // end time
- // if (index < size - 1) {
- // endTimeMap.put(deviceId, temp.get(index + 1).getStartTime(deviceId) - 1);
- // } else {
- // endTimeMap.put(deviceId, tsFileResource.getEndTime(deviceId));
- // }
- // }
- // TsFileResource node = new TsFileResource(startTimeMap, endTimeMap,
- // tsFileResource.getOverflowChangeType(), tsFileResource.getFile());
- // result.add(node);
- // }
- // }
- //
- // private void switchMergeToWaiting(List<TsFileResource> backupIntervalFiles, boolean needEmpty)
- // throws FileNodeProcessorException {
- // LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- // FileNodeProcessorStatus.MERGING_WRITE, FileNodeProcessorStatus.WAITING);
- // writeLock();
- // try {
- // oldMultiPassTokenSet = newMultiPassTokenSet;
- // oldMultiPassCount = new CountDownLatch(newMultiPassCount.get());
- // newMultiPassTokenSet = new HashSet<>();
- // newMultiPassCount = new AtomicInteger(0);
- // List<TsFileResource> result = new ArrayList<>();
- // int beginIndex = 0;
- // if (needEmpty) {
- // TsFileResource empty = backupIntervalFiles.get(0);
- // if (!empty.checkEmpty()) {
- // updateEmpty(empty, result);
- // beginIndex++;
- // }
- // }
- // // reconstruct the file index
- // addAllFileIntoIndex(backupIntervalFiles);
- // // check the merge changed file
- // for (int i = beginIndex; i < backupIntervalFiles.size(); i++) {
- // TsFileResource newFile = newFileNodes.get(i - beginIndex);
- // TsFileResource temp = backupIntervalFiles.get(i);
- // if (newFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- // updateMergeChanged(newFile, temp);
- // }
- // if (!temp.checkEmpty()) {
- // result.add(temp);
- // }
- // }
- // // add new file when merge
- // for (int i = backupIntervalFiles.size() - beginIndex; i < newFileNodes.size(); i++) {
- // TsFileResource fileNode = newFileNodes.get(i);
- // if (fileNode.isClosed()) {
- // result.add(fileNode.backUp());
- // } else {
- // result.add(fileNode);
- // }
- // }
- //
- // isMerging = FileNodeProcessorStatus.WAITING;
- // newFileNodes = result;
- // // reconstruct the index
- // addAllFileIntoIndex(newFileNodes);
- // // clear merge changed
- // for (TsFileResource fileNode : newFileNodes) {
- // fileNode.clearMergeChanged();
- // }
- //
- // synchronized (fileNodeProcessorStore) {
- // fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- // fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // try {
- // writeStoreToDisk(fileNodeProcessorStore);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error(
- // "Merge: failed to insert filenode information to revocery file, the filenode is " +
- // "{}.",
- // getProcessorName(), e);
- // throw new FileNodeProcessorException(
- // "Merge: insert filenode information to revocery file failed, the filenode is "
- // + getProcessorName());
- // }
- // }
- // } finally {
- // writeUnlock();
- // }
- // }
- //
- // private void updateEmpty(TsFileResource empty, List<TsFileResource> result) {
- // for (String deviceId : empty.getStartTimeMap().keySet()) {
- // if (invertedIndexOfFiles.containsKey(deviceId)) {
- // TsFileResource temp = invertedIndexOfFiles.get(deviceId).get(0);
- // if (temp.getMergeChanged().contains(deviceId)) {
- // empty.setOverflowChangeType(OverflowChangeType.CHANGED);
- // break;
- // }
- // }
- // }
- // empty.clearMergeChanged();
- // result.add(empty.backUp());
- // }
- //
- // private void updateMergeChanged(TsFileResource newFile, TsFileResource temp) {
- // for (String deviceId : newFile.getMergeChanged()) {
- // if (temp.getStartTimeMap().containsKey(deviceId)) {
- // temp.setOverflowChangeType(OverflowChangeType.CHANGED);
- // } else {
- // changeTypeToChanged(deviceId, newFile.getStartTime(deviceId),
- // newFile.getEndTime(deviceId));
- // }
- // }
- // }
- //
- //
- // private void switchWaitingToWorking()
- // throws FileNodeProcessorException {
- //
- // LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- // FileNodeProcessorStatus.WAITING, FileNodeProcessorStatus.NONE);
- //
- // if (oldMultiPassCount != null) {
- // LOGGER.info("The old Multiple Pass Token set is {}, the old Multiple Pass Count is {}",
- // oldMultiPassTokenSet,
- // oldMultiPassCount);
- // try {
- // oldMultiPassCount.await();
- // } catch (InterruptedException e) {
- // LOGGER.info(
- // "The filenode processor {} encountered an error when it waits for all old queries over.",
- // getProcessorName());
- // throw new FileNodeProcessorException(e);
- // }
- // }
- //
- // try {
- // writeLock();
- // try {
- // // delete the all files which are in the newFileNodes
- // // notice: the last restore file of the interval file
- //
- // List<String> bufferwriteDirPathList = DIRECTORY_MANAGER.getAllTsFileFolders();
- // List<File> bufferwriteDirList = new ArrayList<>();
- // collectBufferWriteDirs(bufferwriteDirPathList, bufferwriteDirList);
- //
- // Set<String> bufferFiles = new HashSet<>();
- // collectBufferWriteFiles(bufferFiles);
- //
- // // add the restore file, if the last file is not closed
- // if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
- // String bufferFileRestorePath =
- // newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath() + RESTORE_FILE_SUFFIX;
- // bufferFiles.add(bufferFileRestorePath);
- // }
- //
- // deleteBufferWriteFiles(bufferwriteDirList, bufferFiles);
- //
- // // merge switch
- // changeFileNodes();
- //
- // // overflow switch from merge to work
- // overflowProcessor.switchMergeToWork();
- //
- // // insert status to file
- // isMerging = FileNodeProcessorStatus.NONE;
- // synchronized (fileNodeProcessorStore) {
- // fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- // fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // writeStoreToDisk(fileNodeProcessorStore);
- // }
- // } catch (IOException e) {
- // LOGGER.info(
- // "The filenode processor {} encountered an error when its "
- // + "status switched from {} to {}.",
- // getProcessorName(), FileNodeProcessorStatus.NONE,
- // FileNodeProcessorStatus.MERGING_WRITE);
- // throw new FileNodeProcessorException(e);
- // } finally {
- // writeUnlock();
- // }
- // } finally {
- // oldMultiPassTokenSet = null;
- // oldMultiPassCount = null;
- // }
- //
- // }
- //
- // private void collectBufferWriteDirs(List<String> bufferwriteDirPathList,
- // List<File> bufferwriteDirList) {
- // for (String bufferwriteDirPath : bufferwriteDirPathList) {
- // if (bufferwriteDirPath.size() > 0
- // && bufferwriteDirPath.charAt(bufferwriteDirPath.size() - 1)
- // != File.separatorChar) {
- // bufferwriteDirPath = bufferwriteDirPath + File.separatorChar;
- // }
- // bufferwriteDirPath = bufferwriteDirPath + getProcessorName();
- // File bufferwriteDir = new File(bufferwriteDirPath);
- // bufferwriteDirList.add(bufferwriteDir);
- // if (!bufferwriteDir.exists()) {
- // bufferwriteDir.mkdirs();
- // }
- // }
- // }
- //
- // private void collectBufferWriteFiles(Set<String> bufferFiles) {
- // for (TsFileResource bufferFileNode : newFileNodes) {
- // String bufferFilePath = bufferFileNode.getFile().getAbsolutePath();
- // if (bufferFilePath != null) {
- // bufferFiles.add(bufferFilePath);
- // }
- // }
- // }
- //
- // private void deleteBufferWriteFiles(List<File> bufferwriteDirList, Set<String> bufferFiles)
- // throws IOException {
- // for (File bufferwriteDir : bufferwriteDirList) {
- // File[] files = bufferwriteDir.listFiles();
- // if (files == null) {
- // continue;
- // }
- // for (File file : files) {
- // if (!bufferFiles.contains(file.getPathString())) {
- // FileReaderManager.getInstance().closeFileAndRemoveReader(file.getPathString());
- // if (!file.delete()) {
- // LOGGER.warn("Cannot delete BufferWrite file {}", file.getPathString());
- // }
- // }
- // }
- // }
- // }
- //
- // private void changeFileNodes() {
- // for (TsFileResource fileNode : newFileNodes) {
- // if (fileNode.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // fileNode.setOverflowChangeType(OverflowChangeType.CHANGED);
- // }
- // }
- // }
- //
- // private String queryAndWriteDataForMerge(TsFileResource backupIntervalFile)
- // throws IOException, FileNodeProcessorException, PathErrorException {
- // Map<String, Long> startTimeMap = new HashMap<>();
- // Map<String, Long> endTimeMap = new HashMap<>();
- //
- // mergeFileWriter = null;
- // mergeOutputPath = null;
- // mergeBaseDir = null;
- // mergeFileName = null;
- // // modifications are blocked before mergeModification is created to avoid
- // // losing some modification.
- // mergeDeleteLock.lock();
- // QueryContext context = new QueryContext();
- // try {
- // FileReaderManager.getInstance().increaseFileReaderReference(backupIntervalFile.getFilePath(),
- // true);
- // for (String deviceId : backupIntervalFile.getStartTimeMap().keySet()) {
- // // query one deviceId
- // List<Path> pathList = new ArrayList<>();
- // mergeIsChunkGroupHasData = false;
- // mergeStartPos = -1;
- // ChunkGroupFooter footer;
- // int numOfChunk = 0;
- // try {
- // List<String> pathStrings = mManager.getLeafNodePathInNextLevel(deviceId);
- // for (String string : pathStrings) {
- // pathList.add(new Path(string));
- // }
- // } catch (PathErrorException e) {
- // LOGGER.error("Can't get all the paths from MManager, the deviceId is {}", deviceId);
- // throw new FileNodeProcessorException(e);
- // }
- // if (pathList.isEmpty()) {
- // continue;
- // }
- // for (Path path : pathList) {
- // // query one measurement in the special deviceId
- // String measurementId = path.getMeasurement();
- // TSDataType dataType = mManager.getSeriesType(path.getFullPath());
- // OverflowSeriesDataSource overflowSeriesDataSource = overflowProcessor.queryMerge(deviceId,
- // measurementId, dataType, true, context);
- // Filter timeFilter = FilterFactory
- // .and(TimeFilter.gtEq(backupIntervalFile.getStartTime(deviceId)),
- // TimeFilter.ltEq(backupIntervalFile.getEndTime(deviceId)));
- // SingleSeriesExpression seriesFilter = new SingleSeriesExpression(path, timeFilter);
- //
- // for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- // .getOverflowInsertFileList()) {
- // FileReaderManager.getInstance()
- // .increaseFileReaderReference(overflowInsertFile.getFilePath(),
- // false);
- // }
- //
- // IReader seriesReader = SeriesReaderFactory.getInstance()
- // .createSeriesReaderForMerge(backupIntervalFile,
- // overflowSeriesDataSource, seriesFilter, context);
- // numOfChunk += queryAndWriteSeries(seriesReader, path, seriesFilter, dataType,
- // startTimeMap, endTimeMap, overflowSeriesDataSource);
- // }
- // if (mergeIsChunkGroupHasData) {
- // // end the new rowGroupMetadata
- // mergeFileWriter.endChunkGroup(0);
- // }
- // }
- // } finally {
- // FileReaderManager.getInstance().decreaseFileReaderReference(backupIntervalFile.getFilePath(),
- // true);
- //
- // if (mergeDeleteLock.isLocked()) {
- // mergeDeleteLock.unlock();
- // }
- // }
- //
- // if (mergeFileWriter != null) {
- // mergeFileWriter.endFile(fileSchema);
- // }
- // backupIntervalFile.setFile(new File(mergeBaseDir + File.separator + mergeFileName));
- // backupIntervalFile.setOverflowChangeType(OverflowChangeType.NO_CHANGE);
- // backupIntervalFile.setStartTimeMap(startTimeMap);
- // backupIntervalFile.setEndTimeMap(endTimeMap);
- // backupIntervalFile.setModFile(mergingModification);
- // mergingModification = null;
- // return mergeFileName;
- // }
- //
- // private int queryAndWriteSeries(IReader seriesReader, Path path,
- // SingleSeriesExpression seriesFilter, TSDataType dataType,
- // Map<String, Long> startTimeMap, Map<String, Long> endTimeMap,
- // OverflowSeriesDataSource overflowSeriesDataSource)
- // throws IOException {
- // int numOfChunk = 0;
- // try {
- // if (!seriesReader.hasNext()) {
- // LOGGER.debug(
- // "The time-series {} has no data with the filter {} in the filenode processor {}",
- // path, seriesFilter, getProcessorName());
- // } else {
- // numOfChunk++;
- // TimeValuePair timeValuePair = seriesReader.next();
- // if (mergeFileWriter == null) {
- // mergeBaseDir = DIRECTORY_MANAGER.getNextFolderForSequenceFile();
- // mergeFileName = timeValuePair.getTimestamp()
- // + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR + System.currentTimeMillis();
- // mergeOutputPath = constructOutputFilePath(mergeBaseDir, getProcessorName(),
- // mergeFileName);
- // mergeFileName = getProcessorName() + File.separatorChar + mergeFileName;
- // mergeFileWriter = new TsFileIOWriter(new File(mergeOutputPath));
- // mergingModification = new ModificationFile(mergeOutputPath
- // + ModificationFile.FILE_SUFFIX);
- // mergeDeleteLock.unlock();
- // }
- // if (!mergeIsChunkGroupHasData) {
- // // start a new rowGroupMetadata
- // mergeIsChunkGroupHasData = true;
- // // the datasize and numOfChunk is fake
- // // the accurate datasize and numOfChunk will get after insert all this device data.
- // mergeFileWriter.startChunkGroup(path.getDevice());// TODO please check me.
- // mergeStartPos = mergeFileWriter.getPos();
- // }
- // // init the serieswWriteImpl
- // MeasurementSchema measurementSchema = fileSchema
- // .getMeasurementSchema(path.getMeasurement());
- // ChunkBuffer pageWriter = new ChunkBuffer(measurementSchema);
- // int pageSizeThreshold = TSFileConfig.pageSizeInByte;
- // ChunkWriterImpl seriesWriterImpl = new ChunkWriterImpl(measurementSchema, pageWriter,
- // pageSizeThreshold);
- // // insert the series data
- // writeOneSeries(path.getDevice(), seriesWriterImpl, dataType,
- // seriesReader,
- // startTimeMap, endTimeMap, timeValuePair);
- // // flush the series data
- // seriesWriterImpl.writeToFileWriter(mergeFileWriter);
- // }
- // } finally {
- // for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- // .getOverflowInsertFileList()) {
- // FileReaderManager.getInstance()
- // .decreaseFileReaderReference(overflowInsertFile.getFilePath(),
- // false);
- // }
- // }
- // return numOfChunk;
- // }
- //
- //
- // private void writeOneSeries(String deviceId, ChunkWriterImpl seriesWriterImpl,
- // TSDataType dataType, IReader seriesReader, Map<String, Long> startTimeMap,
- // Map<String, Long> endTimeMap, TimeValuePair firstTVPair) throws IOException {
- // long startTime;
- // long endTime;
- // TimeValuePair localTV = firstTVPair;
- // writeTVPair(seriesWriterImpl, dataType, localTV);
- // startTime = endTime = localTV.getTimestamp();
- // if (!startTimeMap.containsKey(deviceId) || startTimeMap.get(deviceId) > startTime) {
- // startTimeMap.put(deviceId, startTime);
- // }
- // if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- // endTimeMap.put(deviceId, endTime);
- // }
- // while (seriesReader.hasNext()) {
- // localTV = seriesReader.next();
- // endTime = localTV.getTimestamp();
- // writeTVPair(seriesWriterImpl, dataType, localTV);
- // }
- // if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- // endTimeMap.put(deviceId, endTime);
- // }
- // }
- //
- // private void writeTVPair(ChunkWriterImpl seriesWriterImpl, TSDataType dataType,
- // TimeValuePair timeValuePair) throws IOException {
- // switch (dataType) {
- // case BOOLEAN:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
- // break;
- // case INT32:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
- // break;
- // case INT64:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
- // break;
- // case FLOAT:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- // break;
- // case DOUBLE:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
- // break;
- // case TEXT:
- // seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
- // break;
- // default:
- // LOGGER.error("Not support data type: {}", dataType);
- // break;
- // }
- // }
- //
- //
- // private String constructOutputFilePath(String baseDir, String processorName, String fileName) {
- //
- // String localBaseDir = baseDir;
- // if (localBaseDir.charAt(localBaseDir.size() - 1) != File.separatorChar) {
- // localBaseDir = localBaseDir + File.separatorChar + processorName;
- // }
- // File dataDir = new File(localBaseDir);
- // if (!dataDir.exists()) {
- // LOGGER.warn("The bufferwrite processor data dir doesn't exists, create new directory {}",
- // localBaseDir);
- // dataDir.mkdirs();
- // }
- // File outputFile = new File(dataDir, fileName);
- // return outputFile.getPathString();
- // }
- //
- // private FileSchema constructFileSchema(String processorName) throws WriteProcessException {
- //
- // List<MeasurementSchema> columnSchemaList;
- // columnSchemaList = mManager.getSchemaForFileName(processorName);
- //
- // FileSchema schema = new FileSchema();
- // for (MeasurementSchema measurementSchema : columnSchemaList) {
- // schema.registerMeasurement(measurementSchema);
- // }
- // return schema;
- //
- // }
- //
- // @Override
- // public boolean canBeClosed() {
- // if (isMerging != FileNodeProcessorStatus.NONE) {
- // LOGGER.info("The filenode {} can't be closed, because the filenode status is {}",
- // getProcessorName(),
- // isMerging);
- // return false;
- // }
- // if (newMultiPassCount.get() != 0) {
- // LOGGER.warn("The filenode {} can't be closed, because newMultiPassCount is {}. The newMultiPassTokenSet is {}",
- // getProcessorName(), newMultiPassCount, newMultiPassTokenSet);
- // return false;
- // }
- //
- // if (oldMultiPassCount == null) {
- // return true;
- // }
- // if (oldMultiPassCount.getCount() == 0) {
- // return true;
- // } else {
- // LOGGER.info("The filenode {} can't be closed, because oldMultiPassCount is {}",
- // getProcessorName(), oldMultiPassCount.getCount());
- // return false;
- // }
- // }
- //
- // @Override
- // public FileNodeFlushFuture flush() throws IOException {
- // Future<Boolean> bufferWriteFlushFuture = null;
- // Future<Boolean> overflowFlushFuture = null;
- // if (bufferWriteProcessor != null) {
- // bufferWriteFlushFuture = bufferWriteProcessor.flush();
- // }
- // if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- // overflowFlushFuture = overflowProcessor.flush();
- // }
- // return new FileNodeFlushFuture(bufferWriteFlushFuture, overflowFlushFuture);
- // }
- //
- // /**
- // * Close the bufferwrite processor.
- // */
- // public Future<Boolean> closeBufferWrite() throws FileNodeProcessorException {
- // if (bufferWriteProcessor == null) {
- // return new ImmediateFuture<>(true);
- // }
- // try {
- // while (!bufferWriteProcessor.canBeClosed()) {
- // waitForBufferWriteClose();
- // }
- // bufferWriteProcessor.close();
- // Future<Boolean> result = bufferWriteProcessor.getCloseFuture();
- // closingBufferWriteProcessor.add(bufferWriteProcessor);
- // bufferWriteProcessor = null;
- // return result;
- // } catch (BufferWriteProcessorException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- //
- //
- //
- // private void waitForBufferWriteClose() {
- // try {
- // LOGGER.info("The bufferwrite {} can't be closed, wait 100ms",
- // bufferWriteProcessor.getProcessorName());
- // TimeUnit.MICROSECONDS.sleep(100);
- // } catch (InterruptedException e) {
- // LOGGER.error("Unexpected interruption", e);
- // Thread.currentThread().interrupt();
- // }
- // }
- //
- // /**
- // * Close the overflow processor.
- // */
- // public void closeOverflow() throws FileNodeProcessorException {
- // if (overflowProcessor == null || overflowProcessor.isClosed()) {
- // return;
- // }
- // try {
- // while (!overflowProcessor.canBeClosed()) {
- // waitForOverflowClose();
- // }
- // overflowProcessor.close();
- // } catch (OverflowProcessorException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- //
- // private void waitForOverflowClose() {
- // try {
- // LOGGER.info("The overflow {} can't be closed, wait 100ms",
- // overflowProcessor.getProcessorName());
- // TimeUnit.MICROSECONDS.sleep(100);
- // } catch (InterruptedException e) {
- // LOGGER.error("Unexpected interruption", e);
- // Thread.currentThread().interrupt();
- // }
- // }
- //
- // @Override
- // public void close() throws FileNodeProcessorException {
- // LOGGER.info("Will close FileNode Processor {}.", getProcessorName());
- // Future<Boolean> result = closeBufferWrite();
- // try {
- // result.get();
- // } catch (InterruptedException | ExecutionException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // closeOverflow();
- // for (TsFileResource fileNode : newFileNodes) {
- // if (fileNode.getModFile() != null) {
- // try {
- // fileNode.getModFile().close();
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // }
- // }
- //
- // /**
- // * deregister the filenode processor.
- // */
- // public void delete() throws ProcessorException {
- // if (TsFileDBConf.isEnableStatMonitor()) {
- // // remove the monitor
- // LOGGER.info("Deregister the filenode processor: {} from monitor.", getProcessorName());
- // StatMonitor.getInstance().deregisterStatistics(statStorageDeltaName);
- // }
- // closeBufferWrite();
- // closeOverflow();
- // for (TsFileResource fileNode : newFileNodes) {
- // if (fileNode.getModFile() != null) {
- // try {
- // fileNode.getModFile().close();
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // }
- // }
- //
- // @Override
- // public long memoryUsage() {
- // long memSize = 0;
- // if (bufferWriteProcessor != null) {
- // memSize += bufferWriteProcessor.memoryUsage();
- // }
- // if (overflowProcessor != null) {
- // memSize += overflowProcessor.memoryUsage();
- // }
- // return memSize;
- // }
- //
- // private void writeStoreToDisk(FileNodeProcessorStore fileNodeProcessorStore)
- // throws FileNodeProcessorException {
- //
- // synchronized (fileNodeRestoreLock) {
- // try (FileOutputStream fileOutputStream = new FileOutputStream(fileNodeRestoreFilePath)) {
- // fileNodeProcessorStore.serialize(fileOutputStream);
- // LOGGER.debug("The filenode processor {} writes restore information to the restore file",
- // getProcessorName());
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // }
- //
- // private FileNodeProcessorStore readStoreFromDisk() throws FileNodeProcessorException {
- //
- // synchronized (fileNodeRestoreLock) {
- // File restoreFile = new File(fileNodeRestoreFilePath);
- // if (!restoreFile.exists() || restoreFile.size() == 0) {
- // try {
- // return new FileNodeProcessorStore(false, new HashMap<>(),
- // new TsFileResource(null, false),
- // new ArrayList<>(), FileNodeProcessorStatus.NONE, 0);
- // } catch (IOException e) {
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // try (FileInputStream inputStream = new FileInputStream(fileNodeRestoreFilePath)) {
- // return FileNodeProcessorStore.deSerialize(inputStream);
- // } catch (IOException e) {
- // LOGGER
- // .error("Failed to deserialize the FileNodeRestoreFile {}, {}", fileNodeRestoreFilePath,
- // e);
- // throw new FileNodeProcessorException(e);
- // }
- // }
- // }
- //
- // String getFileNodeRestoreFilePath() {
- // return fileNodeRestoreFilePath;
- // }
- //
- // /**
- // * Delete data whose timestamp <= 'timestamp' and belong to timeseries deviceId.measurementId.
- // *
- // * @param deviceId the deviceId of the timeseries to be deleted.
- // * @param measurementId the measurementId of the timeseries to be deleted.
- // * @param timestamp the delete range is (0, timestamp].
- // */
- // public void delete(String deviceId, String measurementId, long timestamp) throws IOException {
- // // TODO: how to avoid partial deletion?
- // mergeDeleteLock.lock();
- // long version = versionController.nextVersion();
- //
- // // record what files are updated so we can roll back them in case of exception
- // List<ModificationFile> updatedModFiles = new ArrayList<>();
- //
- // try {
- // String fullPath = deviceId +
- // IoTDBConstant.PATH_SEPARATOR + measurementId;
- // Deletion deletion = new Deletion(fullPath, version, timestamp);
- // if (mergingModification != null) {
- // mergingModification.write(deletion);
- // updatedModFiles.add(mergingModification);
- // }
- // deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- // // delete data in memory
- // OverflowProcessor ofProcessor = getOverflowProcessor(getProcessorName());
- // ofProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- // if (bufferWriteProcessor != null) {
- // bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- // }
- // } catch (Exception e) {
- // // roll back
- // for (ModificationFile modFile : updatedModFiles) {
- // modFile.abort();
- // }
- // throw new IOException(e);
- // } finally {
- // mergeDeleteLock.unlock();
- // }
- // }
- //
- // private void deleteBufferWriteFiles(String deviceId, Deletion deletion,
- // List<ModificationFile> updatedModFiles) throws IOException {
- // BufferWriteProcessor bufferWriteProcessor = getBufferWriteProcessor();
- // TsFileResource resource = null;
- // if (bufferWriteProcessor != null) {
- // //bufferWriteProcessor == null means the bufferWriteProcessor is closed now.
- // resource = bufferWriteProcessor.getCurrentTsFileResource();
- // if (resource != null && resource.containsDevice(deviceId)) {
- // resource.getModFile().write(deletion);
- // updatedModFiles.add(resource.getModFile());
- // }
- // }
- //
- // for (TsFileResource fileNode : newFileNodes) {
- // if (fileNode != resource && fileNode.containsDevice(deviceId)
- // && fileNode.getStartTime(deviceId) <= deletion.getTimestamp()) {
- // fileNode.getModFile().write(deletion);
- // updatedModFiles.add(fileNode.getModFile());
- // }
- // }
- // }
- //
- // /**
- // * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- // */
- // public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- // throws IOException, BufferWriteProcessorException {
- // String fullPath = deviceId +
- // IoTDBConstant.PATH_SEPARATOR + measurementId;
- // long version = versionController.nextVersion();
- // Deletion deletion = new Deletion(fullPath, version, timestamp);
- //
- // List<ModificationFile> updatedModFiles = new ArrayList<>();
- // try {
- // deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- // } catch (IOException e) {
- // for (ModificationFile modificationFile : updatedModFiles) {
- // modificationFile.abort();
- // }
- // throw e;
- // }
- // if (bufferWriteProcessor != null) {
- // try {
- // bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- // } catch (BufferWriteProcessorException e) {
- // throw new IOException(e);
- // }
- // }
- // }
- //
- // /**
- // * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- // */
- // public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- // throws ProcessorException {
- // long version = versionController.nextVersion();
- //
- // OverflowProcessor overflowProcessor = getOverflowProcessor(getProcessorName());
- // List<ModificationFile> updatedModFiles = new ArrayList<>();
- // try {
- // overflowProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- // } catch (IOException e) {
- // for (ModificationFile modificationFile : updatedModFiles) {
- // try {
- // modificationFile.abort();
- // } catch (IOException e1) {
- // throw new ProcessorException(e);
- // }
- // }
- // throw new ProcessorException(e);
- // }
- // }
- //
- // public CopyOnReadLinkedList<BufferWriteProcessor> getClosingBufferWriteProcessor() {
- // for (BufferWriteProcessor processor: closingBufferWriteProcessor.read()) {
- // if (processor.isClosed()) {
- // closingBufferWriteProcessor.remove(processor);
- // }
- // }
- // closingBufferWriteProcessor.clear();
- // return closingBufferWriteProcessor;
- // }
- //
- // @Override
- // public boolean equals(Object o) {
- // if (this == o) {
- // return true;
- // }
- // if (o == null || getClass() != o.getClass()) {
- // return false;
- // }
- // if (!super.equals(o)) {
- // return false;
- // }
- // FileNodeProcessor that = (FileNodeProcessor) o;
- // return isOverflowed == that.isOverflowed &&
- // numOfMergeFile == that.numOfMergeFile &&
- // lastMergeTime == that.lastMergeTime &&
- // multiPassLockToken == that.multiPassLockToken &&
- // Objects.equals(statStorageDeltaName, that.statStorageDeltaName) &&
- // Objects.equals(statParamsHashMap, that.statParamsHashMap) &&
- // Objects.equals(lastUpdateTimeMap, that.lastUpdateTimeMap) &&
- // Objects.equals(flushLastUpdateTimeMap, that.flushLastUpdateTimeMap) &&
- // Objects.equals(invertedIndexOfFiles, that.invertedIndexOfFiles) &&
- // Objects.equals(emptyTsFileResource, that.emptyTsFileResource) &&
- // Objects.equals(newFileNodes, that.newFileNodes) &&
- // isMerging == that.isMerging &&
- // Objects.equals(fileNodeProcessorStore, that.fileNodeProcessorStore) &&
- // Objects.equals(fileNodeRestoreFilePath, that.fileNodeRestoreFilePath) &&
- // Objects.equals(bufferWriteProcessor, that.bufferWriteProcessor) &&
- // Objects.equals(overflowProcessor, that.overflowProcessor) &&
- // Objects.equals(oldMultiPassTokenSet, that.oldMultiPassTokenSet) &&
- // Objects.equals(newMultiPassTokenSet, that.newMultiPassTokenSet) &&
- // Objects.equals(oldMultiPassCount, that.oldMultiPassCount) &&
- // Objects.equals(newMultiPassCount, that.newMultiPassCount) &&
- // Objects.equals(parameters, that.parameters) &&
- // Objects.equals(fileSchema, that.fileSchema) &&
- // Objects.equals(fileNodeFlushAction, that.fileNodeFlushAction) &&
- // Objects.equals(bufferwriteFlushAction, that.bufferwriteFlushAction) &&
- // Objects.equals(overflowFlushAction, that.overflowFlushAction);
- // }
- //
- // @Override
- // public int hashCode() {
- // return processorName.hashCode();
- // }
- //
- // public class MergeRunnale implements Runnable {
- //
- // @Override
- // public void run() {
- // try {
- // ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- // long mergeStartTime = System.currentTimeMillis();
- // merge();
- // long mergeEndTime = System.currentTimeMillis();
- // long intervalTime = mergeEndTime - mergeStartTime;
- // LOGGER.info(
- // "The filenode processor {} merge start time is {}, "
- // + "merge end time is {}, merge consumes {}ms.",
- // getProcessorName(), ofInstant(Instant.ofEpochMilli(mergeStartTime),
- // zoneId), ofInstant(Instant.ofEpochMilli(mergeEndTime),
- // zoneId), intervalTime);
- // } catch (FileNodeProcessorException e) {
- // LOGGER.error("The filenode processor {} encountered an error when merging.",
- // getProcessorName(), e);
- // throw new ErrorDebugException(e);
- // }
- // }
- // }
- //
- // /**
- // * wait for all closing processors finishing their tasks
- // */
- // public void waitforAllClosed() throws FileNodeProcessorException {
- // close();
- // while (getClosingBufferWriteProcessor().size() != 0) {
- // checkAllClosingProcessors();
- // try {
- // Thread.sleep(10);
- // } catch (InterruptedException e) {
- // LOGGER.error("Filenode Processor {} is interrupted when waiting for all closed.", processorName, e);
- // }
- // }
- // }
- //
- //
- // void checkAllClosingProcessors() {
- // Iterator<BufferWriteProcessor> iterator =
- // this.getClosingBufferWriteProcessor().iterator();
- // while (iterator.hasNext()) {
- // BufferWriteProcessor processor = iterator.next();
- // try {
- // if (processor.getCloseFuture().get(10, TimeUnit.MILLISECONDS)) {
- // //if finished, we can remove it.
- // iterator.remove();
- // }
- // } catch (InterruptedException | ExecutionException e) {
- // LOGGER.error("Close bufferwrite processor {} failed.", processor.getProcessorName(), e);
- // } catch (TimeoutException e) {
- // //do nothing.
- // }
- // }
- // this.getClosingBufferWriteProcessor().clear();
- // }
- //}
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.filenode;
-
-import static java.time.ZonedDateTime.ofInstant;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.Processor;
-import org.apache.iotdb.db.engine.bufferwrite.Action;
-import org.apache.iotdb.db.engine.bufferwrite.ActionException;
-import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
-import org.apache.iotdb.db.engine.bufferwrite.FileNodeConstants;
-import org.apache.iotdb.db.engine.modification.Deletion;
-import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.overflow.io.OverflowProcessor;
-import org.apache.iotdb.db.engine.pool.MergeManager;
-import org.apache.iotdb.db.engine.querycontext.GlobalSortedSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
-import org.apache.iotdb.db.engine.querycontext.OverflowSeriesDataSource;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFile;
-import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
-import org.apache.iotdb.db.engine.version.VersionController;
-import org.apache.iotdb.db.exception.BufferWriteProcessorException;
-import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
-import org.apache.iotdb.db.exception.ErrorDebugException;
-import org.apache.iotdb.db.exception.FileNodeProcessorException;
-import org.apache.iotdb.db.exception.OverflowProcessorException;
-import org.apache.iotdb.db.exception.PathErrorException;
-import org.apache.iotdb.db.exception.ProcessorException;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.monitor.IStatistic;
-import org.apache.iotdb.db.monitor.MonitorConstants;
-import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.query.factory.SeriesReaderFactory;
-import org.apache.iotdb.db.query.reader.IReader;
-import org.apache.iotdb.db.sync.conf.Constans;
-import org.apache.iotdb.db.utils.MemUtils;
-import org.apache.iotdb.db.utils.QueryUtils;
-import org.apache.iotdb.db.utils.TimeValuePair;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.read.filter.TimeFilter;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory;
-import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
-import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileNodeProcessor extends Processor implements IStatistic {
-
- private static final String WARN_NO_SUCH_OVERFLOWED_FILE = "Can not find any tsfile which"
- + " will be overflowed in the filenode processor {}, ";
- private static final String RESTORE_FILE_SUFFIX = ".restore";
- private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeProcessor.class);
- private static final IoTDBConfig TsFileDBConf = IoTDBDescriptor.getInstance().getConfig();
- private static final MManager mManager = MManager.getInstance();
- private static final Directories directories = Directories.getInstance();
- private final String statStorageDeltaName;
- private final HashMap<String, AtomicLong> statParamsHashMap = new HashMap<>();
- /**
- * Used to keep the oldest timestamp for each deviceId. The key is deviceId.
- */
- private volatile boolean isOverflowed;
- private Map<String, Long> lastUpdateTimeMap;
- private Map<String, Long> flushLastUpdateTimeMap;
- private Map<String, List<TsFileResource>> invertedIndexOfFiles;
- private TsFileResource emptyTsFileResource;
- private TsFileResource currentTsFileResource;
- private List<TsFileResource> newFileNodes;
- private FileNodeProcessorStatus isMerging;
-
- /**
- * this is used when work->merge operation
- */
- private int numOfMergeFile;
- private FileNodeProcessorStore fileNodeProcessorStore;
- private String fileNodeRestoreFilePath;
- private final Object fileNodeRestoreLock = new Object();
-
- /**
- * last merge time
- */
- private long lastMergeTime = -1;
- private BufferWriteProcessor bufferWriteProcessor = null;
- private OverflowProcessor overflowProcessor = null;
- private Set<Integer> oldMultiPassTokenSet = null;
- private Set<Integer> newMultiPassTokenSet = new HashSet<>();
-
- /**
- * Represent the number of old queries that have not ended.
- * This parameter only decreases but not increase.
- */
- private CountDownLatch oldMultiPassCount = null;
-
- /**
- * Represent the number of new queries that have not ended.
- */
- private AtomicInteger newMultiPassCount = new AtomicInteger(0);
- /**
- * system recovery
- */
- private boolean shouldRecovery = false;
- /**
- * statistic monitor parameters
- */
- private Map<String, Action> parameters;
- private FileSchema fileSchema;
- private Action flushFileNodeProcessorAction = () -> {
- synchronized (fileNodeProcessorStore) {
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- throw new ActionException(e);
- }
- }
- };
- private Action bufferwriteFlushAction = () -> {
- // update the lastUpdateTime Notice: Thread safe
- synchronized (fileNodeProcessorStore) {
- // deep copy
- Map<String, Long> tempLastUpdateMap = new HashMap<>(lastUpdateTimeMap);
- // update flushLastUpdateTimeMap
- for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- flushLastUpdateTimeMap.put(entry.getKey(), entry.getValue() + 1);
- }
- fileNodeProcessorStore.setLastUpdateTimeMap(tempLastUpdateMap);
- }
- };
-
- private Action bufferwriteCloseAction = new Action() {
-
- @Override
- public void act() {
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setLastUpdateTimeMap(lastUpdateTimeMap);
- addLastTimeToIntervalFile();
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- }
- }
-
- private void addLastTimeToIntervalFile() {
-
- if (!newFileNodes.isEmpty()) {
- // end time with one start time
- Map<String, Long> endTimeMap = new HashMap<>();
- for (Entry<String, Long> startTime : currentTsFileResource.getStartTimeMap().entrySet()) {
- String deviceId = startTime.getKey();
- endTimeMap.put(deviceId, lastUpdateTimeMap.get(deviceId));
- }
- currentTsFileResource.setEndTimeMap(endTimeMap);
- }
- }
- };
- private Action overflowFlushAction = () -> {
-
- // update the new TsFileResource List and emptyIntervalFile.
- // Notice: thread safe
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setOverflowed(isOverflowed);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- }
- };
- // Token for query which used to
- private int multiPassLockToken = 0;
- private VersionController versionController;
- private ReentrantLock mergeDeleteLock = new ReentrantLock();
-
- /**
- * This is the modification file of the result of the current merge.
- */
- private ModificationFile mergingModification;
-
- private TsFileIOWriter mergeFileWriter = null;
- private String mergeOutputPath = null;
- private String mergeBaseDir = null;
- private String mergeFileName = null;
- private boolean mergeIsChunkGroupHasData = false;
- private long mergeStartPos;
-
- /**
- * constructor of FileNodeProcessor.
- */
- FileNodeProcessor(String fileNodeDirPath, String processorName)
- throws FileNodeProcessorException {
- super(processorName);
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
- statParamsHashMap.put(statConstant.name(), new AtomicLong(0));
- }
- statStorageDeltaName =
- MonitorConstants.STAT_STORAGE_GROUP_PREFIX + MonitorConstants.MONITOR_PATH_SEPARATOR
- + MonitorConstants.FILE_NODE_PATH + MonitorConstants.MONITOR_PATH_SEPARATOR
- + processorName.replaceAll("\\.", "_");
-
- this.parameters = new HashMap<>();
- String dirPath = fileNodeDirPath;
- if (dirPath.length() > 0
- && dirPath.charAt(dirPath.length() - 1) != File.separatorChar) {
- dirPath = dirPath + File.separatorChar;
- }
-
- File restoreFolder = new File(dirPath + processorName);
- if (!restoreFolder.exists()) {
- restoreFolder.mkdirs();
- LOGGER.info(
- "The restore directory of the filenode processor {} doesn't exist. Create new " +
- "directory {}",
- getProcessorName(), restoreFolder.getAbsolutePath());
- }
- fileNodeRestoreFilePath = new File(restoreFolder, processorName + RESTORE_FILE_SUFFIX)
- .getPath();
- try {
- fileNodeProcessorStore = readStoreFromDisk();
- } catch (FileNodeProcessorException e) {
- LOGGER.error(
- "The fileNode processor {} encountered an error when recoverying restore " +
- "information.", processorName);
- throw new FileNodeProcessorException(e);
- }
- // TODO deep clone the lastupdate time
- lastUpdateTimeMap = fileNodeProcessorStore.getLastUpdateTimeMap();
- emptyTsFileResource = fileNodeProcessorStore.getEmptyTsFileResource();
- newFileNodes = fileNodeProcessorStore.getNewFileNodes();
- isMerging = fileNodeProcessorStore.getFileNodeProcessorStatus();
- numOfMergeFile = fileNodeProcessorStore.getNumOfMergeFile();
- invertedIndexOfFiles = new HashMap<>();
- // deep clone
- flushLastUpdateTimeMap = new HashMap<>();
- for (Entry<String, Long> entry : lastUpdateTimeMap.entrySet()) {
- flushLastUpdateTimeMap.put(entry.getKey(), entry.getValue() + 1);
- }
- // construct the fileschema
- try {
- this.fileSchema = constructFileSchema(processorName);
- } catch (WriteProcessException e) {
- throw new FileNodeProcessorException(e);
- }
- // status is not NONE, or the last intervalFile is not closed
- if (isMerging != FileNodeProcessorStatus.NONE
- || (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed())) {
- shouldRecovery = true;
- } else {
- // add file into the index of file
- addAllFileIntoIndex(newFileNodes);
- }
- // RegistStatService
- if (TsFileDBConf.isEnableStatMonitor()) {
- StatMonitor statMonitor = StatMonitor.getInstance();
- registerStatMetadata();
- statMonitor.registerStatistics(statStorageDeltaName, this);
- }
- try {
- versionController = new SimpleFileVersionController(restoreFolder.getPath());
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- @Override
- public Map<String, AtomicLong> getStatParamsHashMap() {
- return statParamsHashMap;
- }
-
- @Override
- public void registerStatMetadata() {
- Map<String, String> hashMap = new HashMap<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
- hashMap
- .put(statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name(),
- MonitorConstants.DATA_TYPE_INT64);
- }
- StatMonitor.getInstance().registerStatStorageGroup(hashMap);
- }
-
- @Override
- public List<String> getAllPathForStatistic() {
- List<String> list = new ArrayList<>();
- for (MonitorConstants.FileNodeProcessorStatConstants statConstant :
- MonitorConstants.FileNodeProcessorStatConstants.values()) {
- list.add(
- statStorageDeltaName + MonitorConstants.MONITOR_PATH_SEPARATOR + statConstant.name());
- }
- return list;
- }
-
- @Override
- public Map<String, TSRecord> getAllStatisticsValue() {
- Long curTime = System.currentTimeMillis();
- HashMap<String, TSRecord> tsRecordHashMap = new HashMap<>();
- TSRecord tsRecord = new TSRecord(curTime, statStorageDeltaName);
-
- Map<String, AtomicLong> hashMap = getStatParamsHashMap();
- tsRecord.dataPointList = new ArrayList<>();
- for (Map.Entry<String, AtomicLong> entry : hashMap.entrySet()) {
- tsRecord.dataPointList.add(new LongDataPoint(entry.getKey(), entry.getValue().get()));
- }
-
- tsRecordHashMap.put(statStorageDeltaName, tsRecord);
- return tsRecordHashMap;
- }
-
- /**
- * add interval FileNode.
- */
- void addIntervalFileNode(File file) throws ActionException, IOException {
-
- TsFileResource tsFileResource = new TsFileResource(file, false);
- this.currentTsFileResource = tsFileResource;
- newFileNodes.add(tsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- flushFileNodeProcessorAction.act();
- }
-
- /**
- * set interval filenode start time.
- *
- * @param deviceId device ID
- */
- void setIntervalFileNodeStartTime(String deviceId) {
- if (currentTsFileResource.getStartTime(deviceId) == -1) {
- currentTsFileResource.setStartTime(deviceId, flushLastUpdateTimeMap.get(deviceId));
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- }
- invertedIndexOfFiles.get(deviceId).add(currentTsFileResource);
- }
- }
-
- void setIntervalFileNodeStartTime(String deviceId, long time) {
- if (time != -1) {
- currentTsFileResource.setStartTime(deviceId, time);
- } else {
- currentTsFileResource.removeTime(deviceId);
- invertedIndexOfFiles.get(deviceId).remove(currentTsFileResource);
- }
- }
-
- long getIntervalFileNodeStartTime(String deviceId) {
- return currentTsFileResource.getStartTime(deviceId);
- }
-
- private void addAllFileIntoIndex(List<TsFileResource> fileList) {
- // clear map
- invertedIndexOfFiles.clear();
- // add all file to index
- for (TsFileResource fileNode : fileList) {
- if (fileNode.getStartTimeMap().isEmpty()) {
- continue;
- }
- for (String deviceId : fileNode.getStartTimeMap().keySet()) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.put(deviceId, new ArrayList<>());
- }
- invertedIndexOfFiles.get(deviceId).add(fileNode);
- }
- }
- }
-
- public boolean shouldRecovery() {
- return shouldRecovery;
- }
-
- public boolean isOverflowed() {
- return isOverflowed;
- }
-
- /**
- * if overflow insert, update and delete write into this filenode processor, set
- * <code>isOverflowed</code> to true.
- */
- public void setOverflowed(boolean isOverflowed) {
- if (this.isOverflowed != isOverflowed) {
- this.isOverflowed = isOverflowed;
- }
- }
-
- public FileNodeProcessorStatus getFileNodeProcessorStatus() {
- return isMerging;
- }
-
- /**
- * execute filenode recovery.
- */
- public void fileNodeRecovery() throws FileNodeProcessorException, DiskSpaceInsufficientException {
- // restore bufferwrite
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
- //
- // add the current file
- //
- currentTsFileResource = newFileNodes.get(newFileNodes.size() - 1);
-
- // this bufferwrite file is not close by normal operation
- String damagedFilePath = newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath();
- String[] fileNames = damagedFilePath.split("\\" + File.separator);
- // all information to recovery the damaged file.
- // contains file seriesPath, action parameters and processorName
- parameters.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- parameters.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- parameters
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- String baseDir = directories
- .getTsFileFolder(newFileNodes.get(newFileNodes.size() - 1).getBaseDirIndex());
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "The filenode processor {} will recovery the bufferwrite processor, "
- + "the bufferwrite file is {}",
- getProcessorName(), fileNames[fileNames.length - 1]);
- }
-
- try {
- bufferWriteProcessor = new BufferWriteProcessor(baseDir, getProcessorName(),
- fileNames[fileNames.length - 1], parameters, versionController, fileSchema);
- } catch (BufferWriteProcessorException e) {
- LOGGER.error(
- "The filenode processor {} failed to recovery the bufferwrite processor, "
- + "the last bufferwrite file is {}.",
- getProcessorName(), fileNames[fileNames.length - 1]);
- throw new FileNodeProcessorException(e);
- }
- }
- // restore the overflow processor
- LOGGER.info("The filenode processor {} will recovery the overflow processor.",
- getProcessorName());
- parameters.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
- parameters.put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- try {
- overflowProcessor = new OverflowProcessor(getProcessorName(), parameters, fileSchema,
- versionController);
- } catch (IOException e) {
- LOGGER.error("The filenode processor {} failed to recovery the overflow processor.",
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
-
- shouldRecovery = false;
-
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- // re-merge all file
- // if bufferwrite processor is not null, and close
- LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- getProcessorName(), isMerging);
- merge();
- } else if (isMerging == FileNodeProcessorStatus.WAITING) {
- // unlock
- LOGGER.info("The filenode processor {} is recovering, the filenode status is {}.",
- getProcessorName(), isMerging);
- //writeUnlock();
- switchWaitingToWorking();
- } else {
- //writeUnlock();
- }
- // add file into index of file
- addAllFileIntoIndex(newFileNodes);
- }
-
- /**
- * get buffer write processor by processor name and insert time.
- */
- public BufferWriteProcessor getBufferWriteProcessor(String processorName, long insertTime)
- throws FileNodeProcessorException, DiskSpaceInsufficientException {
- if (bufferWriteProcessor == null) {
- Map<String, Action> params = new HashMap<>();
- params.put(FileNodeConstants.BUFFERWRITE_FLUSH_ACTION, bufferwriteFlushAction);
- params.put(FileNodeConstants.BUFFERWRITE_CLOSE_ACTION, bufferwriteCloseAction);
- params
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- String baseDir = directories.getNextFolderForTsfile();
- LOGGER.info("Allocate folder {} for the new bufferwrite processor.", baseDir);
- // construct processor or restore
- try {
- bufferWriteProcessor = new BufferWriteProcessor(baseDir, processorName,
- insertTime + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
- + System.currentTimeMillis(),
- params, versionController, fileSchema);
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(String
- .format("The filenode processor %s failed to get the bufferwrite processor.",
- processorName), e);
- }
- } else if (bufferWriteProcessor.isClosed()) {
- try {
- bufferWriteProcessor.reopen(insertTime + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR
- + System.currentTimeMillis());
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException("Cannot reopen BufferWriteProcessor", e);
- }
- }
- return bufferWriteProcessor;
- }
-
- /**
- * get overflow processor by processor name.
- */
- public OverflowProcessor getOverflowProcessor(String processorName) throws IOException {
- if (overflowProcessor == null) {
- Map<String, Action> params = new HashMap<>();
- // construct processor or restore
- params.put(FileNodeConstants.OVERFLOW_FLUSH_ACTION, overflowFlushAction);
- params
- .put(FileNodeConstants.FILENODE_PROCESSOR_FLUSH_ACTION, flushFileNodeProcessorAction);
- overflowProcessor = new OverflowProcessor(processorName, params, fileSchema,
- versionController);
- } else if (overflowProcessor.isClosed()) {
- overflowProcessor.reopen();
- }
- return overflowProcessor;
- }
-
- /**
- * get overflow processor.
- */
- public OverflowProcessor getOverflowProcessor() {
- if (overflowProcessor == null || overflowProcessor.isClosed()) {
- LOGGER.error("The overflow processor is null when getting the overflowProcessor");
- }
- return overflowProcessor;
- }
-
- public boolean hasOverflowProcessor() {
- return overflowProcessor != null && !overflowProcessor.isClosed();
- }
-
- public void setBufferwriteProcessroToClosed() {
-
- bufferWriteProcessor = null;
- }
-
- public boolean hasBufferwriteProcessor() {
-
- return bufferWriteProcessor != null;
- }
-
- /**
- * set last update time.
- */
- public void setLastUpdateTime(String deviceId, long timestamp) {
- if (!lastUpdateTimeMap.containsKey(deviceId) || lastUpdateTimeMap.get(deviceId) < timestamp) {
- lastUpdateTimeMap.put(deviceId, timestamp);
- }
- if (timestamp == -1) {
- lastUpdateTimeMap.remove(deviceId);
- }
- }
-
- /**
- * get last update time.
- */
- public long getLastUpdateTime(String deviceId) {
-
- if (lastUpdateTimeMap.containsKey(deviceId)) {
- return lastUpdateTimeMap.get(deviceId);
- } else {
- return -1;
- }
- }
-
- /**
- * get flush last update time.
- */
- public long getFlushLastUpdateTime(String deviceId) {
- if (!flushLastUpdateTimeMap.containsKey(deviceId)) {
- flushLastUpdateTimeMap.put(deviceId, 0L);
- }
- return flushLastUpdateTimeMap.get(deviceId);
- }
-
- public Map<String, Long> getLastUpdateTimeMap() {
- return lastUpdateTimeMap;
- }
-
- /**
- * For insert overflow.
- */
- public void changeTypeToChanged(String deviceId, long timestamp) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{},time:{}]",
- getProcessorName(), deviceId, timestamp);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- changeTypeToChanged(temp.get(index), deviceId);
- }
- }
-
- private void changeTypeToChanged(TsFileResource fileNode, String deviceId) {
- fileNode.changeTypeToChanged(isMerging);
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- fileNode.addMergeChanged(deviceId);
- }
- }
-
- /**
- * For update overflow.
- */
- public void changeTypeToChanged(String deviceId, long startTime, long endTime) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{}, start time:{}, end time:{}]",
- getProcessorName(), deviceId, startTime, endTime);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int left = searchIndexNodeByTimestamp(deviceId, startTime, temp);
- int right = searchIndexNodeByTimestamp(deviceId, endTime, temp);
- for (int i = left; i <= right; i++) {
- changeTypeToChanged(temp.get(i), deviceId);
- }
- }
- }
-
- /**
- * For delete overflow.
- */
- public void changeTypeToChangedForDelete(String deviceId, long timestamp) {
- if (!invertedIndexOfFiles.containsKey(deviceId)) {
- LOGGER.warn(
- WARN_NO_SUCH_OVERFLOWED_FILE
- + "the data is [device:{}, delete time:{}]",
- getProcessorName(), deviceId, timestamp);
- emptyTsFileResource.setStartTime(deviceId, 0L);
- emptyTsFileResource.setEndTime(deviceId, getLastUpdateTime(deviceId));
- emptyTsFileResource.changeTypeToChanged(isMerging);
- } else {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = searchIndexNodeByTimestamp(deviceId, timestamp, temp);
- for (int i = 0; i <= index; i++) {
- temp.get(i).changeTypeToChanged(isMerging);
- if (isMerging == FileNodeProcessorStatus.MERGING_WRITE) {
- temp.get(i).addMergeChanged(deviceId);
- }
- }
- }
- }
-
- /**
- * Search the index of the interval by the timestamp.
- *
- * @return index of interval
- */
- private int searchIndexNodeByTimestamp(String deviceId, long timestamp,
- List<TsFileResource> fileList) {
- int index = 1;
- while (index < fileList.size()) {
- if (timestamp < fileList.get(index).getStartTime(deviceId)) {
- break;
- } else {
- index++;
- }
- }
- return index - 1;
- }
-
- /**
- * add multiple pass lock.
- */
- public int addMultiPassCount() {
- LOGGER.debug("Add MultiPassCount: read lock newMultiPassCount.");
- newMultiPassCount.incrementAndGet();
- while (newMultiPassTokenSet.contains(multiPassLockToken)) {
- multiPassLockToken++;
- }
- newMultiPassTokenSet.add(multiPassLockToken);
- LOGGER.debug("Add multi token:{}, nsPath:{}.", multiPassLockToken, getProcessorName());
- return multiPassLockToken;
- }
-
- /**
- * decrease multiple pass count. TODO: use the return value or remove it.
- */
- public boolean decreaseMultiPassCount(int token) throws FileNodeProcessorException {
- if (newMultiPassTokenSet.contains(token)) {
- int newMultiPassCountValue = newMultiPassCount.decrementAndGet();
- if (newMultiPassCountValue < 0) {
- throw new FileNodeProcessorException(String
- .format("Remove MultiPassCount error, newMultiPassCount:%d", newMultiPassCountValue));
- }
- newMultiPassTokenSet.remove(token);
- LOGGER.debug("Remove multi token:{}, nspath:{}, new set:{}, count:{}", token,
- getProcessorName(),
- newMultiPassTokenSet, newMultiPassCount);
- return true;
- } else if (oldMultiPassTokenSet != null && oldMultiPassTokenSet.contains(token)) {
- // remove token first, then unlock
- oldMultiPassTokenSet.remove(token);
- oldMultiPassCount.countDown();
- long oldMultiPassCountValue = oldMultiPassCount.getCount();
- if (oldMultiPassCountValue < 0) {
- throw new FileNodeProcessorException(String
- .format("Remove MultiPassCount error, oldMultiPassCount:%d", oldMultiPassCountValue));
- }
- LOGGER.debug("Remove multi token:{}, old set:{}, count:{}", token, oldMultiPassTokenSet,
- oldMultiPassCount.getCount());
- return true;
- } else {
- LOGGER.error("remove token error:{},new set:{}, old set:{}", token, newMultiPassTokenSet,
- oldMultiPassTokenSet);
- // should add throw exception
- return false;
- }
- }
-
- /**
- * query data.
- */
- public <T extends Comparable<T>> QueryDataSource query(String deviceId, String measurementId,
- QueryContext context) throws FileNodeProcessorException {
- // query overflow data
- MeasurementSchema mSchema;
- TSDataType dataType;
-
- //mSchema = mManager.getSchemaForOnePath(deviceId + "." + measurementId);
- mSchema = fileSchema.getMeasurementSchema(measurementId);
- dataType = mSchema.getType();
-
- OverflowSeriesDataSource overflowSeriesDataSource;
- try {
- overflowSeriesDataSource = overflowProcessor.query(deviceId, measurementId, dataType,
- mSchema.getProps(), context);
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- // tsfile dataØØ
- List<TsFileResource> bufferwriteDataInFiles = new ArrayList<>();
- for (TsFileResource tsFileResource : newFileNodes) {
- // add the same tsFileResource, but not the same reference
- if (tsFileResource.isClosed()) {
- bufferwriteDataInFiles.add(tsFileResource.backUp());
- }
- }
- Pair<ReadOnlyMemChunk, List<ChunkMetaData>> bufferwritedata = new Pair<>(null, null);
- // bufferwrite data
- UnsealedTsFile unsealedTsFile = null;
-
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()
- && !newFileNodes.get(newFileNodes.size() - 1).getStartTimeMap().isEmpty()) {
- unsealedTsFile = new UnsealedTsFile();
- unsealedTsFile.setFilePath(newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath());
- if (bufferWriteProcessor == null) {
- throw new FileNodeProcessorException(String.format(
- "The last of tsfile %s in filenode processor %s is not closed, "
- + "but the bufferwrite processor is null.",
- newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath(), getProcessorName()));
- }
- try {
- bufferwritedata = bufferWriteProcessor
- .queryBufferWriteData(deviceId, measurementId, dataType, mSchema.getProps());
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
-
- try {
- List<Modification> pathModifications = context.getPathModifications(
- currentTsFileResource.getModFile(), deviceId
- + IoTDBConstant.PATH_SEPARATOR + measurementId
- );
- if (!pathModifications.isEmpty()) {
- QueryUtils.modifyChunkMetaData(bufferwritedata.right, pathModifications);
- }
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
-
- unsealedTsFile.setTimeSeriesChunkMetaDatas(bufferwritedata.right);
- }
- GlobalSortedSeriesDataSource globalSortedSeriesDataSource = new GlobalSortedSeriesDataSource(
- new Path(deviceId + "." + measurementId), bufferwriteDataInFiles, unsealedTsFile,
- bufferwritedata.left);
- return new QueryDataSource(globalSortedSeriesDataSource, overflowSeriesDataSource);
-
- }
-
- /**
- * append one specified tsfile to this filenode processor.
- *
- * @param appendFile the appended tsfile information
- * @param appendFilePath the seriesPath of appended file
- */
- public void appendFile(TsFileResource appendFile, String appendFilePath)
- throws FileNodeProcessorException {
- try {
- if (!appendFile.getFile().getParentFile().exists()) {
- appendFile.getFile().getParentFile().mkdirs();
- }
- // move file
- File originFile = new File(appendFilePath);
- File targetFile = appendFile.getFile();
- if (!originFile.exists()) {
- throw new FileNodeProcessorException(
- String.format("The appended file %s does not exist.", appendFilePath));
- }
- if (targetFile.exists()) {
- throw new FileNodeProcessorException(
- String.format("The appended target file %s already exists.",
- appendFile.getFile().getAbsolutePath()));
- }
- if (!originFile.renameTo(targetFile)) {
- LOGGER.warn("File renaming failed when appending new file. Origin: {}, Target: {}",
- originFile.getPath(), targetFile.getPath());
- }
- // append the new tsfile
- this.newFileNodes.add(appendFile);
- // update the lastUpdateTime
- for (Entry<String, Long> entry : appendFile.getEndTimeMap().entrySet()) {
- lastUpdateTimeMap.put(entry.getKey(), entry.getValue());
- }
- bufferwriteFlushAction.act();
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- // reconstruct the inverted index of the newFileNodes
- flushFileNodeProcessorAction.act();
- addAllFileIntoIndex(newFileNodes);
- } catch (Exception e) {
- LOGGER.error("Failed to append the tsfile {} to filenode processor {}.", appendFile,
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
- }
-
- /**
- * get overlap tsfiles which are conflict with the appendFile.
- *
- * @param appendFile the appended tsfile information
- */
- public List<String> getOverlapFiles(TsFileResource appendFile, String uuid)
- throws FileNodeProcessorException {
- List<String> overlapFiles = new ArrayList<>();
- try {
- for (TsFileResource tsFileResource : newFileNodes) {
- getOverlapFiles(appendFile, tsFileResource, uuid, overlapFiles);
- }
- } catch (IOException e) {
- LOGGER.error("Failed to get overlap tsfiles which conflict with the appendFile.");
- throw new FileNodeProcessorException(e);
- }
- return overlapFiles;
- }
-
- private void getOverlapFiles(TsFileResource appendFile, TsFileResource tsFileResource,
- String uuid, List<String> overlapFiles) throws IOException {
- for (Entry<String, Long> entry : appendFile.getStartTimeMap().entrySet()) {
- if (tsFileResource.getStartTimeMap().containsKey(entry.getKey()) &&
- tsFileResource.getEndTime(entry.getKey()) >= entry.getValue()
- && tsFileResource.getStartTime(entry.getKey()) <= appendFile
- .getEndTime(entry.getKey())) {
- String relativeFilePath =
- Constans.SYNC_SERVER + File.separatorChar + uuid + File.separatorChar
- + Constans.BACK_UP_DIRECTORY_NAME
- + File.separatorChar + tsFileResource.getRelativePath();
- File newFile = new File(
- Directories.getInstance().getTsFileFolder(tsFileResource.getBaseDirIndex()),
- relativeFilePath);
- if (!newFile.getParentFile().exists()) {
- newFile.getParentFile().mkdirs();
- }
- java.nio.file.Path link = FileSystems.getDefault().getPath(newFile.getPath());
- java.nio.file.Path target = FileSystems.getDefault()
- .getPath(tsFileResource.getFile().getAbsolutePath());
- Files.createLink(link, target);
- overlapFiles.add(newFile.getPath());
- break;
- }
- }
- }
-
- /**
- * add time series.
- */
- public void addTimeSeries(String measurementId, TSDataType dataType, TSEncoding encoding,
- CompressionType compressor, Map<String, String> props) {
- fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
- compressor, props));
- }
-
- /**
- * submit the merge task to the <code>MergePool</code>.
- *
- * @return null -can't submit the merge task, because this filenode is not overflowed or it is
- * merging now. Future - submit the merge task successfully.
- */
- Future submitToMerge() {
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- if (lastMergeTime > 0) {
- long thisMergeTime = System.currentTimeMillis();
- long mergeTimeInterval = thisMergeTime - lastMergeTime;
- ZonedDateTime lastDateTime = ofInstant(Instant.ofEpochMilli(lastMergeTime),
- zoneId);
- ZonedDateTime thisDateTime = ofInstant(Instant.ofEpochMilli(thisMergeTime),
- zoneId);
- LOGGER.info(
- "The filenode {} last merge time is {}, this merge time is {}, "
- + "merge time interval is {}s",
- getProcessorName(), lastDateTime, thisDateTime, mergeTimeInterval / 1000);
- }
- lastMergeTime = System.currentTimeMillis();
-
- if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- if (overflowProcessor.getFileSize() < IoTDBDescriptor.getInstance()
- .getConfig().getOverflowFileSizeThreshold()) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "Skip this merge taks submission, because the size{} of overflow processor {} "
- + "does not reaches the threshold {}.",
- MemUtils.bytesCntToStr(overflowProcessor.getFileSize()), getProcessorName(),
- MemUtils.bytesCntToStr(
- IoTDBDescriptor.getInstance().getConfig().getOverflowFileSizeThreshold()));
- }
- return null;
- }
- } else {
- LOGGER.info(
- "Skip this merge taks submission, because the filenode processor {} "
- + "has no overflow processor.",
- getProcessorName());
- return null;
- }
- if (isOverflowed && isMerging == FileNodeProcessorStatus.NONE) {
- Runnable mergeThread;
- mergeThread = new MergeRunnale();
- LOGGER.info("Submit the merge task, the merge filenode is {}", getProcessorName());
- return MergeManager.getInstance().submit(mergeThread);
- } else {
- if (!isOverflowed) {
- LOGGER.info(
- "Skip this merge taks submission, because the filenode processor {} is not " +
- "overflowed.",
- getProcessorName());
- } else {
- LOGGER.warn(
- "Skip this merge task submission, because last merge task is not over yet, "
- + "the merge filenode processor is {}",
- getProcessorName());
- }
- }
- return null;
- }
-
- /**
- * Prepare for merge, close the bufferwrite and overflow.
- */
- private void prepareForMerge() {
- try {
- LOGGER.info("The filenode processor {} prepares for merge, closes the bufferwrite processor",
- getProcessorName());
- closeBufferWrite();
- // try to get overflow processor
- getOverflowProcessor(getProcessorName());
- // must close the overflow processor
- while (!getOverflowProcessor().canBeClosed()) {
- waitForClosing();
- }
- LOGGER.info("The filenode processor {} prepares for merge, closes the overflow processor",
- getProcessorName());
- getOverflowProcessor().close();
- } catch (FileNodeProcessorException | OverflowProcessorException | IOException e) {
- LOGGER.error("The filenode processor {} prepares for merge error.", getProcessorName());
- writeUnlock();
- throw new ErrorDebugException(e);
- }
- }
-
- private void waitForClosing() {
- try {
- LOGGER.info(
- "The filenode processor {} prepares for merge, the overflow {} can't be closed, "
- + "wait 100ms,",
- getProcessorName(), getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Merge this storage group, merge the tsfile data with overflow data.
- */
- public void merge() throws FileNodeProcessorException, DiskSpaceInsufficientException {
- // close bufferwrite and overflow, prepare for merge
- LOGGER.info("The filenode processor {} begins to merge.", getProcessorName());
- prepareForMerge();
- // change status from overflowed to no overflowed
- isOverflowed = false;
- // change status from work to merge
- isMerging = FileNodeProcessorStatus.MERGING_WRITE;
- // check the empty file
- Map<String, Long> startTimeMap = emptyTsFileResource.getStartTimeMap();
- mergeCheckEmptyFile(startTimeMap);
-
- for (TsFileResource tsFileResource : newFileNodes) {
- if (tsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- tsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
-
- addAllFileIntoIndex(newFileNodes);
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setOverflowed(isOverflowed);
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- // flush this filenode information
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- LOGGER.error("The filenode processor {} writes restore information error when merging.",
- getProcessorName(), e);
- writeUnlock();
- throw new FileNodeProcessorException(e);
- }
- }
- // add numOfMergeFile to control the number of the merge file
- List<TsFileResource> backupIntervalFiles;
-
- backupIntervalFiles = switchFileNodeToMerge();
- //
- // clear empty file
- //
- boolean needEmtpy = false;
- if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- needEmtpy = true;
- }
- emptyTsFileResource.clear();
- // attention
- try {
- if (overflowProcessor.isClosed()) {
- overflowProcessor.reopen();
- }
- overflowProcessor.switchWorkToMerge();
- } catch (IOException e) {
- LOGGER.error("The filenode processor {} can't switch overflow processor from work to merge.",
- getProcessorName(), e);
- writeUnlock();
- throw new FileNodeProcessorException(e);
- }
- LOGGER.info("The filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.NONE, FileNodeProcessorStatus.MERGING_WRITE);
- writeUnlock();
-
- // query tsfile data and overflow data, and merge them
- int numOfMergeFiles = 0;
- int allNeedMergeFiles = backupIntervalFiles.size();
- for (TsFileResource backupIntervalFile : backupIntervalFiles) {
- numOfMergeFiles++;
- if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.CHANGED) {
- // query data and merge
- String filePathBeforeMerge = backupIntervalFile.getRelativePath();
- try {
- LOGGER.info(
- "The filenode processor {} begins merging the {}/{} tsfile[{}] with "
- + "overflow file, the process is {}%",
- getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- (int) (((numOfMergeFiles - 1) / (float) allNeedMergeFiles) * 100));
- long startTime = System.currentTimeMillis();
- String newFile = queryAndWriteDataForMerge(backupIntervalFile);
- long endTime = System.currentTimeMillis();
- long timeConsume = endTime - startTime;
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- LOGGER.info(
- "The fileNode processor {} has merged the {}/{} tsfile[{}->{}] over, "
- + "start time of merge is {}, end time of merge is {}, "
- + "time consumption is {}ms,"
- + " the process is {}%",
- getProcessorName(), numOfMergeFiles, allNeedMergeFiles, filePathBeforeMerge,
- newFile, ofInstant(Instant.ofEpochMilli(startTime),
- zoneId), ofInstant(Instant.ofEpochMilli(endTime), zoneId), timeConsume,
- numOfMergeFiles / (float) allNeedMergeFiles * 100);
- } catch (IOException | PathErrorException e) {
- LOGGER.error("Merge: query and write data error.", e);
- throw new FileNodeProcessorException(e);
- }
- } else if (backupIntervalFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- LOGGER.error("The overflowChangeType of backupIntervalFile must not be {}",
- OverflowChangeType.MERGING_CHANGE);
- // handle this error, throw one runtime exception
- throw new FileNodeProcessorException(
- "The overflowChangeType of backupIntervalFile must not be "
- + OverflowChangeType.MERGING_CHANGE);
- } else {
- LOGGER.debug(
- "The filenode processor {} is merging, the interval file {} doesn't "
- + "need to be merged.",
- getProcessorName(), backupIntervalFile.getRelativePath());
- }
- }
-
- // change status from merge to wait
- switchMergeToWaiting(backupIntervalFiles, needEmtpy);
-
- // change status from wait to work
- switchWaitingToWorking();
- }
-
- private void mergeCheckEmptyFile(Map<String, Long> startTimeMap) {
- if (emptyTsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- return;
- }
- Iterator<Entry<String, Long>> iterator = emptyTsFileResource.getEndTimeMap().entrySet()
- .iterator();
- while (iterator.hasNext()) {
- Entry<String, Long> entry = iterator.next();
- String deviceId = entry.getKey();
- if (invertedIndexOfFiles.containsKey(deviceId)) {
- invertedIndexOfFiles.get(deviceId).get(0).setOverflowChangeType(OverflowChangeType.CHANGED);
- startTimeMap.remove(deviceId);
- iterator.remove();
- }
- }
- if (emptyTsFileResource.checkEmpty()) {
- emptyTsFileResource.clear();
- } else {
- if (!newFileNodes.isEmpty()) {
- TsFileResource first = newFileNodes.get(0);
- for (String deviceId : emptyTsFileResource.getStartTimeMap().keySet()) {
- first.setStartTime(deviceId, emptyTsFileResource.getStartTime(deviceId));
- first.setEndTime(deviceId, emptyTsFileResource.getEndTime(deviceId));
- first.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- emptyTsFileResource.clear();
- } else {
- emptyTsFileResource.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
- }
-
- private List<TsFileResource> switchFileNodeToMerge() throws FileNodeProcessorException {
- List<TsFileResource> result = new ArrayList<>();
- if (emptyTsFileResource.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- // add empty
- result.add(emptyTsFileResource.backUp());
- if (!newFileNodes.isEmpty()) {
- throw new FileNodeProcessorException(
- String.format("The status of empty file is %s, but the new file list is not empty",
- emptyTsFileResource.getOverflowChangeType()));
- }
- return result;
- }
- if (newFileNodes.isEmpty()) {
- LOGGER.error("No file was changed when merging, the filenode is {}", getProcessorName());
- throw new FileNodeProcessorException(
- "No file was changed when merging, the filenode is " + getProcessorName());
- }
- for (TsFileResource tsFileResource : newFileNodes) {
- updateFileNode(tsFileResource, result);
- }
- return result;
- }
-
- private void updateFileNode(TsFileResource tsFileResource, List<TsFileResource> result) {
- if (tsFileResource.getOverflowChangeType() == OverflowChangeType.NO_CHANGE) {
- result.add(tsFileResource.backUp());
- } else {
- Map<String, Long> startTimeMap = new HashMap<>();
- Map<String, Long> endTimeMap = new HashMap<>();
- for (String deviceId : tsFileResource.getEndTimeMap().keySet()) {
- List<TsFileResource> temp = invertedIndexOfFiles.get(deviceId);
- int index = temp.indexOf(tsFileResource);
- int size = temp.size();
- // start time
- if (index == 0) {
- startTimeMap.put(deviceId, 0L);
- } else {
- startTimeMap.put(deviceId, tsFileResource.getStartTime(deviceId));
- }
- // end time
- if (index < size - 1) {
- endTimeMap.put(deviceId, temp.get(index + 1).getStartTime(deviceId) - 1);
- } else {
- endTimeMap.put(deviceId, tsFileResource.getEndTime(deviceId));
- }
- }
- TsFileResource node = new TsFileResource(startTimeMap, endTimeMap,
- tsFileResource.getOverflowChangeType(), tsFileResource.getFile());
- result.add(node);
- }
- }
-
- private void switchMergeToWaiting(List<TsFileResource> backupIntervalFiles, boolean needEmpty)
- throws FileNodeProcessorException {
- LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.MERGING_WRITE, FileNodeProcessorStatus.WAITING);
- writeLock();
- try {
- oldMultiPassTokenSet = newMultiPassTokenSet;
- oldMultiPassCount = new CountDownLatch(newMultiPassCount.get());
- newMultiPassTokenSet = new HashSet<>();
- newMultiPassCount = new AtomicInteger(0);
- List<TsFileResource> result = new ArrayList<>();
- int beginIndex = 0;
- if (needEmpty) {
- TsFileResource empty = backupIntervalFiles.get(0);
- if (!empty.checkEmpty()) {
- updateEmpty(empty, result);
- beginIndex++;
- }
- }
- // reconstruct the file index
- addAllFileIntoIndex(backupIntervalFiles);
- // check the merge changed file
- for (int i = beginIndex; i < backupIntervalFiles.size(); i++) {
- TsFileResource newFile = newFileNodes.get(i - beginIndex);
- TsFileResource temp = backupIntervalFiles.get(i);
- if (newFile.getOverflowChangeType() == OverflowChangeType.MERGING_CHANGE) {
- updateMergeChanged(newFile, temp);
- }
- if (!temp.checkEmpty()) {
- result.add(temp);
- }
- }
- // add new file when merge
- for (int i = backupIntervalFiles.size() - beginIndex; i < newFileNodes.size(); i++) {
- TsFileResource fileNode = newFileNodes.get(i);
- if (fileNode.isClosed()) {
- result.add(fileNode.backUp());
- } else {
- result.add(fileNode);
- }
- }
-
- isMerging = FileNodeProcessorStatus.WAITING;
- newFileNodes = result;
- // reconstruct the index
- addAllFileIntoIndex(newFileNodes);
- // clear merge changed
- for (TsFileResource fileNode : newFileNodes) {
- fileNode.clearMergeChanged();
- }
-
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- try {
- writeStoreToDisk(fileNodeProcessorStore);
- } catch (FileNodeProcessorException e) {
- LOGGER.error(
- "Merge: failed to write filenode information to revocery file, the filenode is " +
- "{}.",
- getProcessorName(), e);
- throw new FileNodeProcessorException(
- "Merge: write filenode information to revocery file failed, the filenode is "
- + getProcessorName());
- }
- }
- } finally {
- writeUnlock();
- }
- }
-
- private void updateEmpty(TsFileResource empty, List<TsFileResource> result) {
- for (String deviceId : empty.getStartTimeMap().keySet()) {
- if (invertedIndexOfFiles.containsKey(deviceId)) {
- TsFileResource temp = invertedIndexOfFiles.get(deviceId).get(0);
- if (temp.getMergeChanged().contains(deviceId)) {
- empty.setOverflowChangeType(OverflowChangeType.CHANGED);
- break;
- }
- }
- }
- empty.clearMergeChanged();
- result.add(empty.backUp());
- }
-
- private void updateMergeChanged(TsFileResource newFile, TsFileResource temp) {
- for (String deviceId : newFile.getMergeChanged()) {
- if (temp.getStartTimeMap().containsKey(deviceId)) {
- temp.setOverflowChangeType(OverflowChangeType.CHANGED);
- } else {
- changeTypeToChanged(deviceId, newFile.getStartTime(deviceId),
- newFile.getEndTime(deviceId));
- }
- }
- }
-
-
- private void switchWaitingToWorking()
- throws FileNodeProcessorException {
-
- LOGGER.info("The status of filenode processor {} switches from {} to {}.", getProcessorName(),
- FileNodeProcessorStatus.WAITING, FileNodeProcessorStatus.NONE);
-
- if (oldMultiPassCount != null) {
- LOGGER.info("The old Multiple Pass Token set is {}, the old Multiple Pass Count is {}",
- oldMultiPassTokenSet,
- oldMultiPassCount);
- try {
- oldMultiPassCount.await();
- } catch (InterruptedException e) {
- LOGGER.info(
- "The filenode processor {} encountered an error when it waits for all old queries over.",
- getProcessorName());
- throw new FileNodeProcessorException(e);
- }
- }
-
- try {
- writeLock();
- try {
- // delete the all files which are in the newFileNodes
- // notice: the last restore file of the interval file
-
- List<String> bufferwriteDirPathList = directories.getAllTsFileFolders();
- List<File> bufferwriteDirList = new ArrayList<>();
- collectBufferWriteDirs(bufferwriteDirPathList, bufferwriteDirList);
-
- Set<String> bufferFiles = new HashSet<>();
- collectBufferWriteFiles(bufferFiles);
-
- // add the restore file, if the last file is not closed
- if (!newFileNodes.isEmpty() && !newFileNodes.get(newFileNodes.size() - 1).isClosed()) {
- String bufferFileRestorePath =
- newFileNodes.get(newFileNodes.size() - 1).getFile().getAbsolutePath() + RESTORE_FILE_SUFFIX;
- bufferFiles.add(bufferFileRestorePath);
- }
-
- deleteBufferWriteFiles(bufferwriteDirList, bufferFiles);
-
- // merge switch
- changeFileNodes();
-
- // overflow switch from merge to work
- overflowProcessor.switchMergeToWork();
-
- // write status to file
- isMerging = FileNodeProcessorStatus.NONE;
- synchronized (fileNodeProcessorStore) {
- fileNodeProcessorStore.setFileNodeProcessorStatus(isMerging);
- fileNodeProcessorStore.setNewFileNodes(newFileNodes);
- fileNodeProcessorStore.setEmptyTsFileResource(emptyTsFileResource);
- writeStoreToDisk(fileNodeProcessorStore);
- }
- } catch (IOException e) {
- LOGGER.info(
- "The filenode processor {} encountered an error when its "
- + "status switched from {} to {}.",
- getProcessorName(), FileNodeProcessorStatus.NONE,
- FileNodeProcessorStatus.MERGING_WRITE);
- throw new FileNodeProcessorException(e);
- } finally {
- writeUnlock();
- }
- } finally {
- oldMultiPassTokenSet = null;
- oldMultiPassCount = null;
- }
-
- }
-
- private void collectBufferWriteDirs(List<String> bufferwriteDirPathList,
- List<File> bufferwriteDirList) {
- for (String bufferwriteDirPath : bufferwriteDirPathList) {
- if (bufferwriteDirPath.length() > 0
- && bufferwriteDirPath.charAt(bufferwriteDirPath.length() - 1)
- != File.separatorChar) {
- bufferwriteDirPath = bufferwriteDirPath + File.separatorChar;
- }
- bufferwriteDirPath = bufferwriteDirPath + getProcessorName();
- File bufferwriteDir = new File(bufferwriteDirPath);
- bufferwriteDirList.add(bufferwriteDir);
- if (!bufferwriteDir.exists()) {
- bufferwriteDir.mkdirs();
- }
- }
- }
-
- private void collectBufferWriteFiles(Set<String> bufferFiles) {
- for (TsFileResource bufferFileNode : newFileNodes) {
- String bufferFilePath = bufferFileNode.getFile().getAbsolutePath();
- if (bufferFilePath != null) {
- bufferFiles.add(bufferFilePath);
- }
- }
- }
-
- private void deleteBufferWriteFiles(List<File> bufferwriteDirList, Set<String> bufferFiles)
- throws IOException {
- for (File bufferwriteDir : bufferwriteDirList) {
- File[] files = bufferwriteDir.listFiles();
- if (files == null) {
- continue;
- }
- for (File file : files) {
- if (!bufferFiles.contains(file.getPath())) {
- FileReaderManager.getInstance().closeFileAndRemoveReader(file.getPath());
- if (!file.delete()) {
- LOGGER.warn("Cannot delete BufferWrite file {}", file.getPath());
- }
- }
- }
- }
- }
-
- private void changeFileNodes() {
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getOverflowChangeType() != OverflowChangeType.NO_CHANGE) {
- fileNode.setOverflowChangeType(OverflowChangeType.CHANGED);
- }
- }
- }
-
- private String queryAndWriteDataForMerge(TsFileResource backupIntervalFile)
- throws IOException, FileNodeProcessorException, PathErrorException, DiskSpaceInsufficientException {
- Map<String, Long> startTimeMap = new HashMap<>();
- Map<String, Long> endTimeMap = new HashMap<>();
-
- mergeFileWriter = null;
- mergeOutputPath = null;
- mergeBaseDir = null;
- mergeFileName = null;
- // modifications are blocked before mergeModification is created to avoid
- // losing some modification.
- mergeDeleteLock.lock();
- QueryContext context = new QueryContext();
- try {
- FileReaderManager.getInstance().increaseFileReaderReference(backupIntervalFile.getFilePath(),
- true);
- for (String deviceId : backupIntervalFile.getStartTimeMap().keySet()) {
- // query one deviceId
- List<Path> pathList = new ArrayList<>();
- mergeIsChunkGroupHasData = false;
- mergeStartPos = -1;
- ChunkGroupFooter footer;
- int numOfChunk = 0;
- try {
- List<String> pathStrings = mManager.getLeafNodePathInNextLevel(deviceId);
- for (String string : pathStrings) {
- pathList.add(new Path(string));
- }
- } catch (PathErrorException e) {
- LOGGER.error("Can't get all the paths from MManager, the deviceId is {}", deviceId);
- throw new FileNodeProcessorException(e);
- }
- if (pathList.isEmpty()) {
- continue;
- }
- for (Path path : pathList) {
- // query one measurement in the special deviceId
- String measurementId = path.getMeasurement();
- TSDataType dataType = mManager.getSeriesType(path.getFullPath());
- OverflowSeriesDataSource overflowSeriesDataSource = overflowProcessor.queryMerge(deviceId,
- measurementId, dataType, true, context);
- Filter timeFilter = FilterFactory
- .and(TimeFilter.gtEq(backupIntervalFile.getStartTime(deviceId)),
- TimeFilter.ltEq(backupIntervalFile.getEndTime(deviceId)));
- SingleSeriesExpression seriesFilter = new SingleSeriesExpression(path, timeFilter);
-
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
- FileReaderManager.getInstance()
- .increaseFileReaderReference(overflowInsertFile.getFilePath(),
- false);
- }
-
- IReader seriesReader = SeriesReaderFactory.getInstance()
- .createSeriesReaderForMerge(backupIntervalFile,
- overflowSeriesDataSource, seriesFilter, context);
- numOfChunk += queryAndWriteSeries(seriesReader, path, seriesFilter, dataType,
- startTimeMap, endTimeMap, overflowSeriesDataSource);
- }
- if (mergeIsChunkGroupHasData) {
- // end the new rowGroupMetadata
- long size = mergeFileWriter.getPos() - mergeStartPos;
- footer = new ChunkGroupFooter(deviceId, size, numOfChunk);
- mergeFileWriter.endChunkGroup(footer, 0);
- }
- }
- } finally {
- FileReaderManager.getInstance().decreaseFileReaderReference(backupIntervalFile.getFilePath(),
- true);
-
- if (mergeDeleteLock.isLocked()) {
- mergeDeleteLock.unlock();
- }
- }
-
- if (mergeFileWriter != null) {
- mergeFileWriter.endFile(fileSchema);
- }
- backupIntervalFile.setFile(new File(mergeBaseDir + File.separator + mergeFileName));
- backupIntervalFile.setOverflowChangeType(OverflowChangeType.NO_CHANGE);
- backupIntervalFile.setStartTimeMap(startTimeMap);
- backupIntervalFile.setEndTimeMap(endTimeMap);
- backupIntervalFile.setModFile(mergingModification);
- mergingModification = null;
- return mergeFileName;
- }
-
- private int queryAndWriteSeries(IReader seriesReader, Path path,
- SingleSeriesExpression seriesFilter, TSDataType dataType,
- Map<String, Long> startTimeMap, Map<String, Long> endTimeMap,
- OverflowSeriesDataSource overflowSeriesDataSource)
- throws IOException, DiskSpaceInsufficientException {
- int numOfChunk = 0;
- try {
- if (!seriesReader.hasNext()) {
- LOGGER.debug(
- "The time-series {} has no data with the filter {} in the filenode processor {}",
- path, seriesFilter, getProcessorName());
- } else {
- numOfChunk++;
- TimeValuePair timeValuePair = seriesReader.next();
- if (mergeFileWriter == null) {
- mergeBaseDir = directories.getNextFolderForTsfile();
- mergeFileName = timeValuePair.getTimestamp()
- + FileNodeConstants.BUFFERWRITE_FILE_SEPARATOR + System.currentTimeMillis();
- mergeOutputPath = constructOutputFilePath(mergeBaseDir, getProcessorName(),
- mergeFileName);
- mergeFileName = getProcessorName() + File.separatorChar + mergeFileName;
- mergeFileWriter = new TsFileIOWriter(new File(mergeOutputPath));
- mergingModification = new ModificationFile(mergeOutputPath
- + ModificationFile.FILE_SUFFIX);
- mergeDeleteLock.unlock();
- }
- if (!mergeIsChunkGroupHasData) {
- // start a new rowGroupMetadata
- mergeIsChunkGroupHasData = true;
- // the datasize and numOfChunk is fake
- // the accurate datasize and numOfChunk will get after write all this device data.
- mergeFileWriter.startFlushChunkGroup(path.getDevice());// TODO please check me.
- mergeStartPos = mergeFileWriter.getPos();
- }
- // init the serieswWriteImpl
- MeasurementSchema measurementSchema = fileSchema
- .getMeasurementSchema(path.getMeasurement());
- ChunkBuffer pageWriter = new ChunkBuffer(measurementSchema);
- int pageSizeThreshold = TSFileConfig.pageSizeInByte;
- ChunkWriterImpl seriesWriterImpl = new ChunkWriterImpl(measurementSchema, pageWriter,
- pageSizeThreshold);
- // write the series data
- writeOneSeries(path.getDevice(), seriesWriterImpl, dataType,
- seriesReader,
- startTimeMap, endTimeMap, timeValuePair);
- // flush the series data
- seriesWriterImpl.writeToFileWriter(mergeFileWriter);
- }
- } finally {
- for (OverflowInsertFile overflowInsertFile : overflowSeriesDataSource
- .getOverflowInsertFileList()) {
- FileReaderManager.getInstance()
- .decreaseFileReaderReference(overflowInsertFile.getFilePath(),
- false);
- }
- }
- return numOfChunk;
- }
-
-
- private void writeOneSeries(String deviceId, ChunkWriterImpl seriesWriterImpl,
- TSDataType dataType, IReader seriesReader, Map<String, Long> startTimeMap,
- Map<String, Long> endTimeMap, TimeValuePair firstTVPair) throws IOException {
- long startTime;
- long endTime;
- TimeValuePair localTV = firstTVPair;
- writeTVPair(seriesWriterImpl, dataType, localTV);
- startTime = endTime = localTV.getTimestamp();
- if (!startTimeMap.containsKey(deviceId) || startTimeMap.get(deviceId) > startTime) {
- startTimeMap.put(deviceId, startTime);
- }
- if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- endTimeMap.put(deviceId, endTime);
- }
- while (seriesReader.hasNext()) {
- localTV = seriesReader.next();
- endTime = localTV.getTimestamp();
- writeTVPair(seriesWriterImpl, dataType, localTV);
- }
- if (!endTimeMap.containsKey(deviceId) || endTimeMap.get(deviceId) < endTime) {
- endTimeMap.put(deviceId, endTime);
- }
- }
-
- private void writeTVPair(ChunkWriterImpl seriesWriterImpl, TSDataType dataType,
- TimeValuePair timeValuePair) throws IOException {
- switch (dataType) {
- case BOOLEAN:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
- break;
- case INT32:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
- break;
- case INT64:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
- break;
- case FLOAT:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- break;
- case DOUBLE:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
- break;
- case TEXT:
- seriesWriterImpl.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
- break;
- default:
- LOGGER.error("Not support data type: {}", dataType);
- break;
- }
- }
-
-
- private String constructOutputFilePath(String baseDir, String processorName, String fileName) {
-
- String localBaseDir = baseDir;
- if (localBaseDir.charAt(localBaseDir.length() - 1) != File.separatorChar) {
- localBaseDir = localBaseDir + File.separatorChar + processorName;
- }
- File dataDir = new File(localBaseDir);
- if (!dataDir.exists()) {
- LOGGER.warn("The bufferwrite processor data dir doesn't exists, create new directory {}",
- localBaseDir);
- dataDir.mkdirs();
- }
- File outputFile = new File(dataDir, fileName);
- return outputFile.getPath();
- }
-
- private FileSchema constructFileSchema(String processorName) throws WriteProcessException {
-
- List<MeasurementSchema> columnSchemaList;
- columnSchemaList = mManager.getSchemaForFileName(processorName);
-
- FileSchema schema = new FileSchema();
- for (MeasurementSchema measurementSchema : columnSchemaList) {
- schema.registerMeasurement(measurementSchema);
- }
- return schema;
-
- }
-
- @Override
- public boolean canBeClosed() {
- if (isMerging != FileNodeProcessorStatus.NONE) {
- LOGGER.info("The filenode {} can't be closed, because the filenode status is {}",
- getProcessorName(),
- isMerging);
- return false;
- }
- if (newMultiPassCount.get() != 0) {
- LOGGER.warn("The filenode {} can't be closed, because newMultiPassCount is {}. The newMultiPassTokenSet is {}",
- getProcessorName(), newMultiPassCount, newMultiPassTokenSet);
- return false;
- }
-
- if (oldMultiPassCount == null) {
- return true;
- }
- if (oldMultiPassCount.getCount() == 0) {
- return true;
- } else {
- LOGGER.info("The filenode {} can't be closed, because oldMultiPassCount is {}",
- getProcessorName(), oldMultiPassCount.getCount());
- return false;
- }
- }
-
- @Override
- public FileNodeFlushFuture flush() throws IOException {
- Future<Boolean> bufferWriteFlushFuture = null;
- Future<Boolean> overflowFlushFuture = null;
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- bufferWriteFlushFuture = bufferWriteProcessor.flush();
- }
- if (overflowProcessor != null && !overflowProcessor.isClosed()) {
- overflowFlushFuture = overflowProcessor.flush();
- }
- return new FileNodeFlushFuture(bufferWriteFlushFuture, overflowFlushFuture);
- }
-
- /**
- * Close the bufferwrite processor.
- */
- public void closeBufferWrite() throws FileNodeProcessorException {
- if (bufferWriteProcessor == null || bufferWriteProcessor.isClosed()) {
- return;
- }
- try {
- while (!bufferWriteProcessor.canBeClosed()) {
- waitForBufferWriteClose();
- }
- bufferWriteProcessor.close();
- bufferWriteProcessor = null;
- } catch (BufferWriteProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- private void waitForBufferWriteClose() {
- try {
- LOGGER.info("The bufferwrite {} can't be closed, wait 100ms",
- bufferWriteProcessor.getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption", e);
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Close the overflow processor.
- */
- public void closeOverflow() throws FileNodeProcessorException {
- if (overflowProcessor == null || overflowProcessor.isClosed()) {
- return;
- }
- try {
- while (!overflowProcessor.canBeClosed()) {
- waitForOverflowClose();
- }
- overflowProcessor.close();
- } catch (OverflowProcessorException e) {
- throw new FileNodeProcessorException(e);
- }
- }
-
- private void waitForOverflowClose() {
- try {
- LOGGER.info("The overflow {} can't be closed, wait 100ms",
- overflowProcessor.getProcessorName());
- TimeUnit.MICROSECONDS.sleep(100);
- } catch (InterruptedException e) {
- LOGGER.error("Unexpected interruption", e);
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void close() throws FileNodeProcessorException {
- closeBufferWrite();
- closeOverflow();
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getModFile() != null) {
- try {
- fileNode.getModFile().close();
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
- }
-
- /**
- * deregister the filenode processor.
- */
- public void delete() throws ProcessorException {
- if (TsFileDBConf.isEnableStatMonitor()) {
- // remove the monitor
- LOGGER.info("Deregister the filenode processor: {} from monitor.", getProcessorName());
- StatMonitor.getInstance().deregisterStatistics(statStorageDeltaName);
- }
- closeBufferWrite();
- closeOverflow();
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode.getModFile() != null) {
- try {
- fileNode.getModFile().close();
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
- }
-
- @Override
- public long memoryUsage() {
- long memSize = 0;
- if (bufferWriteProcessor != null) {
- memSize += bufferWriteProcessor.memoryUsage();
- }
- if (overflowProcessor != null) {
- memSize += overflowProcessor.memoryUsage();
- }
- return memSize;
- }
-
- private void writeStoreToDisk(FileNodeProcessorStore fileNodeProcessorStore)
- throws FileNodeProcessorException {
-
- synchronized (fileNodeRestoreLock) {
- try (FileOutputStream fileOutputStream = new FileOutputStream(fileNodeRestoreFilePath)) {
- fileNodeProcessorStore.serialize(fileOutputStream);
- LOGGER.debug("The filenode processor {} writes restore information to the restore file",
- getProcessorName());
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- }
-
- private FileNodeProcessorStore readStoreFromDisk() throws FileNodeProcessorException {
-
- synchronized (fileNodeRestoreLock) {
- File restoreFile = new File(fileNodeRestoreFilePath);
- if (!restoreFile.exists() || restoreFile.length() == 0) {
- try {
- return new FileNodeProcessorStore(false, new HashMap<>(),
- new TsFileResource(null, false),
- new ArrayList<>(), FileNodeProcessorStatus.NONE, 0);
- } catch (IOException e) {
- throw new FileNodeProcessorException(e);
- }
- }
- try (FileInputStream inputStream = new FileInputStream(fileNodeRestoreFilePath)) {
- return FileNodeProcessorStore.deSerialize(inputStream);
- } catch (IOException e) {
- LOGGER
- .error("Failed to deserialize the FileNodeRestoreFile {}, {}", fileNodeRestoreFilePath,
- e);
- throw new FileNodeProcessorException(e);
- }
- }
- }
-
- String getFileNodeRestoreFilePath() {
- return fileNodeRestoreFilePath;
- }
-
- /**
- * Delete data whose timestamp <= 'timestamp' and belong to timeseries deviceId.measurementId.
- *
- * @param deviceId the deviceId of the timeseries to be deleted.
- * @param measurementId the measurementId of the timeseries to be deleted.
- * @param timestamp the delete range is (0, timestamp].
- */
- public void delete(String deviceId, String measurementId, long timestamp) throws IOException {
- // TODO: how to avoid partial deletion?
- mergeDeleteLock.lock();
- long version = versionController.nextVersion();
-
- // record what files are updated so we can roll back them in case of exception
- List<ModificationFile> updatedModFiles = new ArrayList<>();
-
- try {
- String fullPath = deviceId +
- IoTDBConstant.PATH_SEPARATOR + measurementId;
- Deletion deletion = new Deletion(fullPath, version, timestamp);
- if (mergingModification != null) {
- mergingModification.write(deletion);
- updatedModFiles.add(mergingModification);
- }
- deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- // delete data in memory
- OverflowProcessor ofProcessor = getOverflowProcessor(getProcessorName());
- ofProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- }
- } catch (Exception e) {
- // roll back
- for (ModificationFile modFile : updatedModFiles) {
- modFile.abort();
- }
- throw new IOException(e);
- } finally {
- mergeDeleteLock.unlock();
- }
- }
-
- private void deleteBufferWriteFiles(String deviceId, Deletion deletion,
- List<ModificationFile> updatedModFiles) throws IOException {
- if (currentTsFileResource != null && currentTsFileResource.containsDevice(deviceId)) {
- currentTsFileResource.getModFile().write(deletion);
- updatedModFiles.add(currentTsFileResource.getModFile());
- }
- for (TsFileResource fileNode : newFileNodes) {
- if (fileNode != currentTsFileResource && fileNode.containsDevice(deviceId)
- && fileNode.getStartTime(deviceId) <= deletion.getTimestamp()) {
- fileNode.getModFile().write(deletion);
- updatedModFiles.add(fileNode.getModFile());
- }
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in BufferWrite. Only used by WAL recovery.
- */
- public void deleteBufferWrite(String deviceId, String measurementId, long timestamp)
- throws IOException, BufferWriteProcessorException {
- String fullPath = deviceId +
- IoTDBConstant.PATH_SEPARATOR + measurementId;
- long version = versionController.nextVersion();
- Deletion deletion = new Deletion(fullPath, version, timestamp);
-
- List<ModificationFile> updatedModFiles = new ArrayList<>();
- try {
- deleteBufferWriteFiles(deviceId, deletion, updatedModFiles);
- } catch (IOException e) {
- for (ModificationFile modificationFile : updatedModFiles) {
- modificationFile.abort();
- }
- throw e;
- }
- if (bufferWriteProcessor != null && !bufferWriteProcessor.isClosed()) {
- try {
- bufferWriteProcessor.delete(deviceId, measurementId, timestamp);
- } catch (BufferWriteProcessorException e) {
- throw new IOException(e);
- }
- }
- }
-
- /**
- * Similar to delete(), but only deletes data in Overflow. Only used by WAL recovery.
- */
- public void deleteOverflow(String deviceId, String measurementId, long timestamp)
- throws IOException {
- long version = versionController.nextVersion();
-
- OverflowProcessor overflowProcessor = getOverflowProcessor(getProcessorName());
- List<ModificationFile> updatedModFiles = new ArrayList<>();
- try {
- overflowProcessor.delete(deviceId, measurementId, timestamp, version, updatedModFiles);
- } catch (IOException e) {
- for (ModificationFile modificationFile : updatedModFiles) {
- modificationFile.abort();
- }
- throw e;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- FileNodeProcessor that = (FileNodeProcessor) o;
- return isOverflowed == that.isOverflowed &&
- numOfMergeFile == that.numOfMergeFile &&
- lastMergeTime == that.lastMergeTime &&
- shouldRecovery == that.shouldRecovery &&
- multiPassLockToken == that.multiPassLockToken &&
- Objects.equals(statStorageDeltaName, that.statStorageDeltaName) &&
- Objects.equals(statParamsHashMap, that.statParamsHashMap) &&
- Objects.equals(lastUpdateTimeMap, that.lastUpdateTimeMap) &&
- Objects.equals(flushLastUpdateTimeMap, that.flushLastUpdateTimeMap) &&
- Objects.equals(invertedIndexOfFiles, that.invertedIndexOfFiles) &&
- Objects.equals(emptyTsFileResource, that.emptyTsFileResource) &&
- Objects.equals(currentTsFileResource, that.currentTsFileResource) &&
- Objects.equals(newFileNodes, that.newFileNodes) &&
- isMerging == that.isMerging &&
- Objects.equals(fileNodeProcessorStore, that.fileNodeProcessorStore) &&
- Objects.equals(fileNodeRestoreFilePath, that.fileNodeRestoreFilePath) &&
- Objects.equals(bufferWriteProcessor, that.bufferWriteProcessor) &&
- Objects.equals(overflowProcessor, that.overflowProcessor) &&
- Objects.equals(oldMultiPassTokenSet, that.oldMultiPassTokenSet) &&
- Objects.equals(newMultiPassTokenSet, that.newMultiPassTokenSet) &&
- Objects.equals(oldMultiPassCount, that.oldMultiPassCount) &&
- Objects.equals(newMultiPassCount, that.newMultiPassCount) &&
- Objects.equals(parameters, that.parameters) &&
- Objects.equals(fileSchema, that.fileSchema) &&
- Objects.equals(flushFileNodeProcessorAction, that.flushFileNodeProcessorAction) &&
- Objects.equals(bufferwriteFlushAction, that.bufferwriteFlushAction) &&
- Objects.equals(bufferwriteCloseAction, that.bufferwriteCloseAction) &&
- Objects.equals(overflowFlushAction, that.overflowFlushAction);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), statStorageDeltaName, statParamsHashMap, isOverflowed,
- lastUpdateTimeMap, flushLastUpdateTimeMap, invertedIndexOfFiles,
- emptyTsFileResource, currentTsFileResource, newFileNodes, isMerging,
- numOfMergeFile, fileNodeProcessorStore, fileNodeRestoreFilePath,
- lastMergeTime, bufferWriteProcessor, overflowProcessor, oldMultiPassTokenSet,
- newMultiPassTokenSet, oldMultiPassCount, newMultiPassCount, shouldRecovery, parameters,
- fileSchema, flushFileNodeProcessorAction, bufferwriteFlushAction,
- bufferwriteCloseAction, overflowFlushAction, multiPassLockToken);
- }
-
- public class MergeRunnale implements Runnable {
-
- @Override
- public void run() {
- try {
- ZoneId zoneId = IoTDBDescriptor.getInstance().getConfig().getZoneID();
- long mergeStartTime = System.currentTimeMillis();
- writeLock();
- merge();
- long mergeEndTime = System.currentTimeMillis();
- long intervalTime = mergeEndTime - mergeStartTime;
- LOGGER.info(
- "The filenode processor {} merge start time is {}, "
- + "merge end time is {}, merge consumes {}ms.",
- getProcessorName(), ofInstant(Instant.ofEpochMilli(mergeStartTime),
- zoneId), ofInstant(Instant.ofEpochMilli(mergeEndTime),
- zoneId), intervalTime);
- } catch (FileNodeProcessorException | DiskSpaceInsufficientException e) {
- LOGGER.error("The filenode processor {} encountered an error when merging.",
- getProcessorName(), e);
- throw new ErrorDebugException(e);
- }
- }
- }
-}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
index 233620c,0000000..1e94c6c
mode 100644,000000..100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
@@@ -1,381 -1,0 +1,382 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.filenodeV2;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
+import org.apache.iotdb.db.exception.FileNodeManagerException;
++import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.exception.PathErrorException;
+import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.service.IService;
+import org.apache.iotdb.db.service.ServiceType;
+import org.apache.iotdb.db.utils.FilePathUtils;
+import org.apache.iotdb.db.writelog.node.WriteLogNode;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileNodeManagerV2 implements IService {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+ /**
+ * a folder (system/info/ by default) that persist FileNodeProcessorStore classes. Ends with
+ * File.separator Each FileNodeManager will have a subfolder.
+ */
+ private final String baseDir;
+
+ /**
+ * This map is used to manage all filenode processor,<br> the key is filenode name which is
+ * storage group seriesPath.
+ */
+ private final ConcurrentHashMap<String, FileNodeProcessorV2> processorMap = new ConcurrentHashMap<>();
+
+ private static final FileNodeManagerV2 INSTANCE = new FileNodeManagerV2();
+
+ public static FileNodeManagerV2 getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * This set is used to store overflowed filenode name.<br> The overflowed filenode will be merge.
+ */
+ private volatile FileNodeManagerStatus fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+
+ private enum FileNodeManagerStatus {
+ NONE, MERGE, CLOSE
+ }
+
+
+ private FileNodeManagerV2() {
+ baseDir = FilePathUtils.regularizePath(config.getFileNodeDir());
+ // create baseDir
+ File dir = new File(baseDir);
+ if (dir.mkdirs()) {
+ LOGGER.info("Base directory {} of all storage groups doesn't exist, create it", dir.getPath());
+ }
+
+ /**
+ * recovery all file node processors.
+ */
+ try {
+ List<String> storageGroups = MManager.getInstance().getAllFileNames();
+ for (String storageGroup: storageGroups) {
+ FileNodeProcessorV2 processor = new FileNodeProcessorV2(baseDir, storageGroup);
+ LOGGER.info("FileNodeProcessor {} is recovered successfully", storageGroup);
+ processorMap.put(storageGroup, processor);
+ }
- } catch (PathErrorException | ProcessorException e) {
++ } catch (ProcessorException | MetadataErrorException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void start() throws StartupException {
+
+ }
+
+ @Override
+ public void stop() {
+ syncCloseAllProcessor();
+ }
+
+ @Override
+ public ServiceType getID() {
+ return ServiceType.FILE_NODE_SERVICE;
+ }
+
+
+ private FileNodeProcessorV2 getProcessor(String devicePath)
+ throws FileNodeManagerException {
+ String storageGroup = "";
+ try {
+ // return the storage group name
+ storageGroup = MManager.getInstance().getFileNameByPath(devicePath);
+ FileNodeProcessorV2 processor;
+ processor = processorMap.get(storageGroup);
+ if (processor == null) {
+ storageGroup = storageGroup.intern();
+ synchronized (storageGroup) {
+ processor = processorMap.get(storageGroup);
+ if (processor == null) {
+ LOGGER.debug("construct a processor instance, the storage group is {}, Thread is {}",
+ storageGroup, Thread.currentThread().getId());
+ processor = new FileNodeProcessorV2(baseDir, storageGroup);
+ synchronized (processorMap) {
+ processorMap.put(storageGroup, processor);
+ }
+ }
+ }
+ }
+ return processor;
+ } catch (PathErrorException | ProcessorException e) {
+ LOGGER.error("Fail to get FileNodeProcessor {}", storageGroup, e);
+ throw new FileNodeManagerException(e);
+ }
+ }
+
+
+ /**
+ * This function is just for unit test.
+ */
+ public synchronized void resetFileNodeManager() {
+ processorMap.clear();
+ }
+
+
+ /**
+ * insert TsRecord into storage group.
+ *
+ * @param insertPlan physical plan of insertion
+ * @return an int value represents the insert type, 0: failed; 1: overflow; 2: bufferwrite
+ */
+ public boolean insert(InsertPlan insertPlan) throws FileNodeManagerException {
+
+ FileNodeProcessorV2 fileNodeProcessor;
+ try {
+ fileNodeProcessor = getProcessor(insertPlan.getDeviceId());
+ } catch (Exception e) {
+ LOGGER.warn("get FileNodeProcessor of device {} failed, because {}", insertPlan.getDeviceId(),
+ e.getMessage(), e);
+ throw new FileNodeManagerException(e);
+ }
+
+ // TODO monitor: update statistics
+ return fileNodeProcessor.insert(insertPlan);
+ }
+
+
+ public void asyncFlushAndSealAllFiles() {
+ synchronized (processorMap) {
+ for (FileNodeProcessorV2 fileNodeProcessor: processorMap.values()) {
+ fileNodeProcessor.asyncForceClose();
+ }
+ }
+ }
+
+ private void writeLog(TSRecord tsRecord, boolean isMonitor, WriteLogNode logNode)
+ throws FileNodeManagerException {
+ // TODO
+ }
+
+
+ /**
+ * update data.
+ */
+ public void update(String deviceId, String measurementId, long startTime, long endTime,
+ TSDataType type, String v) {
+ // TODO
+ }
+
+ /**
+ * delete data.
+ */
+ public void delete(String deviceId, String measurementId, long timestamp)
+ throws FileNodeManagerException {
+ FileNodeProcessorV2 fileNodeProcessor = getProcessor(deviceId);
+ try {
+ fileNodeProcessor.delete(deviceId, measurementId, timestamp);
+ } catch (IOException e) {
+ throw new FileNodeManagerException(e);
+ }
+ }
+
+ private void delete(String processorName,
+ Iterator<Entry<String, FileNodeProcessorV2>> processorIterator)
+ throws FileNodeManagerException {
+ // TODO
+ }
+
+
+ /**
+ * begin query.
+ *
+ * @param deviceId queried deviceId
+ * @return a query token for the device.
+ */
+ public int beginQuery(String deviceId) throws FileNodeManagerException {
+ // TODO
+ return -1;
+ }
+
+ /**
+ * end query.
+ */
+ public void endQuery(String deviceId, int token) throws FileNodeManagerException {
+ // TODO
+ }
+
+ /**
+ * query data.
+ */
+ public QueryDataSourceV2 query(SingleSeriesExpression seriesExpression, QueryContext context)
+ throws FileNodeManagerException {
+ String deviceId = seriesExpression.getSeriesPath().getDevice();
+ String measurementId = seriesExpression.getSeriesPath().getMeasurement();
+ FileNodeProcessorV2 fileNodeProcessor = getProcessor(deviceId);
+ return fileNodeProcessor.query(deviceId, measurementId);
+ }
+
+ /**
+ * Append one specified tsfile to the storage group. <b>This method is only provided for
+ * transmission module</b>
+ *
+ * @param fileNodeName the seriesPath of storage group
+ * @param appendFile the appended tsfile information
+ */
+ public boolean appendFileToFileNode(String fileNodeName, TsFileResourceV2 appendFile,
+ String appendFilePath) throws FileNodeManagerException {
+ // TODO
+ return true;
+ }
+
+ /**
+ * get all overlap tsfiles which are conflict with the appendFile.
+ *
+ * @param fileNodeName the seriesPath of storage group
+ * @param appendFile the appended tsfile information
+ */
+ public List<String> getOverlapFilesFromFileNode(String fileNodeName, TsFileResourceV2 appendFile,
+ String uuid) throws FileNodeManagerException {
+ // TODO
+ return null;
+ }
+
+
+ /**
+ * merge all overflowed filenode.
+ *
+ * @throws FileNodeManagerException FileNodeManagerException
+ */
+ public void mergeAll() throws FileNodeManagerException {
+ // TODO
+ }
+
+ /**
+ * try to close the filenode processor. The name of filenode processor is processorName
+ */
+ private boolean tryToCloseFileNodeProcessor(String processorName) throws FileNodeManagerException {
+ // TODO
+ return false;
+ }
+
+ /**
+ * Force to close the filenode processor.
+ */
+ public void deleteOneFileNode(String processorName) throws FileNodeManagerException {
+ if (fileNodeManagerStatus != FileNodeManagerStatus.NONE) {
+ return;
+ }
+
+ fileNodeManagerStatus = FileNodeManagerStatus.CLOSE;
+ try {
+ if (processorMap.containsKey(processorName)) {
+ deleteFileNodeBlocked(processorName);
+ }
+ } catch (IOException e) {
+ LOGGER.error("Delete the filenode processor {} error.", processorName, e);
+ throw new FileNodeManagerException(e);
+ } finally {
+ fileNodeManagerStatus = FileNodeManagerStatus.NONE;
+ }
+ }
+
+ private void deleteFileNodeBlocked(String processorName) throws IOException {
+ LOGGER.info("Forced to delete the filenode processor {}", processorName);
+ FileNodeProcessorV2 processor = processorMap.get(processorName);
+ processor.syncCloseAndStopFileNode(() -> {
+ try {
+ // delete storage group data file
+ for (String tsfilePath: DirectoryManager.getInstance().getAllTsFileFolders()) {
+ File storageGroupFolder = new File(tsfilePath, processorName);
+ if (storageGroupFolder.exists()) {
+ FileUtils.deleteDirectory(storageGroupFolder);
+ }
+ }
+ // delete storage group info file
+ String fileNodePath = IoTDBDescriptor.getInstance().getConfig().getFileNodeDir();
+ fileNodePath = FilePathUtils.regularizePath(fileNodePath) + processorName;
+ FileUtils.deleteDirectory(new File(fileNodePath));
+ } catch (IOException e) {
+ LOGGER.error("Delete tsfiles failed", e);
+ }
+ synchronized (processorMap) {
+ processorMap.remove(processorName);
+ }
+ return true;
+ });
+ }
+
+
+ /**
+ * add time series.
+ */
+ public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
+ CompressionType compressor,
+ Map<String, String> props) throws FileNodeManagerException {
+ FileNodeProcessorV2 fileNodeProcessor = getProcessor(path.getFullPath());
+ fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
+ }
+
+
+ /**
+ * delete all filenode.
+ */
+ public synchronized boolean deleteAll() {
+ LOGGER.info("Start deleting all filenode");
+ // TODO
+ return true;
+ }
+
+ /**
+ * flush command
+ * Sync asyncCloseOneProcessor all file node processors.
+ */
+ public void syncCloseAllProcessor() {
+ LOGGER.info("Start closing all filenode processor");
+ synchronized (processorMap){
+ for(FileNodeProcessorV2 processor: processorMap.values()){
+ processor.syncCloseFileNode();
+ }
+ }
+ }
+
+}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index 4236d3e,0000000..453073b
mode 100755,000000..100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@@ -1,638 -1,0 +1,645 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.filenodeV2;
+
+import static org.apache.iotdb.tsfile.common.constant.SystemConstant.TSFILE_SUFFIX;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Supplier;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
+import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
+import org.apache.iotdb.db.engine.version.VersionController;
++import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
+import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.exception.UnsealedTsFileProcessorException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.utils.datastructure.TVListAllocator;
+import org.apache.iotdb.db.writelog.recover.TsFileRecoverPerformer;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileNodeProcessorV2 {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(FileNodeProcessorV2.class);
+
+ private FileSchema fileSchema;
+
+ // includes sealed and unsealed sequence tsfiles
+ private List<TsFileResourceV2> sequenceFileList = new ArrayList<>();
+ private UnsealedTsFileProcessorV2 workSequenceTsFileProcessor = null;
+ private CopyOnReadLinkedList<UnsealedTsFileProcessorV2> closingSequenceTsFileProcessor = new CopyOnReadLinkedList<>();
+
+ // includes sealed and unsealed unSequnce tsfiles
+ private List<TsFileResourceV2> unSequenceFileList = new ArrayList<>();
+ private UnsealedTsFileProcessorV2 workUnSequenceTsFileProcessor = null;
+ private CopyOnReadLinkedList<UnsealedTsFileProcessorV2> closingUnSequenceTsFileProcessor = new CopyOnReadLinkedList<>();
+
+ /**
+ * device -> global latest timestamp of each device
+ */
+ private Map<String, Long> latestTimeForEachDevice = new HashMap<>();
+
+ /**
+ * device -> largest timestamp of the latest memtable to be submitted to asyncFlush
+ */
+ private Map<String, Long> latestFlushedTimeForEachDevice = new HashMap<>();
+
+ private String storageGroupName;
+
+ private final ReadWriteLock insertLock = new ReentrantReadWriteLock();
+
+ private final Object closeFileNodeCondition = new Object();
+
+ private final ThreadLocal<Long> timerr = new ThreadLocal<>();
+
+ private final ReadWriteLock closeQueryLock = new ReentrantReadWriteLock();
+ /**
+ * Mark whether to close file node
+ */
+ private volatile boolean toBeClosed;
+
+ private VersionController versionController;
+
+ private ReentrantLock mergeDeleteLock = new ReentrantLock();
+
+ /**
+ * This is the modification file of the result of the current merge.
+ */
+ private ModificationFile mergingModification;
+
+
+ public FileNodeProcessorV2(String baseDir, String storageGroupName) throws ProcessorException {
+ this.storageGroupName = storageGroupName;
+
+ // construct the file schema
+ this.fileSchema = constructFileSchema(storageGroupName);
+
+ /**
+ * version controller
+ */
+ try {
+ File storageGroupInfoDir = new File(baseDir, storageGroupName);
+ if (storageGroupInfoDir.mkdirs()) {
+ LOGGER.info("Storage Group Info Directory {} doesn't exist, create it",
+ storageGroupInfoDir.getPath());
+ }
+
+ versionController = new SimpleFileVersionController(
+ storageGroupInfoDir.getPath());
+ } catch (IOException e) {
+ throw new FileNodeProcessorException(e);
+ }
+
+ recover();
+ }
+
+ private void recover() throws ProcessorException {
+ LOGGER.info("recover FileNodeProcessor {}", storageGroupName);
+ List<File> tsFiles = new ArrayList<>();
+ List<String> seqFileFolders = DirectoryManager.getInstance().getAllTsFileFolders();
+ for (String baseDir : seqFileFolders) {
+ File fileFolder = new File(baseDir, storageGroupName);
+ if (!fileFolder.exists()) {
+ continue;
+ }
+ for (File tsfile : fileFolder.listFiles(file -> file.getName().endsWith(TSFILE_SUFFIX))) {
+ tsFiles.add(tsfile);
+ }
+ }
+ recoverSeqFiles(tsFiles);
+
+ tsFiles.clear();
+ List<String> unseqFileFolder = DirectoryManager.getInstance().getAllOverflowFileFolders();
+ for (String baseDir : unseqFileFolder) {
+ File fileFolder = new File(baseDir, storageGroupName);
+ if (!fileFolder.exists()) {
+ continue;
+ }
+ for (File tsfile : fileFolder.listFiles(file -> file.getName().endsWith(TSFILE_SUFFIX))) {
+ tsFiles.add(tsfile);
+ }
+ }
+ recoverUnseqFiles(tsFiles);
+
+ for (TsFileResourceV2 resource : sequenceFileList) {
+ latestTimeForEachDevice.putAll(resource.getEndTimeMap());
+ latestFlushedTimeForEachDevice.putAll(resource.getEndTimeMap());
+ }
+ }
+
+ private void recoverSeqFiles(List<File> tsfiles) throws ProcessorException {
+ tsfiles.sort(new CompareFileName());
+ for (File tsfile : tsfiles) {
+ TsFileResourceV2 tsFileResource = new TsFileResourceV2(tsfile);
+ sequenceFileList.add(tsFileResource);
+ TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-"
+ , fileSchema, versionController, tsFileResource, false);
+ recoverPerformer.recover();
+ }
+ }
+
+ private void recoverUnseqFiles(List<File> tsfiles) throws ProcessorException {
+ tsfiles.sort(new CompareFileName());
+ for (File tsfile : tsfiles) {
+ TsFileResourceV2 tsFileResource = new TsFileResourceV2(tsfile);
+ unSequenceFileList.add(tsFileResource);
+ TsFileRecoverPerformer recoverPerformer = new TsFileRecoverPerformer(storageGroupName + "-",
+ fileSchema,
+ versionController, tsFileResource, true);
+ recoverPerformer.recover();
+ }
+ }
+
+ class CompareFileName implements Comparator<File> {
+
+ @Override
+ public int compare(File o1, File o2) {
+ String[] items1 = o1.getName().split("-");
+ String[] items2 = o2.getName().split("-");
+ if (Long.valueOf(items1[0]) - Long.valueOf(items2[0]) == 0) {
+ return Long.compare(Long.valueOf(items1[1]), Long.valueOf(items2[1]));
+ } else {
+ return Long.compare(Long.valueOf(items1[0]), Long.valueOf(items2[0]));
+ }
+ }
+ }
+
+ private FileSchema constructFileSchema(String storageGroupName) {
+ List<MeasurementSchema> columnSchemaList;
+ columnSchemaList = MManager.getInstance().getSchemaForFileName(storageGroupName);
+
+ FileSchema schema = new FileSchema();
+ for (MeasurementSchema measurementSchema : columnSchemaList) {
+ schema.registerMeasurement(measurementSchema);
+ }
+ return schema;
+
+ }
+
+
+ /**
+ * add time series.
+ */
+ public void addTimeSeries(String measurementId, TSDataType dataType, TSEncoding encoding,
+ CompressionType compressor, Map<String, String> props) {
+ writeLock();
+ try {
+ fileSchema.registerMeasurement(new MeasurementSchema(measurementId, dataType, encoding,
+ compressor, props));
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * @return -1: failed, 1: Overflow, 2:Bufferwrite
+ */
+ public boolean insert(InsertPlan insertPlan) {
+ writeLock();
+ try {
+ if (toBeClosed) {
+ throw new FileNodeProcessorException(
+ "storage group " + storageGroupName + " is to be closed, this insertion is rejected");
+ }
+ // init map
+ latestTimeForEachDevice.putIfAbsent(insertPlan.getDeviceId(), Long.MIN_VALUE);
+ latestFlushedTimeForEachDevice.putIfAbsent(insertPlan.getDeviceId(), Long.MIN_VALUE);
+
+ // insert to sequence or unSequence file
+ if (insertPlan.getTime() > latestFlushedTimeForEachDevice.get(insertPlan.getDeviceId())) {
+ return insertUnsealedDataFile(insertPlan, true);
+ } else {
+ return insertUnsealedDataFile(insertPlan, false);
+ }
+ } catch (FileNodeProcessorException | IOException e) {
+ LOGGER.error("insert tsRecord to unsealed data file failed, because {}", e.getMessage(), e);
+ return false;
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ private boolean insertUnsealedDataFile(InsertPlan insertPlan, boolean sequence)
+ throws IOException {
- UnsealedTsFileProcessorV2 unsealedTsFileProcessor;
++ UnsealedTsFileProcessorV2 unsealedTsFileProcessor = null;
+ boolean result;
+
+ // create a new BufferWriteProcessor
- if (sequence) {
- if (workSequenceTsFileProcessor == null) {
- workSequenceTsFileProcessor = createTsFileProcessor(true);
- sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
- }
- unsealedTsFileProcessor = workSequenceTsFileProcessor;
- } else {
- if (workUnSequenceTsFileProcessor == null) {
- workUnSequenceTsFileProcessor = createTsFileProcessor(false);
- unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
++ try {
++ if (sequence) {
++ if (workSequenceTsFileProcessor == null) {
++ workSequenceTsFileProcessor = createTsFileProcessor(true);
++ sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
++ }
++ unsealedTsFileProcessor = workSequenceTsFileProcessor;
++ } else {
++ if (workUnSequenceTsFileProcessor == null) {
++ workUnSequenceTsFileProcessor = createTsFileProcessor(false);
++ unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
++ }
++ unsealedTsFileProcessor = workUnSequenceTsFileProcessor;
+ }
- unsealedTsFileProcessor = workUnSequenceTsFileProcessor;
++ } catch (DiskSpaceInsufficientException e) {
++ //TODO handle disk full exception
++ LOGGER.error("dis space is insufficient", e);
+ }
+
+ // insert BufferWrite
+ result = unsealedTsFileProcessor.insert(insertPlan, sequence);
+
+ // try to update the latest time of the device of this tsRecord
+ if (result && latestTimeForEachDevice.get(insertPlan.getDeviceId()) < insertPlan.getTime()) {
+ latestTimeForEachDevice.put(insertPlan.getDeviceId(), insertPlan.getTime());
+ }
+
+ // check memtable size and may asyncFlush the workMemtable
+ if (unsealedTsFileProcessor.shouldFlush()) {
+
+ LOGGER.info("The memtable size {} reaches the threshold, async flush it to tsfile: {}",
+ unsealedTsFileProcessor.getWorkMemTableMemory(),
+ unsealedTsFileProcessor.getTsFileResource().getFile().getAbsolutePath());
+
+ if (unsealedTsFileProcessor.shouldClose()) {
+ asyncCloseTsFileProcessor(unsealedTsFileProcessor, sequence);
+ } else {
+ unsealedTsFileProcessor.asyncFlush();
+ }
+ }
+ return result;
+ }
+
- private UnsealedTsFileProcessorV2 createTsFileProcessor(boolean sequence) throws IOException {
++ private UnsealedTsFileProcessorV2 createTsFileProcessor(boolean sequence)
++ throws IOException, DiskSpaceInsufficientException {
+ String baseDir;
+ long start = System.currentTimeMillis();
+ if (sequence) {
+ baseDir = DirectoryManager.getInstance().getNextFolderForSequenceFile();
+ } else {
+ baseDir = DirectoryManager.getInstance().getNextFolderForUnSequenceFile();
+ }
+ start = System.currentTimeMillis() - start;
+ LOGGER.info("getNextFolder in Directory manager cost: {}", start);
+ new File(baseDir, storageGroupName).mkdirs();
+
+ String filePath = Paths.get(baseDir, storageGroupName,
+ System.currentTimeMillis() + "-" + versionController.nextVersion()).toString()
+ + TSFILE_SUFFIX;
+
+ if (sequence) {
+ return new UnsealedTsFileProcessorV2(storageGroupName, new File(filePath),
+ fileSchema, versionController, this::closeUnsealedTsFileProcessorCallback,
+ this::updateLatestFlushTimeCallback);
+ } else {
+ return new UnsealedTsFileProcessorV2(storageGroupName, new File(filePath),
+ fileSchema, versionController, this::closeUnsealedTsFileProcessorCallback,
+ () -> true);
+ }
+ }
+
+ /**
+ * only called by insert(), thread-safety should be ensured by caller
+ */
+ private void asyncCloseTsFileProcessor(UnsealedTsFileProcessorV2 unsealedTsFileProcessor,
+ boolean sequence) {
+
+ // check file size and may close the BufferWrite
+ if (sequence) {
+ closingSequenceTsFileProcessor.add(unsealedTsFileProcessor);
+ workSequenceTsFileProcessor = null;
+ updateEndTimeMap(unsealedTsFileProcessor);
+ } else {
+ closingUnSequenceTsFileProcessor.add(unsealedTsFileProcessor);
+ workUnSequenceTsFileProcessor = null;
+ }
+
+ // async close tsfile
+ unsealedTsFileProcessor.asyncClose();
+
+ LOGGER.info("The file size {} reaches the threshold, async close tsfile: {}.",
+ unsealedTsFileProcessor.getTsFileResource().getFileSize(),
+ unsealedTsFileProcessor.getTsFileResource().getFile().getAbsolutePath());
+ }
+
+
+ // TODO need a read lock, please consider the concurrency with flush manager threads.
+ public QueryDataSourceV2 query(String deviceId, String measurementId) {
+ insertLock.readLock().lock();
+ try {
+ List<TsFileResourceV2> seqResources = getFileReSourceListForQuery(sequenceFileList,
+ deviceId, measurementId);
+ List<TsFileResourceV2> unseqResources = getFileReSourceListForQuery(unSequenceFileList,
+ deviceId, measurementId);
+ return new QueryDataSourceV2(new Path(deviceId, measurementId), seqResources, unseqResources);
+ } finally {
+ insertLock.readLock().unlock();
+ }
+ }
+
+ private void writeLock() {
+ insertLock.writeLock().lock();
+ }
+
+ private void writeUnlock() {
+ insertLock.writeLock().unlock();
+ }
+
+
+ /**
+ * @param tsFileResources includes sealed and unsealed tsfile resources
+ * @return fill unsealed tsfile resources with memory data and ChunkMetadataList of data in disk
+ */
+ private List<TsFileResourceV2> getFileReSourceListForQuery(List<TsFileResourceV2> tsFileResources,
+ String deviceId, String measurementId) {
+
+ MeasurementSchema mSchema = fileSchema.getMeasurementSchema(measurementId);
+ TSDataType dataType = mSchema.getType();
+
+ List<TsFileResourceV2> tsfileResourcesForQuery = new ArrayList<>();
+ for (TsFileResourceV2 tsFileResource : tsFileResources) {
+ if (!tsFileResource.containsDevice(deviceId)) {
+ continue;
+ }
+ if (!tsFileResource.getStartTimeMap().isEmpty()) {
+ closeQueryLock.readLock().lock();
+ try {
+ if (tsFileResource.isClosed()) {
+ tsfileResourcesForQuery.add(tsFileResource);
+ } else {
+ Pair<ReadOnlyMemChunk, List<ChunkMetaData>> pair;
+ pair = tsFileResource
+ .getUnsealedFileProcessor()
+ .query(deviceId, measurementId, dataType, mSchema.getProps());
+ tsfileResourcesForQuery
+ .add(new TsFileResourceV2(tsFileResource.getFile(),
+ tsFileResource.getStartTimeMap(),
+ tsFileResource.getEndTimeMap(), pair.left, pair.right));
+ }
+ } finally {
+ closeQueryLock.readLock().unlock();
+ }
+ }
+ }
+ return tsfileResourcesForQuery;
+ }
+
+
+ /**
+ * Delete data whose timestamp <= 'timestamp' and belong to timeseries deviceId.measurementId.
+ *
+ * @param deviceId the deviceId of the timeseries to be deleted.
+ * @param measurementId the measurementId of the timeseries to be deleted.
+ * @param timestamp the delete range is (0, timestamp].
+ */
+ public void delete(String deviceId, String measurementId, long timestamp) throws IOException {
+ // TODO: how to avoid partial deletion?
+ writeLock();
+
+ // record what files are updated so we can roll back them in case of exception
+ List<ModificationFile> updatedModFiles = new ArrayList<>();
+
+ try {
+ Long lastUpdateTime = latestTimeForEachDevice.get(deviceId);
+ // no tsfile data, the delete operation is invalid
+ if (lastUpdateTime == null || lastUpdateTime == Long.MIN_VALUE) {
+ LOGGER.debug("No device {} in SG {}, deletion invalid", deviceId, storageGroupName);
+ return;
+ }
+
+ // write log
+ if (IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
+ if (workSequenceTsFileProcessor != null) {
+ workSequenceTsFileProcessor.getLogNode()
+ .write(new DeletePlan(timestamp, new Path(deviceId, measurementId)));
+ }
+ if (workUnSequenceTsFileProcessor != null) {
+ workUnSequenceTsFileProcessor.getLogNode()
+ .write(new DeletePlan(timestamp, new Path(deviceId, measurementId)));
+ }
+ }
+
+ Path fullPath = new Path(deviceId, measurementId);
+ Deletion deletion = new Deletion(fullPath, versionController.nextVersion(), timestamp);
+ if (mergingModification != null) {
+ mergingModification.write(deletion);
+ updatedModFiles.add(mergingModification);
+ }
+
+ deleteFiles(sequenceFileList, deletion, updatedModFiles);
+ deleteFiles(unSequenceFileList, deletion, updatedModFiles);
+
+ } catch (Exception e) {
+ // roll back
+ for (ModificationFile modFile : updatedModFiles) {
+ modFile.abort();
+ }
+ throw new IOException(e);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+
+ private void deleteFiles(List<TsFileResourceV2> tsFileResourceList, Deletion deletion,
+ List<ModificationFile> updatedModFiles)
+ throws IOException {
+ String deviceId = deletion.getDevice();
+ for (TsFileResourceV2 tsFileResource : tsFileResourceList) {
+ if (!tsFileResource.containsDevice(deviceId) ||
+ deletion.getTimestamp() < tsFileResource.getStartTimeMap().get(deviceId)) {
+ continue;
+ }
+
+ // write deletion into modification file
+ tsFileResource.getModFile().write(deletion);
+
+ // delete data in memory of unsealed file
+ if (!tsFileResource.isClosed()) {
+ UnsealedTsFileProcessorV2 tsfileProcessor = tsFileResource.getUnsealedFileProcessor();
+ tsfileProcessor.delete(deletion);
+ }
+
+ // add a record in case of rollback
+ updatedModFiles.add(tsFileResource.getModFile());
+ }
+ }
+
+
+ public void asyncForceClose() {
+ writeLock();
+ LOGGER.info("async force close all file in storage group: {}", storageGroupName);
+ try {
+ if (workSequenceTsFileProcessor != null) {
+ closingSequenceTsFileProcessor.add(workSequenceTsFileProcessor);
+ workSequenceTsFileProcessor.asyncClose();
+ updateEndTimeMap(workSequenceTsFileProcessor);
+ workSequenceTsFileProcessor = null;
+ }
+ if (workUnSequenceTsFileProcessor != null) {
+ closingUnSequenceTsFileProcessor.add(workUnSequenceTsFileProcessor);
+ workUnSequenceTsFileProcessor.asyncClose();
+ workUnSequenceTsFileProcessor = null;
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ /**
+ * when close an UnsealedTsFileProcessor, update its EndTimeMap immediately
+ *
+ * @param tsFileProcessor processor to be closed
+ */
+ private void updateEndTimeMap(UnsealedTsFileProcessorV2 tsFileProcessor) {
+ TsFileResourceV2 resource = tsFileProcessor.getTsFileResource();
+ for (Entry<String, Long> startTime : resource.getStartTimeMap().entrySet()) {
+ String deviceId = startTime.getKey();
+ resource.getEndTimeMap().put(deviceId, latestTimeForEachDevice.get(deviceId));
+ }
+ }
+
+ /**
+ * This method will be blocked until all tsfile processors are closed.
+ */
+ public void syncCloseFileNode() {
+ synchronized (closeFileNodeCondition) {
+ try {
+ asyncForceClose();
+ while (true) {
+ if (closingSequenceTsFileProcessor.isEmpty() && closingUnSequenceTsFileProcessor
+ .isEmpty()) {
+ break;
+ }
+ closeFileNodeCondition.wait();
+ }
+ } catch (InterruptedException e) {
+ LOGGER
+ .error("CloseFileNodeCondition occurs error while waiting for closing the file node {}",
+ storageGroupName, e);
+ }
+ }
+ }
+
+
+ /**
+ * This method will be blocked until this file node can be closed.
+ */
+ public void syncCloseAndStopFileNode(Supplier<Boolean> removeProcessorFromManagerCallback) {
+ synchronized (closeFileNodeCondition) {
+ try {
+ asyncForceClose();
+ toBeClosed = true;
+ while (true) {
+ if (closingSequenceTsFileProcessor.isEmpty() && closingUnSequenceTsFileProcessor
+ .isEmpty()) {
+ removeProcessorFromManagerCallback.get();
+ break;
+ }
+ closeFileNodeCondition.wait();
+ }
+ } catch (InterruptedException e) {
+ LOGGER
+ .error("CloseFileNodeCondition occurs error while waiting for closing the file node {}",
+ storageGroupName, e);
+ }
+ }
+ }
+
+
+ public boolean updateLatestFlushTimeCallback() {
+ writeLock();
+ try {
+ // update the largest timestamp in the last flushing memtable
+ for (Entry<String, Long> entry : latestTimeForEachDevice.entrySet()) {
+ latestFlushedTimeForEachDevice.put(entry.getKey(), entry.getValue());
+ }
+ } finally {
+ writeUnlock();
+ }
+ return true;
+ }
+
+ /**
+ * put the memtable back to the MemTablePool and make the metadata in writer visible
+ */
+ // TODO please consider concurrency with query and insert method.
+ public void closeUnsealedTsFileProcessorCallback(
+ UnsealedTsFileProcessorV2 unsealedTsFileProcessor) {
+ closeQueryLock.writeLock().lock();
+ try {
+ unsealedTsFileProcessor.close();
+ } catch (IOException e) {
+ LOGGER.error("storage group: {} close unsealedTsFileProcessor failed", storageGroupName, e);
+ } finally {
+ closeQueryLock.writeLock().unlock();
+ }
+ if (closingSequenceTsFileProcessor.contains(unsealedTsFileProcessor)) {
+ closingSequenceTsFileProcessor.remove(unsealedTsFileProcessor);
+ } else {
+ closingUnSequenceTsFileProcessor.remove(unsealedTsFileProcessor);
+ }
+ LOGGER.info("signal closing file node condition");
+ synchronized (closeFileNodeCondition) {
+ closeFileNodeCondition.notify();
+ }
+ }
+
+
+ public UnsealedTsFileProcessorV2 getWorkSequenceTsFileProcessor() {
+ return workSequenceTsFileProcessor;
+ }
+
+ public UnsealedTsFileProcessorV2 getWorkUnSequenceTsFileProcessor() {
+ return workUnSequenceTsFileProcessor;
+ }
+
+ public String getStorageGroupName() {
+ return storageGroupName;
+ }
+
+ public int getClosingProcessorSize() {
+ return unSequenceFileList.size() + sequenceFileList.size();
+ }
+}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
index e8366ee,0000000..9923624
mode 100644,000000..100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/ChunkBufferPool.java
@@@ -1,111 -1,0 +1,111 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.memtable;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Each flush task allocates new {@linkplain ChunkBuffer} which might be very large and lead to
+ * high-cost GC. In new design, we try to reuse ChunkBuffer objects by ChunkBufferPool, referring to
+ * {@linkplain MemTablePool}.
+ *
+ * Only for TEST up to now.
+ *
+ * @author kangrong
+ */
+public class ChunkBufferPool {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ChunkBufferPool.class);
+
+ private static final Deque<ChunkBuffer> availableChunkBuffer = new ArrayDeque<>();
+
+ private int size = 0;
+
+ private static final int WAIT_TIME = 2000;
+
+ private ChunkBufferPool() {
+ }
+
+ public ChunkBuffer getEmptyChunkBuffer(Object applier, MeasurementSchema schema) {
+ synchronized (availableChunkBuffer) {
+ //we use the memtable number * maximal series number in one StroageGroup * 2 as the capacity
+ int capacity =
+ 2 * MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor
- .getInstance().getConfig().getMemtableNumber();
++ .getInstance().getConfig().getMemtableNumber() + 100000;
+ if (availableChunkBuffer.isEmpty() && size < capacity) {
+ size++;
+ return new ChunkBuffer(schema);
+ } else if (!availableChunkBuffer.isEmpty()) {
+ ChunkBuffer chunkBuffer = availableChunkBuffer.pop();
+ chunkBuffer.reInit(schema);
+ return chunkBuffer;
+ }
+
+ // wait until some one has released a ChunkBuffer
+ int waitCount = 1;
+ while (true) {
+ if (!availableChunkBuffer.isEmpty()) {
+ return availableChunkBuffer.pop();
+ }
+ try {
+ availableChunkBuffer.wait(WAIT_TIME);
+ } catch (InterruptedException e) {
+ LOGGER.error("{} fails to wait fot ReusableChunkBuffer {}, continue to wait", applier, e);
+ }
+ LOGGER.info("{} has waited for a ReusableChunkBuffer for {}ms", applier,
+ waitCount++ * WAIT_TIME);
+ }
+ }
+ }
+
+ public void putBack(ChunkBuffer chunkBuffer) {
+ synchronized (availableChunkBuffer) {
+ chunkBuffer.reset();
+ //we use the memtable number * maximal series number in one StroageGroup as the capacity
+ int capacity =
+ MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups() * IoTDBDescriptor
+ .getInstance().getConfig().getMemtableNumber();
+ if (size > capacity) {
+ size --;
+ } else {
+ availableChunkBuffer.push(chunkBuffer);
+ }
+ availableChunkBuffer.notify();
+ }
+ }
+
+ public void putBack(ChunkBuffer chunkBuffer, String storageGroup) {
+ putBack(chunkBuffer);
+ }
+
+ public static ChunkBufferPool getInstance() {
+ return InstanceHolder.INSTANCE;
+ }
+
+ private static class InstanceHolder {
+
+ private InstanceHolder() {
+ }
+
+ private static final ChunkBufferPool INSTANCE = new ChunkBufferPool();
+ }
+}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index b0aea4f,0b8b0b0..56f5a66
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@@ -25,8 -25,7 +25,8 @@@ import java.util.HashMap
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
- import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0d827fa,8e8cc9d..dae7a49
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@@ -143,18 -142,10 +146,16 @@@ public class MManager
} else {
initFromLog(logFile);
}
+ seriesNumberInStorageGroups = mgraph.countSeriesNumberInEachStorageGroup();
+ if (seriesNumberInStorageGroups.isEmpty()) {
+ maxSeriesNumberAmongStorageGroup = 0;
+ } else {
+ maxSeriesNumberAmongStorageGroup = seriesNumberInStorageGroups.values().stream()
+ .max(Integer::compareTo).get();
+ }
logWriter = new BufferedWriter(new FileWriter(logFile, true));
writeToLog = true;
-
- } catch (PathErrorException | MetadataArgsErrorException
- } catch (PathErrorException | MetadataErrorException
-- | ClassNotFoundException | IOException e) {
++ } catch (PathErrorException | ClassNotFoundException | IOException | MetadataErrorException e) {
mgraph = new MGraph(ROOT_NAME);
LOGGER.error("Cannot read MGraph from file, using an empty new one");
} finally {
@@@ -193,6 -184,6 +194,10 @@@
lock.writeLock().lock();
try {
this.mgraph = new MGraph(ROOT_NAME);
++ this.checkAndGetDataTypeCache.clear();
++ this.mNodeCache.clear();
++ this.seriesNumberInStorageGroups.clear();
++ this.maxSeriesNumberAmongStorageGroup = 0;
} finally {
lock.writeLock().unlock();
}
@@@ -358,7 -494,8 +529,7 @@@
/**
* function for setting storage level of the given path to mTree.
*/
- public void setStorageLevelToMTree(String path) throws PathErrorException, IOException {
+ public void setStorageLevelToMTree(String path) throws MetadataErrorException {
-
lock.writeLock().lock();
try {
checkAndGetDataTypeCache.clear();
diff --cc iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
index 69f0a95,1cc3333..9f2da49
--- a/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/monitor/StatMonitor.java
@@@ -31,9 -31,9 +31,9 @@@ import org.apache.iotdb.db.concurrent.I
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.exception.FileNodeManagerException;
- import org.apache.iotdb.db.exception.PathErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.MonitorConstants.FileNodeManagerStatConstants;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
index e760f42,f404c91..cf8de37
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/OverflowQPExecutor.java
@@@ -35,9 -34,10 +35,10 @@@ import org.apache.iotdb.db.auth.entity.
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.FileNodeManagerException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
@@@ -72,7 -71,8 +73,6 @@@ import org.apache.iotdb.tsfile.read.exp
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
- import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -439,74 -425,14 +398,11 @@@ public class OverflowQPExecutor extend
}
break;
case DELETE_PATH:
-
- if (deletePathList != null && !deletePathList.isEmpty()) {
- Set<String> pathSet = new HashSet<>();
- // Attention: Monitor storage group seriesPath is not allowed to be deleted
- for (Path p : deletePathList) {
- List<String> subPaths = mManager.getPaths(p.getFullPath());
- if (subPaths.isEmpty()) {
- throw new ProcessorException(String
- .format("There are no timeseries in the prefix of %s seriesPath",
- p.getFullPath()));
- }
- List<String> newSubPaths = new ArrayList<>();
- for (String eachSubPath : subPaths) {
- String filenodeName = mManager.getFileNameByPath(eachSubPath);
-
- if (MonitorConstants.STAT_STORAGE_GROUP_PREFIX.equals(filenodeName)) {
- continue;
- }
- newSubPaths.add(eachSubPath);
- }
- pathSet.addAll(newSubPaths);
- }
- for (String p : pathSet) {
- if (!mManager.pathExist(p)) {
- throw new ProcessorException(String.format(
- "Timeseries %s does not exist and cannot be delete its metadata and data", p));
- }
- }
- List<String> fullPath = new ArrayList<>(pathSet);
- try {
- deleteDataOfTimeSeries(fullPath);
- } catch (ProcessorException e) {
- throw new ProcessorException(e);
- }
-
- Set<String> deleteFielNodes = new HashSet<>();
- for (String p : fullPath) {
- String nameSpacePath = null;
- try {
- nameSpacePath = mManager.getFileNameByPath(p);
- } catch (PathErrorException e) {
- throw new ProcessorException(e);
- }
- // the two map is stored in the storage group node
- schemaMap = mManager.getSchemaMapForOneFileNode(nameSpacePath);
- numSchemaMap = mManager.getNumSchemaMapForOneFileNode(nameSpacePath);
- // Thread safety: just one thread can access/modify the schemaMap
- synchronized (schemaMap) {
- // TODO: don't delete the storage group seriesPath
- // recursively
- path = new Path(p);
- String measurementId = path.getMeasurement();
- if (numSchemaMap.get(measurementId) == 1) {
- numSchemaMap.remove(measurementId);
- schemaMap.remove(measurementId);
- } else {
- numSchemaMap.put(measurementId, numSchemaMap.get(measurementId) - 1);
- }
- String deleteNameSpacePath = mManager.deletePathFromMTree(p);
- if (deleteNameSpacePath != null) {
- deleteFielNodes.add(deleteNameSpacePath);
- }
- }
- }
- for (String deleteFileNode : deleteFielNodes) {
- // close processor
- fileNodeManager.deleteOneFileNode(deleteFileNode);
- }
+ deleteDataOfTimeSeries(deletePathList);
+ Pair<Set<String>, Set<String>> closeDeletedStorageGroupPair =
+ mManager.deletePathsFromMTree(deletePathList);
- for (String closeStorageGroup : closeDeletedStorageGroupPair.left) {
- fileNodeManager.closeOneFileNode(closeStorageGroup);
- }
+ for (String deleteStorageGroup : closeDeletedStorageGroupPair.right) {
+ fileNodeManager.deleteOneFileNode(deleteStorageGroup);
}
break;
case SET_FILE_LEVEL:
diff --cc iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
index 6ad83ef,a834085..39c8ee7
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/QueryProcessExecutor.java
@@@ -106,24 -106,26 +107,24 @@@ public abstract class QueryProcessExecu
try {
boolean result = true;
MManager mManager = MManager.getInstance();
- Set<String> pathSet = new HashSet<>();
- for (Path p : paths) {
- pathSet.addAll(mManager.getPaths(p.getFullPath()));
+ Set<String> existingPaths = new HashSet<>();
+ for (Path p : deletePlan.getPaths()) {
+ existingPaths.addAll(mManager.getPaths(p.getFullPath()));
}
- if (pathSet.isEmpty()) {
- throw new ProcessorException("TimeSeries does not exist and cannot be delete data");
+ if (existingPaths.isEmpty()) {
+ throw new ProcessorException("TimeSeries does not exist and its data cannot be deleted");
}
- for (String onePath : pathSet) {
+ for (String onePath : existingPaths) {
if (!mManager.pathExist(onePath)) {
throw new ProcessorException(
- String.format("TimeSeries %s does not exist and cannot be delete its data", onePath));
+ String.format("TimeSeries %s does not exist and its data cannot be deleted", onePath));
}
}
- List<String> fullPath = new ArrayList<>();
- fullPath.addAll(pathSet);
- for (String path : fullPath) {
- result &= delete(new Path(path), deleteTime);
+ for (String path : existingPaths) {
+ result &= delete(new Path(path), deletePlan.getDeleteTime());
}
return result;
- } catch (PathErrorException e) {
+ } catch (MetadataErrorException e) {
throw new ProcessorException(e);
}
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index de17c57,07d1e75..231e548
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@@ -18,15 -18,10 +18,16 @@@
*/
package org.apache.iotdb.db.qp.physical;
++import java.io.IOException;
import java.io.Serializable;
+import java.nio.ByteBuffer;
import java.util.List;
import org.apache.iotdb.db.qp.logical.Operator;
+import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.db.qp.physical.transfer.SystemLogOperator;
import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
/**
* This class is a abstract class for all type of PhysicalPlan.
@@@ -90,61 -79,4 +91,61 @@@ public abstract class PhysicalPlan
public void setQuery(boolean query) {
isQuery = query;
}
+
+ public long getPlanId() {
+ return planId;
+ }
+
+ public void setPlanId(long planId) {
+ this.planId = planId;
+ }
+
+ public void serializeTo(ByteBuffer buffer) {
+ throw new UnsupportedOperationException("serialize of unimplemented");
+ }
+
+ public void deserializeFrom(ByteBuffer buffer) {
+ throw new UnsupportedOperationException("serialize of unimplemented");
+ }
+
+ protected void putString(ByteBuffer buffer, String value) {
+ if (value == null) {
+ buffer.putInt(NULL_VALUE_LEN);
+ } else {
+ ReadWriteIOUtils.write(value, buffer);
+ }
+ }
+
+ protected String readString(ByteBuffer buffer) {
+ int valueLen = buffer.getInt();
+ if (valueLen == NULL_VALUE_LEN) {
+ return null;
+ }
+ return ReadWriteIOUtils.readStringWithoutLength(buffer, valueLen);
+ }
+
+ public static class Factory {
+
+ private Factory() {
+ // hidden initializer
+ }
+
- public static PhysicalPlan create(ByteBuffer buffer) {
++ public static PhysicalPlan create(ByteBuffer buffer) throws IOException {
+ byte type = buffer.get();
+ PhysicalPlan plan;
+ switch (type) {
+ case SystemLogOperator.INSERT:
+ plan = new InsertPlan();
+ plan.deserializeFrom(buffer);
+ break;
+ case SystemLogOperator.DELETE:
+ plan = new DeletePlan();
+ plan.deserializeFrom(buffer);
+ break;
+ default:
- throw new UnsupportedOperationException("unrecognized log type " + type);
++ throw new IOException("unrecognized log type " + type);
+ }
+ return plan;
+ }
+ }
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index c5caa98,32e985e..6b4ff40
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@@ -41,9 -39,10 +41,10 @@@ import org.apache.iotdb.db.auth.authori
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.exception.ArgsErrorException;
import org.apache.iotdb.db.exception.FileNodeManagerException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.qp.IllegalASTFormatException;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
index c799c65,213e4c9..f384565
--- a/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/sync/receiver/SyncServiceImpl.java
@@@ -42,11 -42,13 +42,11 @@@ import org.apache.commons.lang3.StringU
import org.apache.iotdb.db.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
-import org.apache.iotdb.db.engine.filenode.OverflowChangeType;
-import org.apache.iotdb.db.engine.filenode.TsFileResource;
-import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
+import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
import org.apache.iotdb.db.exception.FileNodeManagerException;
- import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/writelog/io/BatchLogReader.java
index 9b03cc1,1846f12..c842ffe
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/BatchLogReader.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/io/BatchLogReader.java
@@@ -17,39 -17,29 +17,39 @@@
* under the License.
*/
-package org.apache.iotdb.db.query.dataset;
+package org.apache.iotdb.db.writelog.io;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.common.RowRecord;
-import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
-public class AuthDataSet extends QueryDataSet {
+/**
+ * BatchedLogReader reads logs from a binary batch of log in the format of ByteBuffer. The
+ * ByteBuffer must be readable.
+ */
+public class BatchLogReader implements ILogReader{
- List<RowRecord> records = new ArrayList<>();
- int index = 0;
+ private Iterator<PhysicalPlan> planIterator;
+
- public BatchLogReader(ByteBuffer buffer) {
++ public BatchLogReader(ByteBuffer buffer) throws IOException {
+ List<PhysicalPlan> logs = readLogs(buffer);
+ this.planIterator = logs.iterator();
+ }
- private List<PhysicalPlan> readLogs(ByteBuffer buffer) {
- public AuthDataSet(List<Path> paths,
- List<TSDataType> dataTypes) {
- super(paths, dataTypes);
++ private List<PhysicalPlan> readLogs(ByteBuffer buffer) throws IOException {
+ List<PhysicalPlan> plans = new ArrayList<>();
+ while (buffer.position() != buffer.limit()) {
+ plans.add(PhysicalPlan.Factory.create(buffer));
+ }
+ return plans;
}
+
@Override
- public boolean hasNext() throws IOException {
- return index < records.size();
+ public void close() {
}
@Override
diff --cc iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index b64d5e7,0000000..e64f626
mode 100644,000000..100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@@ -1,133 -1,0 +1,134 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.writelog.recover;
+
+import static org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2.RESOURCE_SUFFIX;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
+import org.apache.iotdb.db.engine.memtable.IMemTable;
+import org.apache.iotdb.db.engine.memtable.MemTableFlushTaskV2;
+import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
+import org.apache.iotdb.db.engine.version.VersionController;
+import org.apache.iotdb.db.exception.ProcessorException;
+import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.writer.NativeRestorableIOWriter;
+
+/**
+ * TsFileRecoverPerformer recovers a SeqTsFile to correct status, redoes the WALs since last
+ * crash and removes the redone logs.
+ */
+public class TsFileRecoverPerformer {
+
+ private String insertFilePath;
+ private String logNodePrefix;
+ private FileSchema fileSchema;
+ private VersionController versionController;
+ private LogReplayer logReplayer;
+ private TsFileResourceV2 tsFileResource;
+ private boolean acceptUnseq;
+
+ public TsFileRecoverPerformer(String logNodePrefix,
+ FileSchema fileSchema, VersionController versionController,
+ TsFileResourceV2 currentTsFileResource, boolean acceptUnseq) {
+ this.insertFilePath = currentTsFileResource.getFile().getPath();
+ this.logNodePrefix = logNodePrefix;
+ this.fileSchema = fileSchema;
+ this.versionController = versionController;
+ this.tsFileResource = currentTsFileResource;
+ this.acceptUnseq = acceptUnseq;
+ }
+
+ /**
+ * 1. recover the TsFile by NativeRestorableIOWriter and truncate position of last recovery
+ * 2. redo the WALs to recover unpersisted data
+ * 3. flush and close the file
+ * 4. clean WALs
+ * @throws ProcessorException
+ */
+ public void recover() throws ProcessorException {
+ IMemTable recoverMemTable = new PrimitiveMemTable();
+ this.logReplayer = new LogReplayer(logNodePrefix, insertFilePath, tsFileResource.getModFile(),
+ versionController,
+ tsFileResource, fileSchema, recoverMemTable, acceptUnseq);
+ File insertFile = new File(insertFilePath);
+ if (!insertFile.exists()) {
+ return;
+ }
+ // remove corrupted part of the TsFile
+ NativeRestorableIOWriter restorableTsFileIOWriter;
+ try {
+ restorableTsFileIOWriter = new NativeRestorableIOWriter(insertFile);
+ } catch (IOException e) {
+ throw new ProcessorException(e);
+ }
+
+ if (!restorableTsFileIOWriter.hasCrashed()) {
+ try {
++ // recover two maps of time
+ tsFileResource.deSerialize();
+ return;
+ } catch (IOException e) {
+ throw new ProcessorException("recover the resource file failed: " + insertFilePath
+ + RESOURCE_SUFFIX, e);
+ }
+ } else {
+ // due to failure, the last ChunkGroup may contain the same data as the WALs, so the time
+ // map must be updated first to avoid duplicated insertion
+ for (ChunkGroupMetaData chunkGroupMetaData : restorableTsFileIOWriter.getChunkGroupMetaDatas()) {
+ for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) {
+ tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getStartTime());
+ tsFileResource.updateTime(chunkGroupMetaData.getDeviceID(), chunkMetaData.getEndTime());
+ }
+ }
+ }
+
+ // redo logs
+ logReplayer.replayLogs();
+ if (recoverMemTable.isEmpty()) {
+ return;
+ }
+
+ // flush logs
+ MemTableFlushTaskV2 tableFlushTask = new MemTableFlushTaskV2(recoverMemTable, fileSchema, restorableTsFileIOWriter,
+ logNodePrefix, (a) -> {});
+ tableFlushTask.flushMemTable();
+
+ // close file
+ try {
+ restorableTsFileIOWriter.endFile(fileSchema);
+ tsFileResource.serialize();
+ } catch (IOException e) {
+ throw new ProcessorException("Cannot close file when recovering", e);
+ }
+
+ // clean logs
+ try {
+ MultiFileLogNodeManager.getInstance().deleteNode(logNodePrefix + new File(insertFilePath).getName());
+ } catch (IOException e) {
+ throw new ProcessorException(e);
+ }
+ }
+
+}
diff --cc iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerBenchmark.java
index 20e72c3,35d5f21..4ebf8a5
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerBenchmark.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerBenchmark.java
@@@ -21,12 -21,11 +21,12 @@@ package org.apache.iotdb.db.engine.file
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.exception.FileNodeManagerException;
- import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.sync.test.RandomNum;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
diff --cc iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 5277272,2b6a27d..f5f6bfd
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@@ -29,17 -29,16 +29,17 @@@ import java.io.IOException
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.Directories;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.engine.modification.io.LocalTextModificationAccessor;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
import org.apache.iotdb.db.exception.FileNodeManagerException;
- import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
+import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.TimeValuePair;
diff --cc iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
index c2ff798,9692dd3..506eee6
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/modification/DeletionQueryTest.java
@@@ -26,15 -26,13 +26,15 @@@ import java.io.IOException
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.filenode.FileNodeManager;
+import org.apache.iotdb.db.engine.filenodeV2.FileNodeManagerV2;
import org.apache.iotdb.db.engine.memcontrol.BasicMemController.UsageLevel;
import org.apache.iotdb.db.exception.FileNodeManagerException;
- import org.apache.iotdb.db.exception.MetadataArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.PathErrorException;
+import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.executor.EngineQueryRouter;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
diff --cc iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
index 7dc0700,a4b95a3..5a5aee0
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSequenceDataQueryIT.java
@@@ -84,7 -84,7 +84,6 @@@ public class IoTDBSequenceDataQueryIT
daemon.active();
EnvironmentUtils.envSetUp();
- // Thread.sleep(5000);
- Thread.sleep(5000);
insertData();
}
diff --cc iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
index aa2ef7c,aa2ef7c..1e4cab5
--- a/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/integration/IoTDBSeriesReaderIT.java
@@@ -85,7 -85,7 +85,7 @@@ public class IoTDBSeriesReaderIT
daemon.active();
EnvironmentUtils.envSetUp();
-- Thread.sleep(5000);
++ Thread.sleep(1000);
insertData();
connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
diff --cc iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
index 2ead48c,0ffbd66..2a1948d
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerAdvancedTest.java
@@@ -44,8 -43,8 +43,9 @@@ public class MManagerAdvancedTest
mmanager = MManager.getInstance();
-- mmanager.setStorageLevelToMTree("root.vehicle.d1");
mmanager.setStorageLevelToMTree("root.vehicle.d0");
++ mmanager.setStorageLevelToMTree("root.vehicle.d1");
++ mmanager.setStorageLevelToMTree("root.vehicle.d2");
mmanager.addPathToMTree("root.vehicle.d0.s0", "INT32", "RLE");
mmanager.addPathToMTree("root.vehicle.d0.s1", "INT64", "RLE");
@@@ -125,7 -124,7 +125,7 @@@
@Test
public void testGetNextLevelPath()
- throws PathErrorException, IOException, MetadataArgsErrorException {
- throws PathErrorException, IOException, MetadataErrorException {
++ throws PathErrorException, IOException {
mmanager.addPathToMTree("root.vehicle.d2.s0", "DOUBLE", "RLE");
mmanager.addPathToMTree("root.vehicle.d2.s1", "BOOLEAN", "PLAIN");
mmanager.addPathToMTree("root.vehicle.d2.s2.g0", "TEXT", "PLAIN");
diff --cc iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 61c08d6,086d0be..159a7f1
--- a/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@@ -344,28 -344,4 +344,28 @@@ public class MManagerBasicTest
fail(e.getMessage());
}
}
+
+ @Test
- public void testMaximalSeriesNumberAmongStorageGroup() throws IOException, PathErrorException {
++ public void testMaximalSeriesNumberAmongStorageGroup() throws MetadataErrorException {
+ MManager manager = MManager.getInstance();
+ assertEquals(0, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.setStorageLevelToMTree("root.laptop");
+ assertEquals(0, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.addPathToMTree("root.laptop.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ manager.addPathToMTree("root.laptop.d1.s2", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ assertEquals(2, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.setStorageLevelToMTree("root.vehicle");
+ manager.addPathToMTree("root.vehicle.d1.s1", TSDataType.INT32, TSEncoding.PLAIN,
+ CompressionType.GZIP, null);
+ assertEquals(2, manager.getMaximalSeriesNumberAmongStorageGroups());
+
+ manager.deletePathFromMTree("root.laptop.d1.s1");
+ assertEquals(1, manager.getMaximalSeriesNumberAmongStorageGroups());
+ manager.deletePathFromMTree("root.laptop.d1.s2");
+ assertEquals(1, manager.getMaximalSeriesNumberAmongStorageGroups());
+
+
+ }
}
diff --cc iotdb/src/test/java/org/apache/iotdb/db/qp/physical/transfer/PhysicalPlanLogTransferTest.java
index 41909a4,b2aa2e4..fb72b27
--- a/iotdb/src/test/java/org/apache/iotdb/db/qp/physical/transfer/PhysicalPlanLogTransferTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/qp/physical/transfer/PhysicalPlanLogTransferTest.java
@@@ -22,8 -22,10 +22,9 @@@ import static org.junit.Assert.assertAr
import static org.junit.Assert.assertEquals;
import java.io.IOException;
-import java.util.Arrays;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.exception.ArgsErrorException;
+ import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.qp.QueryProcessorException;
import org.apache.iotdb.db.qp.QueryProcessor;
@@@ -50,9 -52,10 +51,10 @@@ public class PhysicalPlanLogTransferTes
@Test
public void operatorToLog()
- throws IOException, ArgsErrorException, ProcessorException, QueryProcessorException {
+ throws IOException, ArgsErrorException, ProcessorException, QueryProcessorException,
+ MetadataErrorException {
/** Insert Plan test **/
- byte[] insertPlanBytesTest = PhysicalPlanLogTransfer.operatorToLog(insertPlan);
+ byte[] insertPlanBytesTest = PhysicalPlanLogTransfer.planToLog(insertPlan);
Codec<InsertPlan> insertPlanCodec = CodecInstances.multiInsertPlanCodec;
byte[] insertPlanProperty = insertPlanCodec.encode(insertPlan);
assertArrayEquals(insertPlanProperty, insertPlanBytesTest);
@@@ -103,12 -106,13 +105,13 @@@
@Test
public void logToOperator()
- throws IOException, ArgsErrorException, ProcessorException, QueryProcessorException, AuthException {
+ throws IOException, ArgsErrorException, ProcessorException, QueryProcessorException,
+ AuthException, MetadataErrorException {
/** Insert Plan test **/
- byte[] insertPlanBytesTest = PhysicalPlanLogTransfer.operatorToLog(insertPlan);
+ byte[] insertPlanBytesTest = PhysicalPlanLogTransfer.planToLog(insertPlan);
InsertPlan insertPlanTest = (InsertPlan) PhysicalPlanLogTransfer
- .logToOperator(insertPlanBytesTest);
+ .logToPlan(insertPlanBytesTest);
assertEquals(insertPlanTest, insertPlan);
/** Delete Plan test **/
@@@ -132,11 -136,12 +135,12 @@@
assertEquals(metadataPlanTest, metadataPlan);
/** Author Plan test **/
- String sql = "grant role xm privileges 'SET_STORAGE_GROUP','DELETE_TIMESERIES' on root.vehicle.device.sensor";
+ String sql = "grant role xm privileges 'SET_STORAGE_GROUP','DELETE_TIMESERIES' "
+ + "on root.vehicle.device.sensor";
AuthorPlan authorPlan = (AuthorPlan) processor.parseSQLToPhysicalPlan(sql);
- byte[] authorPlanBytesTest = PhysicalPlanLogTransfer.operatorToLog(authorPlan);
+ byte[] authorPlanBytesTest = PhysicalPlanLogTransfer.planToLog(authorPlan);
AuthorPlan authorPlanTest = (AuthorPlan) PhysicalPlanLogTransfer
- .logToOperator(authorPlanBytesTest);
+ .logToPlan(authorPlanBytesTest);
assertEquals(authorPlanTest, authorPlan);
/** LoadData Plan test **/
diff --cc iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
index 586eb79,be34d32..ef8e083
--- a/iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/tools/WalCheckerTest.java
@@@ -113,12 -114,12 +113,11 @@@ public class WalCheckerTest
String[] measurements = new String[]{"s1", "s2", "s3"};
String[] values = new String[]{"5", "6", "7"};
for (int j = 0; j < 10; j++) {
- binaryPlans.add(PhysicalPlanLogTransfer
- .operatorToLog(new InsertPlan(deviceId, j, measurements, values)));
+ new InsertPlan(deviceId, j, measurements, values).serializeTo(binaryPlans);
}
if (i > 2) {
- binaryPlans.add("not a wal".getBytes());
+ binaryPlans.put("not a wal".getBytes());
}
- binaryPlans.flip();
logWriter.write(binaryPlans);
logWriter.force();
diff --cc iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index b7d3f9c,e86ddd9..1322283
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@@ -153,7 -151,7 +153,6 @@@ public class EnvironmentUtils
}
public static void envSetUp() throws StartupException, IOException {
-
- // create all directory
createAllDir();
// disable the memory control
config.setEnableMemMonitor(false);
diff --cc iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
index 1d68e39,22faabf..4dfe658
--- a/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/writelog/PerformanceTest.java
@@@ -36,6 -33,9 +36,7 @@@ import org.apache.iotdb.db.qp.physical.
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode;
import org.apache.iotdb.db.writelog.node.WriteLogNode;
-import org.apache.iotdb.db.qp.physical.transfer.PhysicalPlanLogTransfer;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
++import org.apache.iotdb.tsfile.exception.metadata.MetadataArgsErrorException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index c2112c9,9f7ab47..7c9fbc1
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@@ -576,25 -579,30 +576,25 @@@ public class TsFileSequenceReader imple
currentChunkGroup.setEndOffsetOfChunkGroup(endOffsetOfChunkGroup);
currentChunkGroup.setVersion(versionOfChunkGroup++);
newMetaData.add(currentChunkGroup);
- // though we have read the current ChunkMetaData from Disk, it may be incomplete.
- // because if the file only loses one byte, the ChunkMetaData.deserialize() returns ok,
- // while the last filed of the ChunkMetaData is incorrect.
- // So, only reading the next MASK, can make sure that this ChunkMetaData is complete.
- haveReadAnUnverifiedGroupFooter = true;
+ newChunkGroup = true;
+ truncatedPosition = this.position();
break;
-
default:
- // it is impossible that we read an incorrect data.
+ // the disk file is corrupted, using this file may be dangerous
MetaMarker.handleUnexpectedMarker(marker);
goon = false;
+ LOGGER.error("Unrecognized marker detected, this file may be corrupted");
}
}
- //now we read the tail of the file, so we are sure that the last ChunkGroupFooter is complete.
+ // now we read the tail of the data section, so we are sure that the last ChunkGroupFooter is
+ // complete.
truncatedPosition = this.position() - 1;
} catch (IOException e2) {
- //if it is the end of the file, and we read an unverifiedGroupFooter, we must remove this ChunkGroup
- if (haveReadAnUnverifiedGroupFooter && !newMetaData.isEmpty()) {
- newMetaData.remove(newMetaData.size() - 1);
- }
- } finally {
- //something wrong or all data is complete. We will discard current FileMetadata
- // so that we can continue to write data into this tsfile.
- return truncatedPosition;
+ LOGGER.info("TsFile self-check cannot proceed at position {} after {} chunk groups "
- + "recovered, because", this.position(), newMetaData.size(), e2);
++ + "recovered, because : {}", this.position(), newMetaData.size(), e2.getMessage());
}
+ // Despite the completeness of the data section, we will discard current FileMetadata
+ // so that we can continue to write data into this tsfile.
+ return truncatedPosition;
}
}