You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/07/17 03:06:51 UTC
[incubator-iotdb] branch dev_merge updated: refactor package
structure
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_merge
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/dev_merge by this push:
new 83a9501 refactor package structure
83a9501 is described below
commit 83a9501eb0268dd369e83f4b20a593c99d0077bf
Author: 江天 <jt...@163.com>
AuthorDate: Wed Jul 17 11:04:24 2019 +0800
refactor package structure
---
.../apache/iotdb/db/engine/merge/MergeTask.java | 696 ---------------------
.../db/engine/merge/{ => manage}/MergeManager.java | 3 +-
.../db/engine/merge/manage/MergeResource.java | 179 ++++++
.../db/engine/merge/{ => recover}/MergeLogger.java | 47 +-
.../merge/{ => selector}/MergeFileSelector.java | 4 +-
.../db/engine/merge/{ => task}/MergeCallback.java | 2 +-
.../iotdb/db/engine/merge/task/MergeFileTask.java | 245 ++++++++
.../db/engine/merge/task/MergeSeriesTask.java | 301 +++++++++
.../iotdb/db/engine/merge/task/MergeTask.java | 144 +++++
.../engine/merge/{ => task}/RecoverMergeTask.java | 37 +-
.../engine/storagegroup/StorageGroupProcessor.java | 11 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +-
.../db/{engine/merge => utils}/MergeUtils.java | 59 +-
.../db/engine/merge/MergeFileSelectorTest.java | 1 +
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 1 +
.../iotdb/db/engine/merge/MergePerfTest.java | 1 +
.../iotdb/db/engine/merge/MergeTaskTest.java | 1 +
17 files changed, 988 insertions(+), 746 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
deleted file mode 100644
index d6fab3f..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeTask.java
+++ /dev/null
@@ -1,696 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.engine.merge;
-
-import static org.apache.iotdb.db.engine.merge.MergeUtils.collectFileSeries;
-import static org.apache.iotdb.db.engine.merge.MergeUtils.writeBatchPoint;
-import static org.apache.iotdb.db.engine.merge.MergeUtils.writeTVPair;
-import static org.apache.iotdb.db.utils.QueryUtils.modifyChunkMetaData;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import org.apache.commons.io.FileUtils;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.modification.Modification;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.factory.SeriesReaderFactoryImpl;
-import org.apache.iotdb.db.query.reader.IPointReader;
-import org.apache.iotdb.db.utils.TimeValuePair;
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.BatchData;
-import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
-import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerier;
-import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
-import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
-import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
-import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
-import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
-import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
-import org.apache.iotdb.tsfile.write.schema.FileSchema;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.ForceAppendTsFileWriter;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * MergeTask merges given SeqFiles and UnseqFiles into a new one.
- */
-public class MergeTask implements Callable<Void> {
-
- private static int minChunkPointNum = IoTDBDescriptor.getInstance().getConfig().getChunkMergePointThreshold();
- public static final String MERGE_SUFFIX = ".merge";
- private static final Logger logger = LoggerFactory.getLogger(MergeTask.class);
-
- List<TsFileResource> seqFiles;
- List<TsFileResource> unseqFiles;
- String storageGroupDir;
- MergeLogger mergeLogger;
-
- private TimeValuePair currTimeValuePair;
-
- private Map<TsFileResource, TsFileSequenceReader> fileReaderCache = new HashMap<>();
- private Map<TsFileResource, RestorableTsFileIOWriter> fileWriterCache = new HashMap<>();
- private Map<TsFileResource, List<Modification>> modificationCache = new HashMap<>();
- private Map<TsFileResource, MetadataQuerier> metadataQuerierCache = new HashMap<>();
- private Map<String, MeasurementSchema> measurementSchemaMap = new HashMap<>();
-
- Map<TsFileResource, Integer> mergedChunkCnt = new HashMap<>();
- Map<TsFileResource, Integer> unmergedChunkCnt = new HashMap<>();
- Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes = new HashMap<>();
-
- private QueryContext mergeContext = new QueryContext();
-
- private boolean keepChunkMetadata = false;
-
- private long currDeviceMaxTime;
-
- private MergeCallback callback;
-
- String taskName;
-
- // future feature
- private boolean fullMerge;
-
- private int totalChunkWritten;
-
- public MergeTask(List<TsFileResource> seqFiles,
- List<TsFileResource> unseqFiles, String storageGroupDir, MergeCallback callback,
- String taskName, boolean fullMerge) throws IOException {
- this.seqFiles = seqFiles;
- this.unseqFiles = unseqFiles;
- this.storageGroupDir = storageGroupDir;
- this.callback = callback;
- this.taskName = taskName;
- this.fullMerge = fullMerge;
- }
-
- @Override
- public Void call() throws Exception {
- try {
- doMerge();
- } catch (Exception e) {
- logger.error("Runtime exception in merge {}", taskName, e);
- seqFiles = Collections.emptyList();
- unseqFiles = Collections.emptyList();
- cleanUp(true);
- throw e;
- }
- return null;
- }
-
- private void doMerge() throws IOException {
- if (logger.isInfoEnabled()) {
- logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName, seqFiles.size(),
- unseqFiles.size());
- }
- long startTime = System.currentTimeMillis();
- long totalFileSize = collectFileSizes(seqFiles, unseqFiles);
- this.mergeLogger = new MergeLogger(storageGroupDir);
- logFiles();
-
- List<Path> unmergedSeries = collectPaths();
- mergeSeries(unmergedSeries);
-
- List<TsFileResource> unmergedFiles = seqFiles;
- mergeFiles(unmergedFiles);
-
- cleanUp(true);
- if (logger.isInfoEnabled()) {
- double elapsedTime = (double) (System.currentTimeMillis() - startTime);
- double byteRate = totalFileSize / elapsedTime / 1024 / 1024 * 1000;
- double seriesRate = unmergedSeries.size() / elapsedTime * 1000;
- double chunkRate = totalChunkWritten / elapsedTime * 1000;
- double fileRate = (seqFiles.size() + unseqFiles.size()) / elapsedTime * 1000;
- logger.info("{} ends after {}ms, byteRate: {}MB/s, seriesRate {}/s, chunkRate: {}/s, "
- + "fileRate: {}/s",
- taskName, elapsedTime, byteRate, seriesRate, chunkRate, fileRate);
- }
- }
-
- private long collectFileSizes(List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles) {
- long totalSize = 0;
- for (TsFileResource tsFileResource : seqFiles) {
- totalSize += tsFileResource.getFileSize();
- }
- for (TsFileResource tsFileResource : unseqFiles) {
- totalSize += tsFileResource.getFileSize();
- }
- return totalSize;
- }
-
- void mergeFiles(List<TsFileResource> unmergedFiles) throws IOException {
- // decide whether to write the unmerged chunks to the merge files or to move the merged data
- // back to the origin seqFile's
- if (logger.isInfoEnabled()) {
- logger.info("{} starts to merge {} files", taskName, unmergedFiles.size());
- }
- long startTime = System.currentTimeMillis();
- int cnt = 0;
- for (TsFileResource seqFile : unmergedFiles) {
- int mergedChunkNum = mergedChunkCnt.getOrDefault(seqFile, 0);
- int unmergedChunkNum = unmergedChunkCnt.getOrDefault(seqFile, 0);
- if (mergedChunkNum >= unmergedChunkNum) {
- // move the unmerged data to the new file
- if (logger.isInfoEnabled()) {
- logger.info("{} moving unmerged data of {} to the merged file, {} merged chunks, {} "
- + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
- }
- moveUnmergedToNew(seqFile);
- } else {
- // move the merged data to the old file
- if (logger.isInfoEnabled()) {
- logger.info("{} moving merged data of {} to the old file {} merged chunks, {} "
- + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
- }
- moveMergedToOld(seqFile);
- }
- cnt ++;
- if (logger.isInfoEnabled()) {
- logger.debug("{} has merged {}/{} files", taskName, cnt, unmergedFiles.size());
- }
- }
- if (logger.isInfoEnabled()) {
- logger.info("{} has merged all files after {}ms", taskName, System.currentTimeMillis() - startTime);
- }
- mergeLogger.logMergeEnd();
- }
-
- private void logFiles() throws IOException {
- mergeLogger.logSeqFiles(seqFiles);
- mergeLogger.logUnseqFiles(unseqFiles);
- mergeLogger.logMergeStart();
- }
-
- void mergeSeries(List<Path> unmergedSeries) throws IOException {
- if (logger.isInfoEnabled()) {
- logger.info("{} starts to merge {} series", taskName, unmergedSeries.size());
- }
- long startTime = System.currentTimeMillis();
- for (TsFileResource seqFile : seqFiles) {
- unmergedChunkStartTimes.put(seqFile, new HashMap<>());
- }
- // merge each series and write data into each seqFile's temp merge file
- int mergedCnt = 0;
- double progress = 0.0;
- for (Path path : unmergedSeries) {
- mergeLogger.logTSStart(path);
- mergeOnePath(path);
- mergeLogger.logTSEnd(path);
- mergedCnt ++;
- if (logger.isInfoEnabled()) {
- double newProgress = 100 * mergedCnt / (double) (unmergedSeries.size());
- if (newProgress - progress >= 0.01) {
- progress = newProgress;
- logger.info("{} has merged {}% series", taskName, progress);
- }
- }
- }
- if (logger.isInfoEnabled()) {
- logger.info("{} all series are merged after {}ms", taskName, System.currentTimeMillis() - startTime);
- }
- mergeLogger.logAllTsEnd();
- }
-
-
- void cleanUp(boolean executeCallback) throws IOException {
- logger.info("{} is cleaning up", taskName);
- for (TsFileSequenceReader sequenceReader : fileReaderCache.values()) {
- sequenceReader.close();
- }
-
- fileReaderCache.clear();
- fileWriterCache.clear();
- modificationCache.clear();
- metadataQuerierCache.clear();
- mergedChunkCnt.clear();
- unmergedChunkCnt.clear();
- unmergedChunkStartTimes.clear();
-
- mergeLogger.close();
-
- for (TsFileResource seqFile : seqFiles) {
- File mergeFile = new File(seqFile.getFile().getPath() + MERGE_SUFFIX);
- mergeFile.delete();
- }
-
- File logFile = new File(storageGroupDir, MergeLogger.MERGE_LOG_NAME);
- if (executeCallback) {
- callback.call(seqFiles, unseqFiles, logFile);
- } else {
- logFile.delete();
- }
- }
-
- private void moveMergedToOld(TsFileResource seqFile) throws IOException {
- int mergedChunkNum = mergedChunkCnt.getOrDefault(seqFile, 0);
- if (mergedChunkNum == 0) {
- RestorableTsFileIOWriter newFileWriter = getMergeFileWriter(seqFile);
- newFileWriter.close();
- newFileWriter.getFile().delete();
- fileWriterCache.remove(seqFile);
- return;
- }
-
- seqFile.getMergeQueryLock().writeLock().lock();
- try {
- TsFileIOWriter oldFileWriter;
- try {
- oldFileWriter = new ForceAppendTsFileWriter(seqFile.getFile());
- mergeLogger.logFileMergeStart(seqFile.getFile(), ((ForceAppendTsFileWriter) oldFileWriter).getTruncatePosition());
- logger.debug("{} moving merged chunks of {} to the old file", taskName, seqFile);
- ((ForceAppendTsFileWriter) oldFileWriter).doTruncate();
- } catch (TsFileNotCompleteException e) {
- // this file may already be truncated if this merge is a system reboot merge
- oldFileWriter = new RestorableTsFileIOWriter(seqFile.getFile());
- }
-
- oldFileWriter.filterChunks(unmergedChunkStartTimes.get(seqFile));
-
- RestorableTsFileIOWriter newFileWriter = getMergeFileWriter(seqFile);
- newFileWriter.close();
- try ( TsFileSequenceReader newFileReader =
- new TsFileSequenceReader(newFileWriter.getFile().getPath())) {
- ChunkLoader chunkLoader = new ChunkLoaderImpl(newFileReader);
- List<ChunkGroupMetaData> chunkGroupMetadataList = newFileWriter.getChunkGroupMetaDatas();
- if (logger.isDebugEnabled()) {
- logger.debug("{} find {} merged chunk groups", taskName, chunkGroupMetadataList.size());
- }
- for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetadataList) {
- writeMergedChunkGroup(chunkGroupMetaData, chunkLoader, oldFileWriter);
- }
- }
- oldFileWriter.endFile(new FileSchema(oldFileWriter.getKnownSchema()));
-
- seqFile.serialize();
- mergeLogger.logFileMergeEnd(seqFile.getFile());
- logger.debug("{} moved merged chunks of {} to the old file", taskName, seqFile);
-
- newFileWriter.getFile().delete();
- } finally {
- seqFile.getMergeQueryLock().writeLock().unlock();
- }
- }
-
- private void writeMergedChunkGroup(ChunkGroupMetaData chunkGroupMetaData,
- ChunkLoader chunkLoader, TsFileIOWriter fileWriter)
- throws IOException {
- fileWriter.startChunkGroup(chunkGroupMetaData.getDeviceID());
- long maxVersion = Long.MIN_VALUE;
- for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) {
- Path path = new Path(chunkGroupMetaData.getDeviceID(), chunkMetaData.getMeasurementUid());
- MeasurementSchema measurementSchema = getSchema(path);
- IChunkWriter chunkWriter = getChunkWriter(measurementSchema);
- Chunk chunk = chunkLoader.getChunk(chunkMetaData);
- writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
- chunkWriter.writeToFileWriter(fileWriter);
- maxVersion = maxVersion < chunkMetaData.getVersion() ? chunkMetaData.getVersion() :
- maxVersion;
- }
- fileWriter.endChunkGroup(maxVersion + 1);
- }
-
- private void moveUnmergedToNew(TsFileResource seqFile) throws IOException {
- Map<Path, List<Long>> fileUnmergedChunkStartTimes = this.unmergedChunkStartTimes.get(seqFile);
- RestorableTsFileIOWriter fileWriter = getMergeFileWriter(seqFile);
- ChunkLoader chunkLoader = new ChunkLoaderImpl(getFileReader(seqFile));
-
- mergeLogger.logFileMergeStart(fileWriter.getFile(), fileWriter.getFile().length());
- logger.debug("{} moving unmerged chunks of {} to the new file", taskName, seqFile);
-
- int unmergedChunkNum = unmergedChunkCnt.getOrDefault(seqFile, 0);
-
- if (unmergedChunkNum > 0) {
- for (Entry<Path, List<Long>> entry : fileUnmergedChunkStartTimes.entrySet()) {
- Path path = entry.getKey();
- List<Long> chunkStartTimes = entry.getValue();
- if (chunkStartTimes.isEmpty()) {
- continue;
- }
-
- List<ChunkMetaData> chunkMetaDataList = queryChunkMetadata(path, seqFile);
- MeasurementSchema measurementSchema = getSchema(path);
- IChunkWriter chunkWriter = getChunkWriter(measurementSchema);
-
- if (logger.isDebugEnabled()) {
- logger.debug("{} find {} unmerged chunks", taskName, chunkMetaDataList.size());
- }
-
- fileWriter.startChunkGroup(path.getDevice());
- long maxVersion = writeUnmergedChunks(chunkStartTimes, chunkMetaDataList, chunkLoader,
- chunkWriter, measurementSchema, fileWriter);
- fileWriter.endChunkGroup(maxVersion + 1);
- }
- }
-
- fileWriter.endFile(new FileSchema(fileWriter.getKnownSchema()));
-
- seqFile.serialize();
- mergeLogger.logFileMergeEnd(fileWriter.getFile());
- logger.debug("{} moved unmerged chunks of {} to the new file", taskName, seqFile);
-
- seqFile.getMergeQueryLock().writeLock().lock();
- try {
- seqFile.getFile().delete();
- FileUtils.moveFile(fileWriter.getFile(), seqFile.getFile());
- } finally {
- seqFile.getMergeQueryLock().writeLock().unlock();
- }
- }
-
- private long writeUnmergedChunks(List<Long> chunkStartTimes,
- List<ChunkMetaData> chunkMetaDataList, ChunkLoader chunkLoader, IChunkWriter chunkWriter,
- MeasurementSchema measurementSchema, RestorableTsFileIOWriter fileWriter) throws IOException {
- long maxVersion = 0;
- int chunkIdx = 0;
- for (Long startTime : chunkStartTimes) {
- for (; chunkIdx < chunkMetaDataList.size(); chunkIdx ++) {
- ChunkMetaData metaData = chunkMetaDataList.get(chunkIdx);
- if (metaData.getStartTime() == startTime) {
- Chunk chunk = chunkLoader.getChunk(metaData);
- writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
- chunkWriter.writeToFileWriter(fileWriter);
- maxVersion = metaData.getVersion() > maxVersion ? metaData.getVersion() : maxVersion;
- break;
- }
- }
- }
- return maxVersion;
- }
-
- private void mergeOnePath(Path path) throws IOException {
- IPointReader unseqReader = getUnseqReader(path);
- MeasurementSchema schema = getSchema(path);
- try {
- if (unseqReader.hasNext()) {
- currTimeValuePair = unseqReader.next();
- }
- for (int i = 0; i < seqFiles.size(); i++) {
- pathMergeOneFile(path, i, unseqReader, schema);
- }
- } catch (IOException e) {
- logger.error("Cannot read unseq data of {} during merge", path, e);
- } finally {
- try {
- unseqReader.close();
- } catch (IOException e) {
- logger.error("Cannot close unseqReader when merging path {}", path, e);
- }
- }
- }
-
- private void pathMergeOneFile(Path path, int seqFileIdx, IPointReader unseqReader,
- MeasurementSchema measurementSchema)
- throws IOException {
- TsFileResource currTsFile = seqFiles.get(seqFileIdx);
- unmergedChunkStartTimes.get(currTsFile).put(path, new ArrayList<>());
-
- // if this TsFile receives data later than fileLimitTime, it will overlap the next TsFile,
- // which is forbidden
- String deviceId = path.getDevice();
- long fileLimitTime = Long.MAX_VALUE;
- for (int i = seqFileIdx + 1; i < seqFiles.size(); i++) {
- Long nextStartTime = seqFiles.get(i).getStartTimeMap().get(deviceId);
- if (nextStartTime != null) {
- fileLimitTime = nextStartTime;
- break;
- }
- }
-
- TsFileSequenceReader fileSequenceReader = getFileReader(currTsFile);
- ChunkLoader chunkLoader = new ChunkLoaderImpl(fileSequenceReader);
- List<Modification> modifications = getModifications(currTsFile, path);
- List<ChunkMetaData> seqChunkMeta = queryChunkMetadata(path, currTsFile);
- if (seqChunkMeta.isEmpty()) {
- return;
- }
- modifyChunkMetaData(seqChunkMeta, modifications);
- RestorableTsFileIOWriter mergeFileWriter = getMergeFileWriter(currTsFile);
-
- currDeviceMaxTime = currTsFile.getEndTimeMap().get(path.getDevice());
- // merge unseq data with this file or small chunks in this file into a larger chunk
- mergeFileWriter.startChunkGroup(deviceId);
- if (mergeChunks(seqChunkMeta, fileLimitTime, chunkLoader, measurementSchema,
- unseqReader, mergeFileWriter, currTsFile, path)) {
- mergeFileWriter.endChunkGroup(seqChunkMeta.get(seqChunkMeta.size() - 1).getVersion() + 1);
- mergeLogger.logFilePositionUpdate(mergeFileWriter.getFile());
- }
- currTsFile.updateTime(path.getDevice(), currDeviceMaxTime);
- }
-
- private boolean mergeChunks(List<ChunkMetaData> seqChunkMeta, long fileLimitTime,
- ChunkLoader chunkLoader, MeasurementSchema measurementSchema,
- IPointReader unseqReader, RestorableTsFileIOWriter mergeFileWriter, TsFileResource currFile
- , Path path)
- throws IOException {
- int ptWritten = 0;
- IChunkWriter chunkWriter = getChunkWriter(measurementSchema);
- int mergedChunkNum = 0;
- int unmergedChunkNum = 0;
- for (int i = 0; i < seqChunkMeta.size(); i++) {
- ChunkMetaData currMeta = seqChunkMeta.get(i);
- boolean isLastChunk = i + 1 == seqChunkMeta.size();
- long chunkLimitTime = i + 1 < seqChunkMeta.size() ? seqChunkMeta.get(i + 1).getStartTime()
- : fileLimitTime;
-
- int newPtWritten = writeChunk(chunkLimitTime, ptWritten, chunkLoader, currMeta, chunkWriter,
- isLastChunk,
- measurementSchema, unseqReader);
-
- if (newPtWritten > ptWritten) {
- mergedChunkNum ++;
- ptWritten = newPtWritten;
- } else {
- unmergedChunkNum ++;
- unmergedChunkStartTimes.get(currFile).get(path).add(currMeta.getStartTime());
- }
-
- if (minChunkPointNum >= 0 && ptWritten >= minChunkPointNum || ptWritten > 0 && minChunkPointNum < 0) {
- // the new chunk's size is large enough and it should be flushed
- chunkWriter.writeToFileWriter(mergeFileWriter);
- ptWritten = 0;
- }
- }
- if (ptWritten > 0) {
- // the last merged chunk may still be smaller than the threshold, flush it anyway
- chunkWriter.writeToFileWriter(mergeFileWriter);
- }
- updateChunkCounts(currFile, mergedChunkNum, unmergedChunkNum);
-
- return mergedChunkNum > 0;
- }
-
- private void updateChunkCounts(TsFileResource currFile, int newMergedChunkNum,
- int newUnmergedChunkNum) {
- mergedChunkCnt.compute(currFile, (tsFileResource, anInt) -> anInt == null ? newMergedChunkNum
- : anInt + newMergedChunkNum);
- unmergedChunkCnt.compute(currFile, (tsFileResource, anInt) -> anInt == null ? newUnmergedChunkNum
- : anInt + newUnmergedChunkNum);
- }
-
- private int writeChunk(long chunkLimitTime, int ptWritten, ChunkLoader chunkLoader,
- ChunkMetaData currMeta, IChunkWriter chunkWriter, boolean isLastChunk,
- MeasurementSchema measurementSchema, IPointReader unseqReader) throws IOException {
-
- // unseq data is not over and this chunk's time range cover the overflow point
- boolean chunkOverlap =
- currTimeValuePair != null && currTimeValuePair.getTimestamp() < chunkLimitTime;
- // a small chunk has been written, this chunk merge with it to create a larger chunk
- // or this chunk is too small and it is not the last chunk, merge it with the next chunk
- boolean chunkTooSmall =
- ptWritten > 0 || (minChunkPointNum >= 0 && currMeta.getNumOfPoints() < minChunkPointNum && !isLastChunk);
- boolean chunkModified = currMeta.getDeletedAt() > Long.MIN_VALUE;
- int newPtWritten = ptWritten;
-
- if (!chunkOverlap && (chunkTooSmall || chunkModified || fullMerge)) {
- // just rewrite the (modified) chunk
- Chunk chunk = chunkLoader.getChunk(currMeta);
- newPtWritten += writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
- } else if (chunkOverlap) {
- // this chunk may merge with the current point
- Chunk chunk = chunkLoader.getChunk(currMeta);
- newPtWritten += writeChunkWithUnseq(chunk, chunkWriter, measurementSchema.getType(),
- unseqReader, chunkLimitTime);
- }
- return newPtWritten;
- }
-
- private int writeChunkWithUnseq(Chunk chunk, IChunkWriter chunkWriter,
- TSDataType dataType, IPointReader unseqReader, long chunkLimitTime) throws IOException {
- int cnt = 0;
- ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
- while (chunkReader.hasNextBatch()) {
- BatchData batchData = chunkReader.nextBatch();
- cnt += mergeWriteBatch(batchData, chunkWriter, dataType, unseqReader);
- }
- while (currTimeValuePair != null && currTimeValuePair.getTimestamp() < chunkLimitTime) {
- writeTVPair(currTimeValuePair, chunkWriter, dataType);
- if (currTimeValuePair.getTimestamp() > currDeviceMaxTime) {
- currDeviceMaxTime = currTimeValuePair.getTimestamp();
- }
- currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
- cnt ++;
- }
- totalChunkWritten ++;
- return cnt;
- }
-
- private int mergeWriteBatch(BatchData batchData, IChunkWriter chunkWriter,
- TSDataType dataType, IPointReader unseqReader) throws IOException {
- int cnt = 0;
- for (int i = 0; i < batchData.length(); i++) {
- long time = batchData.getTimeByIndex(i);
- // merge data in batch and data in unseqReader
- while (currTimeValuePair != null && currTimeValuePair.getTimestamp() < time) {
- writeTVPair(currTimeValuePair, chunkWriter, dataType);
- currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
- cnt++;
- }
- if (currTimeValuePair != null && currTimeValuePair.getTimestamp() == time) {
- writeTVPair(currTimeValuePair, chunkWriter, dataType);
- currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
- cnt++;
- } else {
- writeBatchPoint(batchData, i, dataType, chunkWriter);
- cnt++;
- }
- }
- return cnt;
- }
-
- private int writeChunkWithoutUnseq(Chunk chunk, IChunkWriter chunkWriter,
- MeasurementSchema measurementSchema) throws IOException {
- ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
- TSDataType dataType = measurementSchema.getType();
- while (chunkReader.hasNextBatch()) {
- BatchData batchData = chunkReader.nextBatch();
- for (int i = 0; i < batchData.length(); i++) {
- writeBatchPoint(batchData, i, dataType, chunkWriter);
- }
- }
- totalChunkWritten ++;
- return chunk.getHeader().getNumOfPages();
- }
-
- private TsFileSequenceReader getFileReader(TsFileResource tsFileResource) throws IOException {
- TsFileSequenceReader reader = fileReaderCache.get(tsFileResource);
- if (reader == null) {
- reader = new TsFileSequenceReader(tsFileResource.getFile().getPath());
- fileReaderCache.put(tsFileResource, reader);
- }
- return reader;
- }
-
- private List<Modification> getModifications(TsFileResource tsFileResource, Path path) {
- List<Modification> modifications = modificationCache.computeIfAbsent(tsFileResource,
- tsFileResource1 -> (List<Modification>) tsFileResource.getModFile().getModifications());
- List<Modification> pathModifications = new ArrayList<>();
- Iterator<Modification> modificationIterator = modifications.iterator();
- // each path is visited only once in a merge, so the modifications can be removed after visiting
- while (modificationIterator.hasNext()) {
- Modification modification = modificationIterator.next();
- if (modification.getPath().equals(path)) {
- pathModifications.add(modification);
- modificationIterator.remove();
- }
- }
- return pathModifications;
- }
-
- List<Path> collectPaths() throws IOException {
- Set<Path> pathSet = new HashSet<>();
- for (TsFileResource tsFileResource : unseqFiles) {
- TsFileSequenceReader sequenceReader = getFileReader(tsFileResource);
- measurementSchemaMap.putAll(sequenceReader.readFileMetadata().getMeasurementSchema());
- pathSet.addAll(collectFileSeries(sequenceReader));
- }
- for (TsFileResource tsFileResource : seqFiles) {
- TsFileSequenceReader sequenceReader = getFileReader(tsFileResource);
- measurementSchemaMap.putAll(sequenceReader.readFileMetadata().getMeasurementSchema());
- pathSet.addAll(collectFileSeries(sequenceReader));
- }
- List<Path> ret = new ArrayList<>(pathSet);
- ret.sort(Comparator.comparing(Path::getFullPath));
- return ret;
- }
-
- private MeasurementSchema getSchema(Path path) {
- return measurementSchemaMap.get(path.getMeasurement());
- }
-
- RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource) throws IOException {
- RestorableTsFileIOWriter writer = fileWriterCache.get(resource);
- if (writer == null) {
- writer = new RestorableTsFileIOWriter(new File(resource.getFile().getPath() + MERGE_SUFFIX));
- fileWriterCache.put(resource, writer);
- }
- return writer;
- }
-
- private MetadataQuerier getMetadataQuerier(TsFileResource seqFile) throws IOException {
- MetadataQuerier metadataQuerier = metadataQuerierCache.get(seqFile);
- if (metadataQuerier == null) {
- metadataQuerier = new MetadataQuerierByFileImpl(getFileReader(seqFile));
- metadataQuerierCache.put(seqFile, metadataQuerier);
- }
- return metadataQuerier;
- }
-
- List<ChunkMetaData> queryChunkMetadata(Path path, TsFileResource seqFile)
- throws IOException {
- MetadataQuerier metadataQuerier = getMetadataQuerier(seqFile);
- List<ChunkMetaData> chunkMetaDataList = metadataQuerier.getChunkMetaDataList(path);
- if (!keepChunkMetadata) {
- metadataQuerier.clear();
- }
- return chunkMetaDataList;
- }
-
- private IPointReader getUnseqReader(Path path) throws IOException {
- return SeriesReaderFactoryImpl.getInstance().createUnseqSeriesReader(path, unseqFiles,
- mergeContext, null);
- }
-
- private IChunkWriter getChunkWriter(MeasurementSchema measurementSchema) {
- return new ChunkWriterImpl(measurementSchema,
- new ChunkBuffer(measurementSchema), TSFileConfig.pageCheckSizeThreshold);
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java
similarity index 97%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeManager.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java
index 56c6abb..98132c1 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeManager.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.engine.merge.manage;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
new file mode 100644
index 0000000..d8cc445
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.manage;
+
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.factory.SeriesReaderFactoryImpl;
+import org.apache.iotdb.db.query.reader.IPointReader;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.controller.MetadataQuerier;
+import org.apache.iotdb.tsfile.read.controller.MetadataQuerierByFileImpl;
+import org.apache.iotdb.tsfile.write.chunk.ChunkBuffer;
+import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+
+public class MergeResource {
+
+ private List<TsFileResource> seqFiles;
+ private List<TsFileResource> unseqFiles;
+
+ // future feature
+ private boolean keepChunkMetadata = false;
+
+ private QueryContext mergeContext = new QueryContext();
+
+ private Map<TsFileResource, TsFileSequenceReader> fileReaderCache = new HashMap<>();
+ private Map<TsFileResource, RestorableTsFileIOWriter> fileWriterCache = new HashMap<>();
+ private Map<TsFileResource, List<Modification>> modificationCache = new HashMap<>();
+ private Map<TsFileResource, MetadataQuerier> metadataQuerierCache = new HashMap<>();
+ private Map<String, MeasurementSchema> measurementSchemaMap = new HashMap<>();
+
+ public MergeResource(
+ List<TsFileResource> seqFiles,
+ List<TsFileResource> unseqFiles) {
+ this.seqFiles = seqFiles;
+ this.unseqFiles = unseqFiles;
+ }
+
+ public void clear() throws IOException {
+ for (TsFileSequenceReader sequenceReader : fileReaderCache.values()) {
+ sequenceReader.close();
+ }
+
+ fileReaderCache.clear();
+ fileWriterCache.clear();
+ modificationCache.clear();
+ metadataQuerierCache.clear();
+ }
+
+ public MeasurementSchema getSchema(Path path) {
+ return measurementSchemaMap.get(path.getMeasurement());
+ }
+
+ public RestorableTsFileIOWriter getMergeFileWriter(TsFileResource resource) throws IOException {
+ RestorableTsFileIOWriter writer = fileWriterCache.get(resource);
+ if (writer == null) {
+ writer = new RestorableTsFileIOWriter(new File(resource.getFile().getPath() + MERGE_SUFFIX));
+ fileWriterCache.put(resource, writer);
+ }
+ return writer;
+ }
+
+ private MetadataQuerier getMetadataQuerier(TsFileResource seqFile) throws IOException {
+ MetadataQuerier metadataQuerier = metadataQuerierCache.get(seqFile);
+ if (metadataQuerier == null) {
+ metadataQuerier = new MetadataQuerierByFileImpl(getFileReader(seqFile));
+ metadataQuerierCache.put(seqFile, metadataQuerier);
+ }
+ return metadataQuerier;
+ }
+
+ public List<ChunkMetaData> queryChunkMetadata(Path path, TsFileResource seqFile)
+ throws IOException {
+ MetadataQuerier metadataQuerier = getMetadataQuerier(seqFile);
+ List<ChunkMetaData> chunkMetaDataList = metadataQuerier.getChunkMetaDataList(path);
+ if (!keepChunkMetadata) {
+ metadataQuerier.clear();
+ }
+ return chunkMetaDataList;
+ }
+
+ public IPointReader getUnseqReader(Path path) throws IOException {
+ return SeriesReaderFactoryImpl.getInstance().createUnseqSeriesReader(path, unseqFiles,
+ mergeContext, null);
+ }
+
+ public IChunkWriter getChunkWriter(MeasurementSchema measurementSchema) {
+ return new ChunkWriterImpl(measurementSchema,
+ new ChunkBuffer(measurementSchema), TSFileConfig.pageCheckSizeThreshold);
+ }
+
+ public TsFileSequenceReader getFileReader(TsFileResource tsFileResource) throws IOException {
+ TsFileSequenceReader reader = fileReaderCache.get(tsFileResource);
+ if (reader == null) {
+ reader = new TsFileSequenceReader(tsFileResource.getFile().getPath());
+ fileReaderCache.put(tsFileResource, reader);
+ }
+ return reader;
+ }
+
+ public List<Modification> getModifications(TsFileResource tsFileResource, Path path) {
+ // copy from TsFileResource so queries are not affected
+ List<Modification> modifications = modificationCache.computeIfAbsent(tsFileResource,
+ tsFileResource1 -> new ArrayList<>(tsFileResource.getModFile().getModifications()));
+ List<Modification> pathModifications = new ArrayList<>();
+ Iterator<Modification> modificationIterator = modifications.iterator();
+ // each path is visited only once in a merge, so the modifications can be removed after visiting
+ while (modificationIterator.hasNext()) {
+ Modification modification = modificationIterator.next();
+ if (modification.getPath().equals(path)) {
+ pathModifications.add(modification);
+ modificationIterator.remove();
+ }
+ }
+ return pathModifications;
+ }
+
+ public void removeFileWriter(TsFileResource tsFileResource) throws IOException {
+ RestorableTsFileIOWriter newFileWriter = fileWriterCache.remove(tsFileResource);
+ if (newFileWriter != null) {
+ newFileWriter.close();
+ newFileWriter.getFile().delete();
+ fileWriterCache.remove(tsFileResource);
+ }
+ }
+
+ public List<TsFileResource> getSeqFiles() {
+ return seqFiles;
+ }
+
+ public List<TsFileResource> getUnseqFiles() {
+ return unseqFiles;
+ }
+
+ public void setSeqFiles(List<TsFileResource> seqFiles) {
+ this.seqFiles = seqFiles;
+ }
+
+ public void setUnseqFiles(
+ List<TsFileResource> unseqFiles) {
+ this.unseqFiles = unseqFiles;
+ }
+
+ public Map<String, MeasurementSchema> getMeasurementSchemaMap() {
+ return measurementSchemaMap;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeLogger.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java
similarity index 64%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeLogger.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java
index 48ab20e..3e38936 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeLogger.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/recover/MergeLogger.java
@@ -17,31 +17,32 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.engine.merge.recover;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.read.common.Path;
public class MergeLogger {
- static final String MERGE_LOG_NAME = "merge.log";
+ public static final String MERGE_LOG_NAME = "merge.log";
- static final String STR_SEQ_FILES = "seqFiles";
- static final String STR_UNSEQ_FILES = "unseqFiles";
- static final String STR_START = "start";
- static final String STR_END = "end";
- static final String STR_ALL_TS_END = "all ts end";
- static final String STR_MERGE_START = "merge start";
- static final String STR_MERGE_END = "merge end";
+ public static final String STR_SEQ_FILES = "seqFiles";
+ public static final String STR_UNSEQ_FILES = "unseqFiles";
+ public static final String STR_START = "start";
+ public static final String STR_END = "end";
+ public static final String STR_ALL_TS_END = "all ts end";
+ public static final String STR_MERGE_START = "merge start";
+ public static final String STR_MERGE_END = "merge end";
private BufferedWriter logStream;
- MergeLogger(String storageGroupDir) throws IOException {
+ public MergeLogger(String storageGroupDir) throws IOException {
logStream = new BufferedWriter(new FileWriter(new File(storageGroupDir, MERGE_LOG_NAME), true));
}
@@ -49,7 +50,7 @@ public class MergeLogger {
logStream.close();
}
- void logSeqFiles(List<TsFileResource> seqFiles) throws IOException {
+ private void logSeqFiles(List<TsFileResource> seqFiles) throws IOException {
logStream.write(STR_SEQ_FILES);
logStream.newLine();
for (TsFileResource tsFileResource : seqFiles) {
@@ -59,7 +60,7 @@ public class MergeLogger {
logStream.flush();
}
- void logUnseqFiles(List<TsFileResource> unseqFiles) throws IOException {
+ private void logUnseqFiles(List<TsFileResource> unseqFiles) throws IOException {
logStream.write(STR_UNSEQ_FILES);
logStream.newLine();
for (TsFileResource tsFileResource : unseqFiles) {
@@ -69,51 +70,57 @@ public class MergeLogger {
logStream.flush();
}
- void logMergeStart() throws IOException {
+ private void logMergeStart() throws IOException {
logStream.write(STR_MERGE_START);
logStream.newLine();
logStream.flush();
}
- void logTSStart(Path path) throws IOException {
+ public void logTSStart(Path path) throws IOException {
logStream.write(path.getFullPath() + " " + STR_START);
logStream.newLine();
logStream.flush();
}
- void logFilePositionUpdate(File file) throws IOException {
+ public void logFilePositionUpdate(File file) throws IOException {
logStream.write(String.format("%s %d", file.getAbsolutePath(), file.length()));
logStream.newLine();
logStream.flush();
}
- void logTSEnd(Path path) throws IOException {
+ public void logTSEnd(Path path) throws IOException {
logStream.write(path.getFullPath() + " " + STR_END);
logStream.newLine();
logStream.flush();
}
- void logAllTsEnd() throws IOException {
+ public void logAllTsEnd() throws IOException {
logStream.write(STR_ALL_TS_END);
logStream.newLine();
logStream.flush();
}
- void logFileMergeStart(File file, long position) throws IOException {
+ public void logFileMergeStart(File file, long position) throws IOException {
logStream.write(String.format("%s %d", file.getAbsolutePath(), position));
logStream.newLine();
logStream.flush();
}
- void logFileMergeEnd(File file) throws IOException {
+ public void logFileMergeEnd(File file) throws IOException {
logStream.write(file.getAbsolutePath() + " " + STR_END);
logStream.newLine();
logStream.flush();
}
- void logMergeEnd() throws IOException {
+ public void logMergeEnd() throws IOException {
logStream.write(STR_MERGE_END);
logStream.newLine();
logStream.flush();
}
+
+ public void logFiles(MergeResource resource) throws IOException {
+ logSeqFiles(resource.getSeqFiles());
+ logUnseqFiles(resource.getUnseqFiles());
+ logMergeStart();
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeFileSelector.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileSelector.java
similarity index 99%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeFileSelector.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileSelector.java
index 5e89370..136a4f2 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeFileSelector.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/selector/MergeFileSelector.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.engine.merge.selector;
-import static org.apache.iotdb.db.engine.merge.MergeUtils.collectFileSeries;
+import static org.apache.iotdb.db.utils.MergeUtils.collectFileSeries;
import java.io.IOException;
import java.util.ArrayList;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeCallback.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java
similarity index 96%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeCallback.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java
index 302455f..7377a2c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeCallback.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeCallback.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.engine.merge.task;
import java.io.File;
import java.util.List;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
new file mode 100644
index 0000000..af129cb
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import static org.apache.iotdb.db.utils.MergeUtils.writeChunkWithoutUnseq;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.exception.write.TsFileNotCompleteException;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.FileSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.ForceAppendTsFileWriter;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class MergeFileTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeFileTask.class);
+
+ private String taskName;
+ private Map<TsFileResource, Integer> mergedChunkCnt;
+ private Map<TsFileResource, Integer> unmergedChunkCnt;
+ private Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes;
+ private MergeLogger mergeLogger;
+ private MergeResource resource;
+ private List<TsFileResource> unmergedFiles;
+
+ MergeFileTask(String taskName,
+ Map<TsFileResource, Integer> mergedChunkCnt,
+ Map<TsFileResource, Integer> unmergedChunkCnt,
+ Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes,
+ MergeLogger mergeLogger, MergeResource resource, List<TsFileResource> unmergedSeqFiles) {
+ this.taskName = taskName;
+ this.mergedChunkCnt = mergedChunkCnt;
+ this.unmergedChunkCnt = unmergedChunkCnt;
+ this.unmergedChunkStartTimes = unmergedChunkStartTimes;
+ this.mergeLogger = mergeLogger;
+ this.resource = resource;
+ this.unmergedFiles = unmergedSeqFiles;
+ }
+
+ void mergeFiles() throws IOException {
+ // decide whether to write the unmerged chunks to the merge files or to move the merged data
+ // back to the origin seqFile's
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} files", taskName, unmergedFiles.size());
+ }
+ long startTime = System.currentTimeMillis();
+ int cnt = 0;
+ for (TsFileResource seqFile : unmergedFiles) {
+ int mergedChunkNum = mergedChunkCnt.getOrDefault(seqFile, 0);
+ int unmergedChunkNum = unmergedChunkCnt.getOrDefault(seqFile, 0);
+ if (mergedChunkNum >= unmergedChunkNum) {
+ // move the unmerged data to the new file
+ if (logger.isInfoEnabled()) {
+ logger.info("{} moving unmerged data of {} to the merged file, {} merged chunks, {} "
+ + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
+ }
+ moveUnmergedToNew(seqFile);
+ } else {
+ // move the merged data to the old file
+ if (logger.isInfoEnabled()) {
+ logger.info("{} moving merged data of {} to the old file {} merged chunks, {} "
+ + "unmerged chunks", taskName, seqFile.getFile().getName(), mergedChunkNum, unmergedChunkNum);
+ }
+ moveMergedToOld(seqFile);
+ }
+ cnt ++;
+ if (logger.isInfoEnabled()) {
+ logger.debug("{} has merged {}/{} files", taskName, cnt, unmergedFiles.size());
+ }
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("{} has merged all files after {}ms", taskName, System.currentTimeMillis() - startTime);
+ }
+ mergeLogger.logMergeEnd();
+ }
+
+ private void moveMergedToOld(TsFileResource seqFile) throws IOException {
+ int mergedChunkNum = mergedChunkCnt.getOrDefault(seqFile, 0);
+ if (mergedChunkNum == 0) {
+ resource.removeFileWriter(seqFile);
+ return;
+ }
+
+ seqFile.getMergeQueryLock().writeLock().lock();
+ try {
+ TsFileIOWriter oldFileWriter;
+ try {
+ oldFileWriter = new ForceAppendTsFileWriter(seqFile.getFile());
+ mergeLogger.logFileMergeStart(seqFile.getFile(), ((ForceAppendTsFileWriter) oldFileWriter).getTruncatePosition());
+ logger.debug("{} moving merged chunks of {} to the old file", taskName, seqFile);
+ ((ForceAppendTsFileWriter) oldFileWriter).doTruncate();
+ } catch (TsFileNotCompleteException e) {
+ // this file may already be truncated if this merge is a system reboot merge
+ oldFileWriter = new RestorableTsFileIOWriter(seqFile.getFile());
+ }
+
+ oldFileWriter.filterChunks(unmergedChunkStartTimes.get(seqFile));
+
+ RestorableTsFileIOWriter newFileWriter = resource.getMergeFileWriter(seqFile);
+ newFileWriter.close();
+ try ( TsFileSequenceReader newFileReader =
+ new TsFileSequenceReader(newFileWriter.getFile().getPath())) {
+ ChunkLoader chunkLoader = new ChunkLoaderImpl(newFileReader);
+ List<ChunkGroupMetaData> chunkGroupMetadataList = newFileWriter.getChunkGroupMetaDatas();
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} find {} merged chunk groups", taskName, chunkGroupMetadataList.size());
+ }
+ for (ChunkGroupMetaData chunkGroupMetaData : chunkGroupMetadataList) {
+ writeMergedChunkGroup(chunkGroupMetaData, chunkLoader, oldFileWriter);
+ }
+ }
+ oldFileWriter.endFile(new FileSchema(oldFileWriter.getKnownSchema()));
+
+ seqFile.serialize();
+ mergeLogger.logFileMergeEnd(seqFile.getFile());
+ logger.debug("{} moved merged chunks of {} to the old file", taskName, seqFile);
+
+ newFileWriter.getFile().delete();
+ } finally {
+ seqFile.getMergeQueryLock().writeLock().unlock();
+ }
+ }
+
+ private void writeMergedChunkGroup(ChunkGroupMetaData chunkGroupMetaData,
+ ChunkLoader chunkLoader, TsFileIOWriter fileWriter)
+ throws IOException {
+ fileWriter.startChunkGroup(chunkGroupMetaData.getDeviceID());
+ long maxVersion = Long.MIN_VALUE;
+ for (ChunkMetaData chunkMetaData : chunkGroupMetaData.getChunkMetaDataList()) {
+ Path path = new Path(chunkGroupMetaData.getDeviceID(), chunkMetaData.getMeasurementUid());
+ MeasurementSchema measurementSchema = resource.getSchema(path);
+ IChunkWriter chunkWriter = resource.getChunkWriter(measurementSchema);
+ Chunk chunk = chunkLoader.getChunk(chunkMetaData);
+ writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
+ chunkWriter.writeToFileWriter(fileWriter);
+ maxVersion = maxVersion < chunkMetaData.getVersion() ? chunkMetaData.getVersion() :
+ maxVersion;
+ }
+ fileWriter.endChunkGroup(maxVersion + 1);
+ }
+
+ private void moveUnmergedToNew(TsFileResource seqFile) throws IOException {
+ Map<Path, List<Long>> fileUnmergedChunkStartTimes = this.unmergedChunkStartTimes.get(seqFile);
+ RestorableTsFileIOWriter fileWriter = resource.getMergeFileWriter(seqFile);
+ ChunkLoader chunkLoader = new ChunkLoaderImpl(resource.getFileReader(seqFile));
+
+ mergeLogger.logFileMergeStart(fileWriter.getFile(), fileWriter.getFile().length());
+ logger.debug("{} moving unmerged chunks of {} to the new file", taskName, seqFile);
+
+ int unmergedChunkNum = unmergedChunkCnt.getOrDefault(seqFile, 0);
+
+ if (unmergedChunkNum > 0) {
+ for (Entry<Path, List<Long>> entry : fileUnmergedChunkStartTimes.entrySet()) {
+ Path path = entry.getKey();
+ List<Long> chunkStartTimes = entry.getValue();
+ if (chunkStartTimes.isEmpty()) {
+ continue;
+ }
+
+ List<ChunkMetaData> chunkMetaDataList = resource.queryChunkMetadata(path, seqFile);
+ MeasurementSchema measurementSchema = resource.getSchema(path);
+ IChunkWriter chunkWriter = resource.getChunkWriter(measurementSchema);
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("{} find {} unmerged chunks", taskName, chunkMetaDataList.size());
+ }
+
+ fileWriter.startChunkGroup(path.getDevice());
+ long maxVersion = writeUnmergedChunks(chunkStartTimes, chunkMetaDataList, chunkLoader,
+ chunkWriter, measurementSchema, fileWriter);
+ fileWriter.endChunkGroup(maxVersion + 1);
+ }
+ }
+
+ fileWriter.endFile(new FileSchema(fileWriter.getKnownSchema()));
+
+ seqFile.serialize();
+ mergeLogger.logFileMergeEnd(fileWriter.getFile());
+ logger.debug("{} moved unmerged chunks of {} to the new file", taskName, seqFile);
+
+ seqFile.getMergeQueryLock().writeLock().lock();
+ try {
+ seqFile.getFile().delete();
+ FileUtils.moveFile(fileWriter.getFile(), seqFile.getFile());
+ } finally {
+ seqFile.getMergeQueryLock().writeLock().unlock();
+ }
+ }
+
+ private long writeUnmergedChunks(List<Long> chunkStartTimes,
+ List<ChunkMetaData> chunkMetaDataList, ChunkLoader chunkLoader, IChunkWriter chunkWriter,
+ MeasurementSchema measurementSchema, RestorableTsFileIOWriter fileWriter) throws IOException {
+ long maxVersion = 0;
+ int chunkIdx = 0;
+ for (Long startTime : chunkStartTimes) {
+ for (; chunkIdx < chunkMetaDataList.size(); chunkIdx ++) {
+ ChunkMetaData metaData = chunkMetaDataList.get(chunkIdx);
+ if (metaData.getStartTime() == startTime) {
+ Chunk chunk = chunkLoader.getChunk(metaData);
+ writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
+ chunkWriter.writeToFileWriter(fileWriter);
+ maxVersion = metaData.getVersion() > maxVersion ? metaData.getVersion() : maxVersion;
+ break;
+ }
+ }
+ }
+ return maxVersion;
+ }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeSeriesTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeSeriesTask.java
new file mode 100644
index 0000000..e6ec446
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeSeriesTask.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import static org.apache.iotdb.db.utils.MergeUtils.writeBatchPoint;
+import static org.apache.iotdb.db.utils.MergeUtils.writeTVPair;
+import static org.apache.iotdb.db.utils.QueryUtils.modifyChunkMetaData;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.query.reader.IPointReader;
+import org.apache.iotdb.db.utils.TimeValuePair;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoader;
+import org.apache.iotdb.tsfile.read.controller.ChunkLoaderImpl;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
+import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class MergeSeriesTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(MergeSeriesTask.class);
+ private static int minChunkPointNum = IoTDBDescriptor.getInstance().getConfig().getChunkMergePointThreshold();
+
+ private Map<TsFileResource, Integer> mergedChunkCnt;
+ private Map<TsFileResource, Integer> unmergedChunkCnt;
+ private Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes;
+ private MergeLogger mergeLogger;
+ private List<Path> unmergedSeries;
+
+ private String taskName;
+ private MergeResource resource;
+ private TimeValuePair currTimeValuePair;
+ private long currDeviceMaxTime;
+ private boolean fullMerge;
+
+ int totalChunkWritten;
+
+ MergeSeriesTask(
+ Map<TsFileResource, Integer> mergedChunkCnt,
+ Map<TsFileResource, Integer> unmergedChunkCnt,
+ Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes,
+ String taskName, MergeLogger mergeLogger, MergeResource mergeResource, boolean fullMerge,
+ List<Path> unmergedSeries) {
+ this.mergedChunkCnt = mergedChunkCnt;
+ this.unmergedChunkCnt = unmergedChunkCnt;
+ this.unmergedChunkStartTimes = unmergedChunkStartTimes;
+ this.taskName = taskName;
+ this.mergeLogger = mergeLogger;
+ this.resource = mergeResource;
+ this.fullMerge = fullMerge;
+ this.unmergedSeries = unmergedSeries;
+ }
+
+ void mergeSeries() throws IOException {
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} series", taskName, unmergedSeries.size());
+ }
+ long startTime = System.currentTimeMillis();
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ unmergedChunkStartTimes.put(seqFile, new HashMap<>());
+ }
+ // merge each series and write data into each seqFile's temp merge file
+ int mergedCnt = 0;
+ double progress = 0.0;
+ for (Path path : unmergedSeries) {
+ mergeLogger.logTSStart(path);
+ mergeOnePath(path);
+ mergeLogger.logTSEnd(path);
+ mergedCnt ++;
+ if (logger.isInfoEnabled()) {
+ double newProgress = 100 * mergedCnt / (double) (unmergedSeries.size());
+ if (newProgress - progress >= 0.01) {
+ progress = newProgress;
+ logger.info("{} has merged {}% series", taskName, progress);
+ }
+ }
+ }
+ if (logger.isInfoEnabled()) {
+ logger.info("{} all series are merged after {}ms", taskName, System.currentTimeMillis() - startTime);
+ }
+ mergeLogger.logAllTsEnd();
+ }
+
+ private void mergeOnePath(Path path) throws IOException {
+ IPointReader unseqReader = resource.getUnseqReader(path);
+ MeasurementSchema schema = resource.getSchema(path);
+ try {
+ if (unseqReader.hasNext()) {
+ currTimeValuePair = unseqReader.next();
+ }
+ for (int i = 0; i < resource.getSeqFiles().size(); i++) {
+ pathMergeOneFile(path, i, unseqReader, schema);
+ }
+ } catch (IOException e) {
+ logger.error("Cannot read unseq data of {} during merge", path, e);
+ } finally {
+ try {
+ unseqReader.close();
+ } catch (IOException e) {
+ logger.error("Cannot close unseqReader when merging path {}", path, e);
+ }
+ }
+ }
+
+ private void pathMergeOneFile(Path path, int seqFileIdx, IPointReader unseqReader,
+ MeasurementSchema measurementSchema)
+ throws IOException {
+ TsFileResource currTsFile = resource.getSeqFiles().get(seqFileIdx);
+ unmergedChunkStartTimes.get(currTsFile).put(path, new ArrayList<>());
+
+ // if this TsFile receives data later than fileLimitTime, it will overlap the next TsFile,
+ // which is forbidden
+ String deviceId = path.getDevice();
+ long fileLimitTime = Long.MAX_VALUE;
+ for (int i = seqFileIdx + 1; i < resource.getSeqFiles().size(); i++) {
+ Long nextStartTime = resource.getSeqFiles().get(i).getStartTimeMap().get(deviceId);
+ if (nextStartTime != null) {
+ fileLimitTime = nextStartTime;
+ break;
+ }
+ }
+
+ TsFileSequenceReader fileSequenceReader = resource.getFileReader(currTsFile);
+ ChunkLoader chunkLoader = new ChunkLoaderImpl(fileSequenceReader);
+ List<Modification> modifications = resource.getModifications(currTsFile, path);
+ List<ChunkMetaData> seqChunkMeta = resource.queryChunkMetadata(path, currTsFile);
+ if (seqChunkMeta.isEmpty()) {
+ return;
+ }
+ modifyChunkMetaData(seqChunkMeta, modifications);
+ RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(currTsFile);
+
+ currDeviceMaxTime = currTsFile.getEndTimeMap().get(path.getDevice());
+ // merge unseq data with this file or small chunks in this file into a larger chunk
+ mergeFileWriter.startChunkGroup(deviceId);
+ if (mergeChunks(seqChunkMeta, fileLimitTime, chunkLoader, measurementSchema,
+ unseqReader, mergeFileWriter, currTsFile, path)) {
+ mergeFileWriter.endChunkGroup(seqChunkMeta.get(seqChunkMeta.size() - 1).getVersion() + 1);
+ mergeLogger.logFilePositionUpdate(mergeFileWriter.getFile());
+ }
+ currTsFile.updateTime(path.getDevice(), currDeviceMaxTime);
+ }
+
+ private boolean mergeChunks(List<ChunkMetaData> seqChunkMeta, long fileLimitTime,
+ ChunkLoader chunkLoader, MeasurementSchema measurementSchema,
+ IPointReader unseqReader, RestorableTsFileIOWriter mergeFileWriter, TsFileResource currFile
+ , Path path)
+ throws IOException {
+ int ptWritten = 0;
+ IChunkWriter chunkWriter = resource.getChunkWriter(measurementSchema);
+ int mergedChunkNum = 0;
+ int unmergedChunkNum = 0;
+ for (int i = 0; i < seqChunkMeta.size(); i++) {
+ ChunkMetaData currMeta = seqChunkMeta.get(i);
+ boolean isLastChunk = i + 1 == seqChunkMeta.size();
+ long chunkLimitTime = i + 1 < seqChunkMeta.size() ? seqChunkMeta.get(i + 1).getStartTime()
+ : fileLimitTime;
+
+ int newPtWritten = writeChunk(chunkLimitTime, ptWritten, chunkLoader, currMeta, chunkWriter,
+ isLastChunk,
+ measurementSchema, unseqReader);
+
+ if (newPtWritten > ptWritten) {
+ mergedChunkNum ++;
+ ptWritten = newPtWritten;
+ } else {
+ unmergedChunkNum ++;
+ unmergedChunkStartTimes.get(currFile).get(path).add(currMeta.getStartTime());
+ }
+
+ if (minChunkPointNum >= 0 && ptWritten >= minChunkPointNum || ptWritten > 0 && minChunkPointNum < 0) {
+ // the new chunk's size is large enough and it should be flushed
+ chunkWriter.writeToFileWriter(mergeFileWriter);
+ ptWritten = 0;
+ }
+ }
+ if (ptWritten > 0) {
+ // the last merged chunk may still be smaller than the threshold, flush it anyway
+ chunkWriter.writeToFileWriter(mergeFileWriter);
+ }
+ updateChunkCounts(currFile, mergedChunkNum, unmergedChunkNum);
+
+ return mergedChunkNum > 0;
+ }
+
+ private void updateChunkCounts(TsFileResource currFile, int newMergedChunkNum,
+ int newUnmergedChunkNum) {
+ mergedChunkCnt.compute(currFile, (tsFileResource, anInt) -> anInt == null ? newMergedChunkNum
+ : anInt + newMergedChunkNum);
+ unmergedChunkCnt.compute(currFile, (tsFileResource, anInt) -> anInt == null ? newUnmergedChunkNum
+ : anInt + newUnmergedChunkNum);
+ }
+
+ private int writeChunk(long chunkLimitTime, int ptWritten, ChunkLoader chunkLoader,
+ ChunkMetaData currMeta, IChunkWriter chunkWriter, boolean isLastChunk,
+ MeasurementSchema measurementSchema, IPointReader unseqReader) throws IOException {
+
+ // unseq data is not over and this chunk's time range cover the overflow point
+ boolean chunkOverlap =
+ currTimeValuePair != null && currTimeValuePair.getTimestamp() < chunkLimitTime;
+ // a small chunk has been written, this chunk merge with it to create a larger chunk
+ // or this chunk is too small and it is not the last chunk, merge it with the next chunk
+ boolean chunkTooSmall =
+ ptWritten > 0 || (minChunkPointNum >= 0 && currMeta.getNumOfPoints() < minChunkPointNum && !isLastChunk);
+ boolean chunkModified = currMeta.getDeletedAt() > Long.MIN_VALUE;
+ int newPtWritten = ptWritten;
+
+ if (!chunkOverlap && (chunkTooSmall || chunkModified || fullMerge)) {
+ // just rewrite the (modified) chunk
+ Chunk chunk = chunkLoader.getChunk(currMeta);
+ newPtWritten += MergeUtils.writeChunkWithoutUnseq(chunk, chunkWriter, measurementSchema);
+ totalChunkWritten ++;
+ } else if (chunkOverlap) {
+ // this chunk may merge with the current point
+ Chunk chunk = chunkLoader.getChunk(currMeta);
+ newPtWritten += writeChunkWithUnseq(chunk, chunkWriter, measurementSchema.getType(),
+ unseqReader, chunkLimitTime);
+ totalChunkWritten ++;
+ }
+ return newPtWritten;
+ }
+
+ private int writeChunkWithUnseq(Chunk chunk, IChunkWriter chunkWriter,
+ TSDataType dataType, IPointReader unseqReader, long chunkLimitTime) throws IOException {
+ int cnt = 0;
+ ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
+ while (chunkReader.hasNextBatch()) {
+ BatchData batchData = chunkReader.nextBatch();
+ cnt += mergeWriteBatch(batchData, chunkWriter, dataType, unseqReader);
+ }
+ while (currTimeValuePair != null && currTimeValuePair.getTimestamp() < chunkLimitTime) {
+ writeTVPair(currTimeValuePair, chunkWriter, dataType);
+ if (currTimeValuePair.getTimestamp() > currDeviceMaxTime) {
+ currDeviceMaxTime = currTimeValuePair.getTimestamp();
+ }
+ currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
+ cnt ++;
+ }
+ return cnt;
+ }
+
+ private int mergeWriteBatch(BatchData batchData, IChunkWriter chunkWriter,
+ TSDataType dataType, IPointReader unseqReader) throws IOException {
+ int cnt = 0;
+ for (int i = 0; i < batchData.length(); i++) {
+ long time = batchData.getTimeByIndex(i);
+ // merge data in batch and data in unseqReader
+ while (currTimeValuePair != null && currTimeValuePair.getTimestamp() < time) {
+ writeTVPair(currTimeValuePair, chunkWriter, dataType);
+ currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
+ cnt++;
+ }
+ if (currTimeValuePair != null && currTimeValuePair.getTimestamp() == time) {
+ writeTVPair(currTimeValuePair, chunkWriter, dataType);
+ currTimeValuePair = unseqReader.hasNext() ? unseqReader.next() : null;
+ cnt++;
+ } else {
+ writeBatchPoint(batchData, i, dataType, chunkWriter);
+ cnt++;
+ }
+ }
+ return cnt;
+ }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
new file mode 100644
index 0000000..43e3d0e
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeTask.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.merge.task;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.utils.MergeUtils;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MergeTask merges given SeqFiles and UnseqFiles into a new one.
+ */
+public class MergeTask implements Callable<Void> {
+
+ public static final String MERGE_SUFFIX = ".merge";
+ private static final Logger logger = LoggerFactory.getLogger(MergeTask.class);
+
+ MergeResource resource;
+ String storageGroupDir;
+ MergeLogger mergeLogger;
+
+ Map<TsFileResource, Integer> mergedChunkCnt = new HashMap<>();
+ Map<TsFileResource, Integer> unmergedChunkCnt = new HashMap<>();
+ Map<TsFileResource, Map<Path, List<Long>>> unmergedChunkStartTimes = new HashMap<>();
+
+ private MergeCallback callback;
+
+ String taskName;
+
+ boolean fullMerge;
+
+ public MergeTask(List<TsFileResource> seqFiles,
+ List<TsFileResource> unseqFiles, String storageGroupDir, MergeCallback callback,
+ String taskName, boolean fullMerge) throws IOException {
+ this.resource = new MergeResource(seqFiles, unseqFiles);
+ this.storageGroupDir = storageGroupDir;
+ this.callback = callback;
+ this.taskName = taskName;
+ this.fullMerge = fullMerge;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ doMerge();
+ } catch (Exception e) {
+ logger.error("Runtime exception in merge {}", taskName, e);
+ resource.setSeqFiles(Collections.emptyList());
+ resource.setUnseqFiles(Collections.emptyList());
+ cleanUp(true);
+ throw e;
+ }
+ return null;
+ }
+
+ private void doMerge() throws IOException {
+ if (logger.isInfoEnabled()) {
+ logger.info("{} starts to merge {} seqFiles, {} unseqFiles", taskName,
+ resource.getSeqFiles().size(), resource.getUnseqFiles().size());
+ }
+ long startTime = System.currentTimeMillis();
+ long totalFileSize = MergeUtils.collectFileSizes(resource.getSeqFiles(),
+ resource.getUnseqFiles());
+ mergeLogger = new MergeLogger(storageGroupDir);
+ mergeLogger.logFiles(resource);
+
+ List<Path> unmergedSeries = MergeUtils.collectPaths(resource);
+ MergeSeriesTask mergeSeriesTask = new MergeSeriesTask(mergedChunkCnt, unmergedChunkCnt,
+ unmergedChunkStartTimes, taskName, mergeLogger, resource, fullMerge, unmergedSeries);
+ mergeSeriesTask.mergeSeries();
+
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName,mergedChunkCnt, unmergedChunkCnt,
+ unmergedChunkStartTimes, mergeLogger, resource, resource.getSeqFiles());
+ mergeFileTask.mergeFiles();
+
+ cleanUp(true);
+ if (logger.isInfoEnabled()) {
+ double elapsedTime = (double) (System.currentTimeMillis() - startTime);
+ double byteRate = totalFileSize / elapsedTime / 1024 / 1024 * 1000;
+ double seriesRate = unmergedSeries.size() / elapsedTime * 1000;
+ double chunkRate = mergeSeriesTask.totalChunkWritten / elapsedTime * 1000;
+ double fileRate =
+ (resource.getSeqFiles().size() + resource.getUnseqFiles().size()) / elapsedTime * 1000;
+ logger.info("{} ends after {}ms, byteRate: {}MB/s, seriesRate {}/s, chunkRate: {}/s, "
+ + "fileRate: {}/s",
+ taskName, elapsedTime, byteRate, seriesRate, chunkRate, fileRate);
+ }
+ }
+
+
+
+ void cleanUp(boolean executeCallback) throws IOException {
+ logger.info("{} is cleaning up", taskName);
+
+ resource.clear();
+
+ mergedChunkCnt.clear();
+ unmergedChunkCnt.clear();
+ unmergedChunkStartTimes.clear();
+
+ mergeLogger.close();
+
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
+ File mergeFile = new File(seqFile.getFile().getPath() + MERGE_SUFFIX);
+ mergeFile.delete();
+ }
+
+ File logFile = new File(storageGroupDir, MergeLogger.MERGE_LOG_NAME);
+ if (executeCallback) {
+ callback.call(resource.getSeqFiles(), resource.getUnseqFiles(), logFile);
+ } else {
+ logFile.delete();
+ }
+ }
+
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/RecoverMergeTask.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java
similarity index 88%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/RecoverMergeTask.java
rename to iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java
index 56e7951..ebcc425 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/RecoverMergeTask.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/task/RecoverMergeTask.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.engine.merge.task;
import java.io.BufferedReader;
import java.io.File;
@@ -31,6 +31,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
+import org.apache.iotdb.db.utils.MergeUtils;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -82,9 +84,13 @@ public class RecoverMergeTask extends MergeTask {
mergeLogger = new MergeLogger(storageGroupDir);
truncateFiles();
recoverChunkCounts();
- mergeSeries(unmergedPaths);
+ MergeSeriesTask mergeSeriesTask = new MergeSeriesTask(mergedChunkCnt, unmergedChunkCnt,
+ unmergedChunkStartTimes, taskName, mergeLogger, resource, fullMerge, unmergedPaths);
+ mergeSeriesTask.mergeSeries();
- mergeFiles(seqFiles);
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName,mergedChunkCnt, unmergedChunkCnt,
+ unmergedChunkStartTimes, mergeLogger, resource, resource.getSeqFiles());
+ mergeFileTask.mergeFiles();
}
cleanUp(continueMerge);
break;
@@ -93,7 +99,10 @@ public class RecoverMergeTask extends MergeTask {
mergeLogger = new MergeLogger(storageGroupDir);
truncateFiles();
recoverChunkCounts();
- mergeFiles(unmergedFiles);
+ resource.setSeqFiles(unmergedFiles);
+ MergeFileTask mergeFileTask = new MergeFileTask(taskName,mergedChunkCnt, unmergedChunkCnt,
+ unmergedChunkStartTimes, mergeLogger, resource, unmergedFiles);
+ mergeFileTask.mergeFiles();
} else {
// NOTICE: although some of the seqFiles may have been truncated, later TsFile recovery
// will recover them, so they are not a concern here
@@ -116,8 +125,8 @@ public class RecoverMergeTask extends MergeTask {
// move the merged chunks or the unmerged chunks
private void recoverChunkCounts() throws IOException {
logger.debug("{} recovering chunk counts", taskName);
- for (TsFileResource tsFileResource : seqFiles) {
- RestorableTsFileIOWriter mergeFileWriter = getMergeFileWriter(tsFileResource);
+ for (TsFileResource tsFileResource : resource.getSeqFiles()) {
+ RestorableTsFileIOWriter mergeFileWriter = resource.getMergeFileWriter(tsFileResource);
mergeFileWriter.makeMetadataVisible();
unmergedChunkStartTimes.put(tsFileResource, new HashMap<>());
for(Path path : mergedPaths) {
@@ -130,7 +139,7 @@ public class RecoverMergeTask extends MergeTask {
RestorableTsFileIOWriter mergeFileWriter) throws IOException {
unmergedChunkStartTimes.get(tsFileResource).put(path, new ArrayList<>());
- List<ChunkMetaData> seqFileChunks = queryChunkMetadata(path, tsFileResource);
+ List<ChunkMetaData> seqFileChunks = resource.queryChunkMetadata(path, tsFileResource);
List<ChunkMetaData> mergeFileChunks =
mergeFileWriter.getVisibleMetadataList(path.getDevice(), path.getMeasurement(), null);
mergedChunkCnt.compute(tsFileResource, (k, v) -> v == null ? mergeFileChunks.size() :
@@ -190,18 +199,18 @@ public class RecoverMergeTask extends MergeTask {
}
if (currLine.equals(MergeLogger.STR_MERGE_START)) {
status = Status.FILES_LOGGED;
- seqFiles = mergeSeqFiles;
- unseqFiles = mergeUnseqFiles;
- for (TsFileResource seqFile : seqFiles) {
+ resource.setSeqFiles(mergeSeqFiles);
+ resource.setUnseqFiles(mergeUnseqFiles);
+ for (TsFileResource seqFile : resource.getSeqFiles()) {
File mergeFile = new File(seqFile.getFile().getPath() + MergeTask.MERGE_SUFFIX);
fileLastPositions.put(mergeFile, 0L);
}
- unmergedPaths = collectPaths();
+ unmergedPaths = MergeUtils.collectPaths(resource);
analyzeMergedSeries(bufferedReader, unmergedPaths);
}
if (currLine.equals(MergeLogger.STR_ALL_TS_END)) {
status = Status.ALL_TS_MERGED;
- unmergedFiles = seqFiles;
+ unmergedFiles = resource.getSeqFiles();
analyzeMergedFiles(bufferedReader);
}
if (currLine.equals(MergeLogger.STR_MERGE_END)) {
@@ -218,7 +227,7 @@ public class RecoverMergeTask extends MergeTask {
if (currLine.equals(MergeLogger.STR_UNSEQ_FILES)) {
break;
}
- Iterator<TsFileResource> iterator = seqFiles.iterator();
+ Iterator<TsFileResource> iterator = resource.getSeqFiles().iterator();
while (iterator.hasNext()) {
TsFileResource seqFile = iterator.next();
if (seqFile.getFile().getAbsolutePath().equals(currLine)) {
@@ -240,7 +249,7 @@ public class RecoverMergeTask extends MergeTask {
if (currLine.equals(MergeLogger.STR_MERGE_START)) {
break;
}
- Iterator<TsFileResource> iterator = unseqFiles.iterator();
+ Iterator<TsFileResource> iterator = resource.getUnseqFiles().iterator();
while (iterator.hasNext()) {
TsFileResource unseqFile = iterator.next();
if (unseqFile.getFile().getAbsolutePath().equals(currLine)) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 45bce38..a9ae74d 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -18,7 +18,7 @@
*/
package org.apache.iotdb.db.engine.storagegroup;
-import static org.apache.iotdb.db.engine.merge.MergeTask.MERGE_SUFFIX;
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@@ -38,10 +38,10 @@ import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.engine.merge.MergeFileSelector;
-import org.apache.iotdb.db.engine.merge.MergeManager;
-import org.apache.iotdb.db.engine.merge.MergeTask;
-import org.apache.iotdb.db.engine.merge.RecoverMergeTask;
+import org.apache.iotdb.db.engine.merge.selector.MergeFileSelector;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.merge.task.RecoverMergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
@@ -51,7 +51,6 @@ import org.apache.iotdb.db.engine.version.SimpleFileVersionController;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.exception.MergeException;
-import org.apache.iotdb.db.exception.MetadataErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.exception.StorageGroupProcessorException;
import org.apache.iotdb.db.exception.TsFileProcessorException;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index f87cf6c..582b4ed 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.cost.statistic.Measurement;
import org.apache.iotdb.db.engine.StorageEngine;
-import org.apache.iotdb.db.engine.merge.MergeManager;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.builder.ExceptionBuilder;
import org.apache.iotdb.db.monitor.StatMonitor;
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeUtils.java b/iotdb/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
similarity index 58%
rename from iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeUtils.java
rename to iotdb/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
index 42ae1b6..9a3a830 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/merge/MergeUtils.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/utils/MergeUtils.java
@@ -17,26 +17,34 @@
* under the License.
*/
-package org.apache.iotdb.db.engine.merge;
+package org.apache.iotdb.db.utils;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.iotdb.db.engine.merge.manage.MergeResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReader;
+import org.apache.iotdb.tsfile.read.reader.chunk.ChunkReaderWithoutFilter;
import org.apache.iotdb.tsfile.write.chunk.IChunkWriter;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-class MergeUtils {
+public class MergeUtils {
private MergeUtils() {
// util class
}
- static void writeTVPair(TimeValuePair timeValuePair, IChunkWriter chunkWriter,
+ public static void writeTVPair(TimeValuePair timeValuePair, IChunkWriter chunkWriter,
TSDataType dataType) {
switch (dataType) {
case TEXT:
@@ -62,7 +70,7 @@ class MergeUtils {
}
}
- static void writeBatchPoint(BatchData batchData, int i, TSDataType dataType,
+ public static void writeBatchPoint(BatchData batchData, int i, TSDataType dataType,
IChunkWriter chunkWriter) {
switch (dataType) {
case TEXT:
@@ -88,7 +96,7 @@ class MergeUtils {
}
}
- static List<Path> collectFileSeries(TsFileSequenceReader sequenceReader) throws IOException {
+ public static List<Path> collectFileSeries(TsFileSequenceReader sequenceReader) throws IOException {
TsFileMetaData metaData = sequenceReader.readFileMetadata();
Set<String> deviceIds = metaData.getDeviceMap().keySet();
Set<String> measurements = metaData.getMeasurementSchema().keySet();
@@ -101,4 +109,45 @@ class MergeUtils {
return paths;
}
+ public static List<Path> collectPaths(MergeResource resource)
+ throws IOException {
+ Set<Path> pathSet = new HashSet<>();
+ for (TsFileResource tsFileResource : resource.getUnseqFiles()) {
+ TsFileSequenceReader sequenceReader = resource.getFileReader(tsFileResource);
+ resource.getMeasurementSchemaMap().putAll(sequenceReader.readFileMetadata().getMeasurementSchema());
+ pathSet.addAll(collectFileSeries(sequenceReader));
+ }
+ for (TsFileResource tsFileResource : resource.getSeqFiles()) {
+ TsFileSequenceReader sequenceReader = resource.getFileReader(tsFileResource);
+ resource.getMeasurementSchemaMap().putAll(sequenceReader.readFileMetadata().getMeasurementSchema());
+ pathSet.addAll(collectFileSeries(sequenceReader));
+ }
+ List<Path> ret = new ArrayList<>(pathSet);
+ ret.sort(Comparator.comparing(Path::getFullPath));
+ return ret;
+ }
+
+ public static long collectFileSizes(List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles) {
+ long totalSize = 0;
+ for (TsFileResource tsFileResource : seqFiles) {
+ totalSize += tsFileResource.getFileSize();
+ }
+ for (TsFileResource tsFileResource : unseqFiles) {
+ totalSize += tsFileResource.getFileSize();
+ }
+ return totalSize;
+ }
+
+ public static int writeChunkWithoutUnseq(Chunk chunk, IChunkWriter chunkWriter,
+ MeasurementSchema measurementSchema) throws IOException {
+ ChunkReader chunkReader = new ChunkReaderWithoutFilter(chunk);
+ TSDataType dataType = measurementSchema.getType();
+ while (chunkReader.hasNextBatch()) {
+ BatchData batchData = chunkReader.nextBatch();
+ for (int i = 0; i < batchData.length(); i++) {
+ writeBatchPoint(batchData, i, dataType, chunkWriter);
+ }
+ }
+ return chunk.getHeader().getNumOfPages();
+ }
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeFileSelectorTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeFileSelectorTest.java
index 542c5b7..4fe8b1f 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeFileSelectorTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeFileSelectorTest.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.engine.merge;
import static org.junit.Assert.assertEquals;
import java.util.List;
+import org.apache.iotdb.db.engine.merge.selector.MergeFileSelector;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.junit.Test;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
index c24dcd3..de76ac0 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeLogTest.java
@@ -28,6 +28,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.List;
import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.After;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java
index cb37c39..3975639 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergePerfTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
index 5fc678d..96b6aae 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/merge/MergeTaskTest.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.util.Collections;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.sequence.SequenceSeriesReader;