You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/11/11 07:18:09 UTC
[iotdb] branch new_vector updated: fix some ci
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/new_vector by this push:
new 419ffec fix some ci
419ffec is described below
commit 419ffec5aff988237b940f54622b2707a985eab2
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Thu Nov 11 15:17:37 2021 +0800
fix some ci
---
.../db/engine/storagegroup/TsFileProcessor.java | 8 ++-
.../db/engine/storagegroup/TsFileResource.java | 19 ++++++-
.../apache/iotdb/db/metadata/path/AlignedPath.java | 61 ++++++++++++++++++++--
.../iotdb/db/metadata/path/MeasurementPath.java | 38 ++++++++++++--
.../apache/iotdb/db/metadata/path/PartialPath.java | 12 ++++-
.../metadata/DiskAlignedChunkMetadataLoader.java | 5 --
.../chunk/metadata/DiskChunkMetadataLoader.java | 61 ++++++++++++----------
.../metadata/MemAlignedChunkMetadataLoader.java | 5 --
.../chunk/metadata/MemChunkMetadataLoader.java | 9 +---
.../query/timegenerator/ServerTimeGenerator.java | 2 +-
.../read/controller/IChunkMetadataLoader.java | 2 -
.../tsfile/write/schema/IMeasurementSchema.java | 5 --
.../write/schema/UnaryMeasurementSchema.java | 9 ----
.../write/schema/VectorMeasurementSchema.java | 30 -----------
.../write/writer/VectorMeasurementSchemaStub.java | 7 ---
15 files changed, 161 insertions(+), 112 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 2489839..33cc7ef 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -57,6 +57,7 @@ import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -1298,10 +1299,13 @@ public class TsFileProcessor {
}
}
+ List<IChunkMetadata> chunkMetadataList =
+ fullPath.getVisibleMetadataListFromWriter(writer, tsFileResource, context);
+
// get in memory data
- if (!readOnlyMemChunks.isEmpty()) {
+ if (!readOnlyMemChunks.isEmpty() || !chunkMetadataList.isEmpty()) {
tsfileResourcesForQuery.add(
- fullPath.createTsFileResource(readOnlyMemChunks, tsFileResource));
+ fullPath.createTsFileResource(readOnlyMemChunks, chunkMetadataList, tsFileResource));
}
} catch (QueryProcessException e) {
logger.error(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 21c46cf..c973886 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.db.exception.PartitionViolationException;
import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
@@ -51,6 +52,7 @@ import java.io.OutputStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -108,6 +110,12 @@ public class TsFileResource {
private boolean isSeq;
+ /**
+ * Chunk metadata list of unsealed tsfile. Only be set in a temporal TsFileResource in a query
+ * process.
+ */
+ private List<IChunkMetadata> chunkMetadataList;
+
/** Mem chunk data. Only be set in a temporal TsFileResource in a query process. */
private List<ReadOnlyMemChunk> readOnlyMemChunk;
@@ -155,6 +163,7 @@ public class TsFileResource {
this.closed = other.closed;
this.deleted = other.deleted;
this.isMerging = other.isMerging;
+ this.chunkMetadataList = other.chunkMetadataList;
this.readOnlyMemChunk = other.readOnlyMemChunk;
this.tsFileLock = other.tsFileLock;
this.fsFactory = other.fsFactory;
@@ -182,11 +191,14 @@ public class TsFileResource {
/** unsealed TsFile, for query */
public TsFileResource(
- List<ReadOnlyMemChunk> readOnlyMemChunk, TsFileResource originTsFileResource)
+ List<ReadOnlyMemChunk> readOnlyMemChunk,
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource originTsFileResource)
throws IOException {
this.file = originTsFileResource.file;
this.timeIndex = originTsFileResource.timeIndex;
this.timeIndexType = originTsFileResource.timeIndexType;
+ this.chunkMetadataList = chunkMetadataList;
this.readOnlyMemChunk = readOnlyMemChunk;
this.originTsFileResource = originTsFileResource;
this.version = originTsFileResource.version;
@@ -305,6 +317,10 @@ public class TsFileResource {
return fsFactory.getFile(file + RESOURCE_SUFFIX).exists();
}
+ public List<IChunkMetadata> getChunkMetadataList() {
+ return new ArrayList<>(chunkMetadataList);
+ }
+
public List<ReadOnlyMemChunk> getReadOnlyMemChunk() {
return readOnlyMemChunk;
}
@@ -375,6 +391,7 @@ public class TsFileResource {
modFile = null;
}
processor = null;
+ chunkMetadataList = null;
timeIndex.close();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index 0cfd926..997996b 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.db.metadata.path;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.AlignedReadOnlyMemChunk;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -31,10 +33,13 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.executor.fill.AlignedLastPointReader;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.AlignedSeriesReader;
+import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -43,6 +48,7 @@ import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import java.io.IOException;
import java.io.Serializable;
@@ -269,10 +275,14 @@ public class AlignedPath extends PartialPath {
@Override
public TsFileResource createTsFileResource(
- List<ReadOnlyMemChunk> readOnlyMemChunk, TsFileResource originTsFileResource)
+ List<ReadOnlyMemChunk> readOnlyMemChunk,
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource originTsFileResource)
throws IOException {
- TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, originTsFileResource);
- tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk));
+ TsFileResource tsFileResource =
+ new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource);
+ tsFileResource.setTimeSeriesMetadata(
+ generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList));
return tsFileResource;
}
@@ -281,7 +291,8 @@ public class AlignedPath extends PartialPath {
* have chunkMetadata, but query will use these, so we need to generate it for them.
*/
private AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
- List<ReadOnlyMemChunk> readOnlyMemChunk) throws IOException {
+ List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList)
+ throws IOException {
TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata();
timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1);
timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
@@ -303,6 +314,18 @@ public class AlignedPath extends PartialPath {
valueTimeSeriesMetadataList.add(valueMetadata);
}
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata;
+ timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ valueTimeSeriesMetadataList
+ .get(i)
+ .getStatistics()
+ .mergeStatistics(
+ alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics());
+ }
+ }
+
for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
if (!memChunk.isEmpty()) {
AlignedChunkMetadata alignedChunkMetadata =
@@ -348,4 +371,34 @@ public class AlignedPath extends PartialPath {
return new AlignedReadOnlyMemChunk(
getMeasurementSchema(), alignedTvListCopy, curSize, deletionList);
}
+
+ @Override
+ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
+ RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) {
+ ModificationFile modificationFile = tsFileResource.getModFile();
+ List<List<Modification>> modifications = context.getPathModifications(modificationFile, this);
+
+ List<AlignedChunkMetadata> chunkMetadataList = new ArrayList<>();
+ List<ChunkMetadata> timeChunkMetadataList =
+ writer.getVisibleMetadataList(getDevice(), "", getSeriesType());
+ List<List<ChunkMetadata>> valueChunkMetadataList = new ArrayList<>();
+ for (int i = 0; i < measurementList.size(); i++) {
+ valueChunkMetadataList.add(
+ writer.getVisibleMetadataList(
+ getDevice(), measurementList.get(i), schemaList.get(i).getType()));
+ }
+
+ for (int i = 0; i < timeChunkMetadataList.size(); i++) {
+ List<IChunkMetadata> valueChunkMetadata = new ArrayList<>();
+ for (List<ChunkMetadata> chunkMetadata : valueChunkMetadataList) {
+ valueChunkMetadata.add(chunkMetadata.get(i));
+ }
+ chunkMetadataList.add(
+ new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata));
+ }
+
+ QueryUtils.modifyAlignedChunkMetaData(chunkMetadataList, modifications);
+ chunkMetadataList.removeIf(context::chunkNotSatisfy);
+ return new ArrayList<>(chunkMetadataList);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
index b9edd98..0fcab99 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
@@ -20,6 +20,8 @@ package org.apache.iotdb.db.metadata.path;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
+import org.apache.iotdb.db.engine.modification.Modification;
+import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -29,17 +31,21 @@ import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.executor.fill.LastPointReader;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.utils.QueryUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.datastructure.TVList;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -182,10 +188,14 @@ public class MeasurementPath extends PartialPath {
@Override
public TsFileResource createTsFileResource(
- List<ReadOnlyMemChunk> readOnlyMemChunk, TsFileResource originTsFileResource)
+ List<ReadOnlyMemChunk> readOnlyMemChunk,
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource originTsFileResource)
throws IOException {
- TsFileResource tsFileResource = new TsFileResource(readOnlyMemChunk, originTsFileResource);
- tsFileResource.setTimeSeriesMetadata(generateTimeSeriesMetadata(readOnlyMemChunk));
+ TsFileResource tsFileResource =
+ new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource);
+ tsFileResource.setTimeSeriesMetadata(
+ generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList));
return tsFileResource;
}
@@ -193,7 +203,8 @@ public class MeasurementPath extends PartialPath {
* Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't
* have chunkMetadata, but query will use these, so we need to generate it for them.
*/
- private TimeseriesMetadata generateTimeSeriesMetadata(List<ReadOnlyMemChunk> readOnlyMemChunk)
+ private TimeseriesMetadata generateTimeSeriesMetadata(
+ List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList)
throws IOException {
TimeseriesMetadata timeSeriesMetadata = new TimeseriesMetadata();
timeSeriesMetadata.setMeasurementId(measurementSchema.getMeasurementId());
@@ -203,6 +214,10 @@ public class MeasurementPath extends PartialPath {
Statistics<? extends Serializable> seriesStatistics =
Statistics.getStatsByType(timeSeriesMetadata.getTSDataType());
+ // flush chunkMetadataList one by one
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
+ }
for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
if (!memChunk.isEmpty()) {
@@ -235,4 +250,19 @@ public class MeasurementPath extends PartialPath {
curSize,
deletionList);
}
+
+ @Override
+ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
+ RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) {
+ ModificationFile modificationFile = tsFileResource.getModFile();
+ List<Modification> modifications = context.getPathModifications(modificationFile, this);
+
+ List<IChunkMetadata> chunkMetadataList =
+ new ArrayList<>(
+ writer.getVisibleMetadataList(getDevice(), getMeasurement(), getSeriesType()));
+
+ QueryUtils.modifyChunkMetaData(chunkMetadataList, modifications);
+ chunkMetadataList.removeIf(context::chunkNotSatisfy);
+ return chunkMetadataList;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
index eb6e0fc..4aae3e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java
@@ -32,11 +32,13 @@ import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.SeriesReader;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -408,10 +410,13 @@ public class PartialPath extends Path implements Comparable<Path> {
}
public TsFileResource createTsFileResource(
- List<ReadOnlyMemChunk> readOnlyMemChunk, TsFileResource originTsFileResource)
+ List<ReadOnlyMemChunk> readOnlyMemChunk,
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource originTsFileResource)
throws IOException {
throw new UnsupportedOperationException("Should call exact sub class!");
}
+
/**
* get the ReadOnlyMemChunk from the given MemTable.
*
@@ -422,4 +427,9 @@ public class PartialPath extends Path implements Comparable<Path> {
throws QueryProcessException, IOException {
throw new UnsupportedOperationException("Should call exact sub class!");
}
+
+ public List<IChunkMetadata> getVisibleMetadataListFromWriter(
+ RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) {
+ throw new UnsupportedOperationException("Should call exact sub class!");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
index de2c652..05ce7b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
@@ -106,9 +106,4 @@ public class DiskAlignedChunkMetadataLoader implements IChunkMetadataLoader {
return new ArrayList<>(alignedChunkMetadataList);
}
-
- @Override
- public boolean isMemChunkMetadataLoader() {
- return false;
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 0cc30e7..8459f55 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -59,6 +59,38 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
List<IChunkMetadata> chunkMetadataList =
((TimeseriesMetadata) timeSeriesMetadata).getChunkMetadataList();
+ setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
+
+ /*
+ * remove not satisfied ChunkMetaData
+ */
+ chunkMetadataList.removeIf(
+ chunkMetaData ->
+ (filter != null
+ && !filter.satisfyStartEndTime(
+ chunkMetaData.getStartTime(), chunkMetaData.getEndTime()))
+ || chunkMetaData.getStartTime() > chunkMetaData.getEndTime());
+
+ // For chunkMetadata from old TsFile, do not set version
+ for (IChunkMetadata metadata : chunkMetadataList) {
+ if (!metadata.isFromOldTsFile()) {
+ metadata.setVersion(resource.getVersion());
+ }
+ }
+
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After removed by filter Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
+ return chunkMetadataList;
+ }
+
+ public static void setDiskChunkLoader(
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource resource,
+ PartialPath seriesPath,
+ QueryContext context) {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath);
@@ -89,34 +121,5 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
chunkMetadata.setChunkLoader(new DiskChunkLoader(context.isDebug()));
}
});
-
- /*
- * remove not satisfied ChunkMetaData
- */
- chunkMetadataList.removeIf(
- chunkMetaData ->
- (filter != null
- && !filter.satisfyStartEndTime(
- chunkMetaData.getStartTime(), chunkMetaData.getEndTime()))
- || chunkMetaData.getStartTime() > chunkMetaData.getEndTime());
-
- // For chunkMetadata from old TsFile, do not set version
- for (IChunkMetadata metadata : chunkMetadataList) {
- if (!metadata.isFromOldTsFile()) {
- metadata.setVersion(resource.getVersion());
- }
- }
-
- if (context.isDebug()) {
- DEBUG_LOGGER.info("After removed by filter Chunk meta data list is: ");
- chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
- }
-
- return chunkMetadataList;
- }
-
- @Override
- public boolean isMemChunkMetadataLoader() {
- return false;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
index 16ee5ab..11fe5a8 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
@@ -75,9 +75,4 @@ public class MemAlignedChunkMetadataLoader implements IChunkMetadataLoader {
}
return chunkMetadataList;
}
-
- @Override
- public boolean isMemChunkMetadataLoader() {
- return true;
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 9affdf0..daec942 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import java.util.ArrayList;
import java.util.List;
public class MemChunkMetadataLoader implements IChunkMetadataLoader {
@@ -47,7 +46,8 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
@Override
public List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) {
- List<IChunkMetadata> chunkMetadataList = new ArrayList<>();
+ List<IChunkMetadata> chunkMetadataList = resource.getChunkMetadataList();
+ DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
List<ReadOnlyMemChunk> memChunks = resource.getReadOnlyMemChunk();
if (memChunks != null) {
@@ -72,9 +72,4 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
}
return chunkMetadataList;
}
-
- @Override
- public boolean isMemChunkMetadataLoader() {
- return true;
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
index 1933e2f..7bd4fe6 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/timegenerator/ServerTimeGenerator.java
@@ -90,10 +90,10 @@ public class ServerTimeGenerator extends TimeGenerator {
if (expression.getType() == ExpressionType.SERIES) {
SingleSeriesExpression seriesExpression = (SingleSeriesExpression) expression;
MeasurementPath measurementPath = (MeasurementPath) seriesExpression.getSeriesPath();
- pathList.add(measurementPath.getDevicePath());
// change the MeasurementPath to AlignedPath if the MeasurementPath's isUnderAlignedEntity ==
// true
seriesExpression.setSeriesPath(measurementPath.transformToExactPath());
+ pathList.add((PartialPath) seriesExpression.getSeriesPath());
} else {
getAndTransformPartialPathFromExpression(
((IBinaryExpression) expression).getLeft(), pathList);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index 441e36d..fffe261 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -29,6 +29,4 @@ public interface IChunkMetadataLoader {
/** read all chunk metadata of one time series in one file. */
List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata)
throws IOException;
-
- boolean isMemChunkMetadataLoader();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/IMeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/IMeasurementSchema.java
index 1e8920f..5680d0e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/IMeasurementSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/IMeasurementSchema.java
@@ -19,11 +19,9 @@
package org.apache.iotdb.tsfile.write.schema;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
-import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import java.io.IOException;
import java.io.OutputStream;
@@ -70,9 +68,6 @@ public interface IMeasurementSchema {
int serializeTo(OutputStream outputStream) throws IOException;
- List<IChunkMetadata> getVisibleMetadataListFromWriter(
- RestorableTsFileIOWriter writer, String deviceId);
-
/*
1. used in cluster module to avoid useless field transfer(such as props in MeasurementSchema)
2. add a flag bit at the beginning to distinguish between MeasurementSchema(0) and VectorMeasurementSchema(1)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/UnaryMeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/UnaryMeasurementSchema.java
index d3090ba..e2a867b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/UnaryMeasurementSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/UnaryMeasurementSchema.java
@@ -21,20 +21,17 @@ package org.apache.iotdb.tsfile.write.schema;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
-import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.utils.StringContainer;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -290,12 +287,6 @@ public class UnaryMeasurementSchema
return byteLen;
}
- @Override
- public List<IChunkMetadata> getVisibleMetadataListFromWriter(
- RestorableTsFileIOWriter writer, String deviceId) {
- return new ArrayList<>(writer.getVisibleMetadataList(deviceId, measurementId, getType()));
- }
-
/** function for serializing data to byte buffer. */
@Override
public int serializeTo(ByteBuffer buffer) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java
index fec2796..5b79bd0 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/VectorMeasurementSchema.java
@@ -22,15 +22,11 @@ package org.apache.iotdb.tsfile.write.schema;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
-import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.utils.StringContainer;
-import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import java.io.IOException;
import java.io.InputStream;
@@ -283,32 +279,6 @@ public class VectorMeasurementSchema
}
@Override
- public List<IChunkMetadata> getVisibleMetadataListFromWriter(
- RestorableTsFileIOWriter writer, String deviceId) {
- List<IChunkMetadata> chunkMetadataList = new ArrayList<>();
- List<ChunkMetadata> timeChunkMetadataList =
- writer.getVisibleMetadataList(deviceId, "", getType());
- List<List<ChunkMetadata>> valueChunkMetadataList = new ArrayList<>();
- List<String> valueMeasurementIdList = getSubMeasurementsList();
- List<TSDataType> valueDataTypeList = getSubMeasurementsTSDataTypeList();
- for (int i = 0; i < valueMeasurementIdList.size(); i++) {
- valueChunkMetadataList.add(
- writer.getVisibleMetadataList(
- deviceId, valueMeasurementIdList.get(i), valueDataTypeList.get(i)));
- }
-
- for (int i = 0; i < timeChunkMetadataList.size(); i++) {
- List<IChunkMetadata> valueChunkMetadata = new ArrayList<>();
- for (List<ChunkMetadata> chunkMetadata : valueChunkMetadataList) {
- valueChunkMetadata.add(chunkMetadata.get(i));
- }
- chunkMetadataList.add(
- new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata));
- }
- return chunkMetadataList;
- }
-
- @Override
public int partialSerializeTo(OutputStream outputStream) throws IOException {
ReadWriteIOUtils.write((byte) 1, outputStream);
return 1 + serializeTo(outputStream);
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
index 343d790..8faef39 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/VectorMeasurementSchemaStub.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.tsfile.write.writer;
import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder;
-import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
@@ -113,12 +112,6 @@ public class VectorMeasurementSchemaStub extends VectorMeasurementSchema {
}
@Override
- public List<IChunkMetadata> getVisibleMetadataListFromWriter(
- RestorableTsFileIOWriter writer, String deviceId) {
- return null;
- }
-
- @Override
public int partialSerializeTo(OutputStream outputStream) {
return 0;
}