You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/11/01 06:25:32 UTC
[iotdb] branch new_vector updated (f0e4a8e -> 92504cc)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
from f0e4a8e remove more if type=vector
new 17ca413 delete useless code line
new 9f2cf3e change query interface
new 92504cc Merge branch 'new_vector' of https://github.com/apache/iotdb into new_vector
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/db/engine/memtable/AbstractMemTable.java | 86 ++++++++---------
.../apache/iotdb/db/engine/memtable/IMemTable.java | 7 +-
.../engine/storagegroup/StorageGroupProcessor.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 37 ++------
.../apache/iotdb/db/metadata/path/AlignedPath.java | 8 ++
.../iotdb/db/metadata/path/MeasurementPath.java | 7 ++
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 1 -
.../db/engine/memtable/PrimitiveMemTableTest.java | 94 +++++++++---------
.../storagegroup/StorageGroupProcessorTest.java | 24 ++---
.../engine/storagegroup/TsFileProcessorTest.java | 105 +++++++--------------
.../iotdb/db/writelog/recover/LogReplayerTest.java | 25 +++--
11 files changed, 176 insertions(+), 223 deletions(-)
[iotdb] 01/03: delete useless code line
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 17ca4130137fad78e6d9b1973c235babf4d8a94b
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Nov 1 11:50:00 2021 +0800
delete useless code line
---
server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 0a28ab6..f4e1b67 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -264,7 +264,6 @@ public class FileLoaderUtils {
Chunk chunk = chunkLoader.loadChunk((ChunkMetadata) chunkMetaData);
chunk.setFromOldFile(chunkMetaData.isFromOldTsFile());
chunkReader = new ChunkReader(chunk, timeFilter);
- chunkReader.hasNextSatisfiedPage();
} else {
VectorChunkMetadata vectorChunkMetadata = (VectorChunkMetadata) chunkMetaData;
Chunk timeChunk = vectorChunkMetadata.getTimeChunk();
[iotdb] 02/03: change query interface
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9f2cf3e121d26b3cc7eba1232a96bf514af99c01
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Nov 1 14:23:44 2021 +0800
change query interface
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 86 ++++++++---------
.../apache/iotdb/db/engine/memtable/IMemTable.java | 7 +-
.../engine/storagegroup/StorageGroupProcessor.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 37 ++------
.../apache/iotdb/db/metadata/path/AlignedPath.java | 8 ++
.../iotdb/db/metadata/path/MeasurementPath.java | 7 ++
.../db/engine/memtable/PrimitiveMemTableTest.java | 94 +++++++++---------
.../storagegroup/StorageGroupProcessorTest.java | 24 ++---
.../engine/storagegroup/TsFileProcessorTest.java | 105 +++++++--------------
.../iotdb/db/writelog/recover/LogReplayerTest.java | 25 +++--
10 files changed, 176 insertions(+), 222 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index edd63a4..a71f62e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -364,50 +364,50 @@ public abstract class AbstractMemTable implements IMemTable {
@Override
public ReadOnlyMemChunk query(
- String deviceId,
- String measurement,
- IMeasurementSchema partialVectorSchema,
- long ttlLowerBound,
- List<TimeRange> deletionList)
+ PartialPath fullPath, long ttlLowerBound, List<TimeRange> deletionList)
throws IOException, QueryProcessException {
- if (partialVectorSchema.getType() == TSDataType.VECTOR) {
- if (!memTableMap.containsKey(deviceId)) {
- return null;
- }
- IWritableMemChunk vectorMemChunk =
- memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId());
- if (vectorMemChunk == null) {
- return null;
- }
-
- List<String> measurementIdList = partialVectorSchema.getSubMeasurementsList();
- List<Integer> columns = new ArrayList<>();
- IMeasurementSchema vectorSchema = vectorMemChunk.getSchema();
- for (String queryingMeasurement : measurementIdList) {
- columns.add(vectorSchema.getSubMeasurementsList().indexOf(queryingMeasurement));
- }
- // get sorted tv list is synchronized so different query can get right sorted list reference
- TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(columns);
- int curSize = vectorTvListCopy.size();
- return new ReadOnlyMemChunk(partialVectorSchema, vectorTvListCopy, curSize, deletionList);
- } else {
- if (!checkPath(deviceId, measurement)) {
- return null;
- }
- IWritableMemChunk memChunk =
- memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId());
- // get sorted tv list is synchronized so different query can get right sorted list reference
- TVList chunkCopy = memChunk.getSortedTvListForQuery();
- int curSize = chunkCopy.size();
- return new ReadOnlyMemChunk(
- measurement,
- partialVectorSchema.getType(),
- partialVectorSchema.getEncodingType(),
- chunkCopy,
- partialVectorSchema.getProps(),
- curSize,
- deletionList);
- }
+ // if (partialVectorSchema.getType() == TSDataType.VECTOR) {
+ // if (!memTableMap.containsKey(deviceId)) {
+ // return null;
+ // }
+ // IWritableMemChunk vectorMemChunk =
+ // memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId());
+ // if (vectorMemChunk == null) {
+ // return null;
+ // }
+ //
+ // List<String> measurementIdList = partialVectorSchema.getSubMeasurementsList();
+ // List<Integer> columns = new ArrayList<>();
+ // IMeasurementSchema vectorSchema = vectorMemChunk.getSchema();
+ // for (String queryingMeasurement : measurementIdList) {
+ // columns.add(vectorSchema.getSubMeasurementsList().indexOf(queryingMeasurement));
+ // }
+ // // get sorted tv list is synchronized so different query can get right sorted list
+ // reference
+ // TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(columns);
+ // int curSize = vectorTvListCopy.size();
+ // return new ReadOnlyMemChunk(partialVectorSchema, vectorTvListCopy, curSize,
+ // deletionList);
+ // } else {
+ // if (!checkPath(deviceId, measurement)) {
+ // return null;
+ // }
+ // IWritableMemChunk memChunk =
+ // memTableMap.get(deviceId).get(partialVectorSchema.getMeasurementId());
+ // // get sorted tv list is synchronized so different query can get right sorted list
+ // reference
+ // TVList chunkCopy = memChunk.getSortedTvListForQuery();
+ // int curSize = chunkCopy.size();
+ // return new ReadOnlyMemChunk(
+ // measurement,
+ // partialVectorSchema.getType(),
+ // partialVectorSchema.getEncodingType(),
+ // chunkCopy,
+ // partialVectorSchema.getProps(),
+ // curSize,
+ // deletionList);
+ // }
+ return null;
}
@SuppressWarnings("squid:S3776") // high Cognitive Complexity
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index 08aeea9..1dc2446 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -106,12 +106,7 @@ public interface IMemTable {
void insertAlignedTablet(InsertTabletPlan insertTabletPlan, int start, int end)
throws WriteProcessException;
- ReadOnlyMemChunk query(
- String deviceId,
- String measurement,
- IMeasurementSchema schema,
- long ttlLowerBound,
- List<TimeRange> deletionList)
+ ReadOnlyMemChunk query(PartialPath fullPath, long ttlLowerBound, List<TimeRange> deletionList)
throws IOException, QueryProcessException, MetadataException;
/** putBack all the memory resources. */
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index efd430c..ee6ed00 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -82,7 +82,6 @@ import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.apache.commons.io.FileUtils;
@@ -1790,8 +1789,6 @@ public class StorageGroupProcessor {
(timeFilter == null ? "null" : timeFilter));
}
- IMeasurementSchema schema = IoTDB.metaManager.getSeriesSchema(fullPath);
-
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
long ttlLowerBound =
dataTTL != Long.MAX_VALUE ? System.currentTimeMillis() - dataTTL : Long.MIN_VALUE;
@@ -1822,7 +1819,7 @@ public class StorageGroupProcessor {
} else {
tsFileResource
.getUnsealedFileProcessor()
- .query(deviceId, fullPath.getMeasurement(), schema, context, tsfileResourcesForQuery);
+ .query(fullPath, context, tsfileResourcesForQuery);
}
} catch (IOException e) {
throw new MetadataException(e);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 83f123c..eaa19e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.db.engine.storagegroup;
import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
import org.apache.iotdb.db.engine.StorageEngine;
@@ -62,7 +61,6 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
import org.slf4j.Logger;
@@ -1212,20 +1210,16 @@ public class TsFileProcessor {
* construct a deletion list from a memtable
*
* @param memTable memtable
- * @param deviceId device id
- * @param measurement measurement name
* @param timeLowerBound time water mark
*/
private List<TimeRange> constructDeletionList(
- IMemTable memTable, String deviceId, String measurement, long timeLowerBound)
- throws MetadataException {
+ IMemTable memTable, PartialPath fullPath, long timeLowerBound) {
List<TimeRange> deletionList = new ArrayList<>();
deletionList.add(new TimeRange(Long.MIN_VALUE, timeLowerBound));
for (Modification modification : getModificationsForMemtable(memTable)) {
if (modification instanceof Deletion) {
Deletion deletion = (Deletion) modification;
- if (deletion.getPath().matchFullPath(new PartialPath(deviceId, measurement))
- && deletion.getEndTime() > timeLowerBound) {
+ if (deletion.getPath().matchFullPath(fullPath) && deletion.getEndTime() > timeLowerBound) {
long lowerBound = Math.max(deletion.getStartTime(), timeLowerBound);
deletionList.add(new TimeRange(lowerBound, deletion.getEndTime()));
}
@@ -1238,17 +1232,10 @@ public class TsFileProcessor {
* get the chunk(s) in the memtable (one from work memtable and the other ones in flushing
* memtables and then compact them into one TimeValuePairSorter). Then get the related
* ChunkMetadata of data on disk.
- *
- * @param deviceId device id
- * @param measurementId measurements id
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public void query(
- String deviceId,
- String measurementId,
- IMeasurementSchema schema,
- QueryContext context,
- List<TsFileResource> tsfileResourcesForQuery)
+ PartialPath fullPath, QueryContext context, List<TsFileResource> tsfileResourcesForQuery)
throws IOException, MetadataException {
if (logger.isDebugEnabled()) {
logger.debug(
@@ -1264,32 +1251,28 @@ public class TsFileProcessor {
continue;
}
List<TimeRange> deletionList =
- constructDeletionList(
- flushingMemTable, deviceId, measurementId, context.getQueryTimeLowerBound());
+ constructDeletionList(flushingMemTable, fullPath, context.getQueryTimeLowerBound());
ReadOnlyMemChunk memChunk =
- flushingMemTable.query(
- deviceId, measurementId, schema, context.getQueryTimeLowerBound(), deletionList);
+ flushingMemTable.query(fullPath, context.getQueryTimeLowerBound(), deletionList);
if (memChunk != null) {
readOnlyMemChunks.add(memChunk);
}
}
if (workMemTable != null) {
ReadOnlyMemChunk memChunk =
- workMemTable.query(
- deviceId, measurementId, schema, context.getQueryTimeLowerBound(), null);
+ workMemTable.query(fullPath, context.getQueryTimeLowerBound(), null);
if (memChunk != null) {
readOnlyMemChunks.add(memChunk);
}
}
ModificationFile modificationFile = tsFileResource.getModFile();
- List<Modification> modifications =
- context.getPathModifications(
- modificationFile,
- new PartialPath(deviceId + IoTDBConstant.PATH_SEPARATOR + measurementId));
+ List<Modification> modifications = context.getPathModifications(modificationFile, fullPath);
List<IChunkMetadata> chunkMetadataList =
- schema.getVisibleMetadataListFromWriter(writer, deviceId);
+ fullPath
+ .getMeasurementSchema()
+ .getVisibleMetadataListFromWriter(writer, fullPath.getDevice());
QueryUtils.modifyChunkMetaData(chunkMetadataList, modifications);
chunkMetadataList.removeIf(context::chunkNotSatisfy);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index d325291..4d3640d 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -59,6 +59,14 @@ public class AlignedPath extends PartialPath {
this.measurementList = subSensorsList;
}
+ public AlignedPath(
+ String vectorPath, List<String> measurementList, List<IMeasurementSchema> schemaList)
+ throws IllegalPathException {
+ super(vectorPath);
+ this.measurementList = measurementList;
+ this.schemaList = schemaList;
+ }
+
public AlignedPath(String vectorPath, String subSensor) throws IllegalPathException {
super(vectorPath);
measurementList = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
index 1861d65..b328990 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.metadata.path;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.SeriesReader;
@@ -47,6 +48,12 @@ public class MeasurementPath extends PartialPath {
super(measurementPath.getNodes());
}
+ public MeasurementPath(String device, String measurement, IMeasurementSchema measurementSchema)
+ throws IllegalPathException {
+ super(device, measurement);
+ this.measurementSchema = measurementSchema;
+ }
+
public IMeasurementSchema getMeasurementSchema() {
return measurementSchema;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
index 07637bf..3b45f08 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
+import org.apache.iotdb.db.metadata.path.AlignedPath;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
import org.apache.iotdb.db.utils.MathUtils;
@@ -38,7 +40,6 @@ import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
-import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
import org.junit.Assert;
import org.junit.Before;
@@ -129,8 +130,8 @@ public class PrimitiveMemTableTest {
i,
i);
}
- ReadOnlyMemChunk memChunk =
- memTable.query(
+ MeasurementPath fullPath =
+ new MeasurementPath(
deviceId,
measurementId[0],
new UnaryMeasurementSchema(
@@ -138,9 +139,8 @@ public class PrimitiveMemTableTest {
TSDataType.INT32,
TSEncoding.RLE,
CompressionType.UNCOMPRESSED,
- Collections.emptyMap()),
- Long.MIN_VALUE,
- null);
+ Collections.emptyMap()));
+ ReadOnlyMemChunk memChunk = memTable.query(fullPath, Long.MIN_VALUE, null);
IPointReader iterator = memChunk.getPointReader();
for (int i = 0; i < dataSize; i++) {
iterator.hasNextTimeValuePair();
@@ -167,20 +167,17 @@ public class PrimitiveMemTableTest {
aRet.getTimestamp(),
aRet.getValue().getValue());
}
- IPointReader tvPair =
- memTable
- .query(
- deviceId,
+ MeasurementPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ sensorId,
+ new UnaryMeasurementSchema(
sensorId,
- new UnaryMeasurementSchema(
- sensorId,
- dataType,
- encoding,
- CompressionType.UNCOMPRESSED,
- Collections.emptyMap()),
- Long.MIN_VALUE,
- null)
- .getPointReader();
+ dataType,
+ encoding,
+ CompressionType.UNCOMPRESSED,
+ Collections.emptyMap()));
+ IPointReader tvPair = memTable.query(fullPath, Long.MIN_VALUE, null).getPointReader();
Arrays.sort(ret);
TimeValuePair last = null;
for (int i = 0; i < ret.length; i++) {
@@ -214,20 +211,18 @@ public class PrimitiveMemTableTest {
throws IOException, QueryProcessException, MetadataException {
memTable.write(genInsertTablePlan(), 0, 100);
- IPointReader tvPair =
- memTable
- .query(
- "root.sg.device5",
- "sensor1",
- new VectorMeasurementSchema(
- "$#$0",
- new String[] {"sensor1"},
- new TSDataType[] {TSDataType.INT64},
- new TSEncoding[] {TSEncoding.GORILLA},
- CompressionType.UNCOMPRESSED),
- Long.MIN_VALUE,
- null)
- .getPointReader();
+ AlignedPath fullPath =
+ new AlignedPath(
+ "root.sg.device5",
+ Collections.singletonList("sensor1"),
+ Collections.singletonList(
+ new UnaryMeasurementSchema(
+ "sensor1",
+ TSDataType.INT64,
+ TSEncoding.GORILLA,
+ CompressionType.UNCOMPRESSED,
+ Collections.emptyMap())));
+ IPointReader tvPair = memTable.query(fullPath, Long.MIN_VALUE, null).getPointReader();
for (int i = 0; i < 100; i++) {
tvPair.hasNextTimeValuePair();
TimeValuePair next = tvPair.nextTimeValuePair();
@@ -235,20 +230,25 @@ public class PrimitiveMemTableTest {
Assert.assertEquals(i, next.getValue().getLong());
}
- tvPair =
- memTable
- .query(
- "root.sg.device5",
- "$#$1",
- new VectorMeasurementSchema(
- "$#$0",
- new String[] {"sensor0", "sensor1"},
- new TSDataType[] {TSDataType.BOOLEAN, TSDataType.INT64},
- new TSEncoding[] {TSEncoding.PLAIN, TSEncoding.GORILLA},
- CompressionType.UNCOMPRESSED),
- Long.MIN_VALUE,
- null)
- .getPointReader();
+ fullPath =
+ new AlignedPath(
+ "root.sg.device5",
+ Arrays.asList("sensor0", "sensor1"),
+ Arrays.asList(
+ new UnaryMeasurementSchema(
+ "sensor0",
+ TSDataType.BOOLEAN,
+ TSEncoding.PLAIN,
+ CompressionType.UNCOMPRESSED,
+ Collections.emptyMap()),
+ new UnaryMeasurementSchema(
+ "sensor1",
+ TSDataType.INT64,
+ TSEncoding.GORILLA,
+ CompressionType.UNCOMPRESSED,
+ Collections.emptyMap())));
+
+ tvPair = memTable.query(fullPath, Long.MIN_VALUE, null).getPointReader();
for (int i = 0; i < 100; i++) {
tvPair.hasNextTimeValuePair();
TimeValuePair next = tvPair.nextTimeValuePair();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
index 9cfac4e..07c0b97 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
@@ -122,21 +123,22 @@ public class StorageGroupProcessorTest {
processor.insert(new InsertRowPlan(record));
}
+ PartialPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ measurementId,
+ new UnaryMeasurementSchema(
+ measurementId,
+ TSDataType.INT32,
+ TSEncoding.RLE,
+ CompressionType.UNCOMPRESSED,
+ Collections.emptyMap()));
+
processor.delete(new PartialPath(deviceId, measurementId), 0, 15L, -1, null);
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
for (TsFileProcessor tsfileProcessor : processor.getWorkUnsequenceTsFileProcessors()) {
- tsfileProcessor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId,
- TSDataType.INT32,
- TSEncoding.RLE,
- CompressionType.UNCOMPRESSED,
- Collections.emptyMap()),
- EnvironmentUtils.TEST_QUERY_CONTEXT,
- tsfileResourcesForQuery);
+ tsfileProcessor.query(fullPath, EnvironmentUtils.TEST_QUERY_CONTEXT, tsfileResourcesForQuery);
}
Assert.assertEquals(1, tsfileResourcesForQuery.size());
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index ad1e7a1..2625175 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.rescon.SystemInfo;
@@ -106,13 +107,13 @@ public class TsFileProcessorTest {
this.sgInfo.initTsFileProcessorInfo(processor);
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ MeasurementPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ measurementId,
+ new UnaryMeasurementSchema(
+ measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props));
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.isEmpty());
for (int i = 1; i <= 100; i++) {
@@ -123,13 +124,7 @@ public class TsFileProcessorTest {
// query data in memory
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertFalse(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
List<ReadOnlyMemChunk> memChunks = tsfileResourcesForQuery.get(0).getReadOnlyMemChunk();
for (ReadOnlyMemChunk chunk : memChunks) {
@@ -146,13 +141,7 @@ public class TsFileProcessorTest {
processor.syncFlush();
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
assertEquals(1, tsfileResourcesForQuery.get(0).getChunkMetadataList().size());
assertEquals(
@@ -181,13 +170,13 @@ public class TsFileProcessorTest {
this.sgInfo.initTsFileProcessorInfo(processor);
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ MeasurementPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ measurementId,
+ new UnaryMeasurementSchema(
+ measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props));
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.isEmpty());
for (int i = 1; i <= 100; i++) {
@@ -198,13 +187,7 @@ public class TsFileProcessorTest {
// query data in memory
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertFalse(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
int num = 1;
List<ReadOnlyMemChunk> memChunks = tsfileResourcesForQuery.get(0).getReadOnlyMemChunk();
@@ -222,13 +205,7 @@ public class TsFileProcessorTest {
processor.syncFlush();
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
assertEquals(1, tsfileResourcesForQuery.get(0).getChunkMetadataList().size());
assertEquals(
@@ -286,13 +263,13 @@ public class TsFileProcessorTest {
this.sgInfo.initTsFileProcessorInfo(processor);
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ MeasurementPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ measurementId,
+ new UnaryMeasurementSchema(
+ measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props));
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.isEmpty());
for (int flushId = 0; flushId < 10; flushId++) {
@@ -306,13 +283,7 @@ public class TsFileProcessorTest {
processor.syncFlush();
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertFalse(tsfileResourcesForQuery.isEmpty());
assertTrue(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
assertEquals(10, tsfileResourcesForQuery.get(0).getChunkMetadataList().size());
@@ -342,13 +313,13 @@ public class TsFileProcessorTest {
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ MeasurementPath fullPath =
+ new MeasurementPath(
+ deviceId,
+ measurementId,
+ new UnaryMeasurementSchema(
+ measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props));
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertTrue(tsfileResourcesForQuery.isEmpty());
for (int i = 1; i <= 100; i++) {
@@ -359,13 +330,7 @@ public class TsFileProcessorTest {
// query data in memory
tsfileResourcesForQuery.clear();
- processor.query(
- deviceId,
- measurementId,
- new UnaryMeasurementSchema(
- measurementId, dataType, encoding, CompressionType.UNCOMPRESSED, props),
- context,
- tsfileResourcesForQuery);
+ processor.query(fullPath, context, tsfileResourcesForQuery);
assertFalse(tsfileResourcesForQuery.isEmpty());
assertFalse(tsfileResourcesForQuery.get(0).getReadOnlyMemChunk().isEmpty());
List<ReadOnlyMemChunk> memChunks = tsfileResourcesForQuery.get(0).getReadOnlyMemChunk();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
index c05e412..112e5bf 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/recover/LogReplayerTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
@@ -163,8 +164,8 @@ public class LogReplayerTest {
});
for (int i = 0; i < 5; i++) {
- ReadOnlyMemChunk memChunk =
- memTable.query(
+ MeasurementPath fullPath =
+ new MeasurementPath(
"root.sg.device" + i,
"sensor" + i,
new UnaryMeasurementSchema(
@@ -172,19 +173,16 @@ public class LogReplayerTest {
TSDataType.INT64,
TSEncoding.RLE,
CompressionType.UNCOMPRESSED,
- Collections.emptyMap()),
- Long.MIN_VALUE,
- null);
+ Collections.emptyMap()));
+ ReadOnlyMemChunk memChunk = memTable.query(fullPath, Long.MIN_VALUE, null);
IPointReader iterator = memChunk.getPointReader();
- if (i == 0) {
- assertFalse(iterator.hasNextTimeValuePair());
- } else {
+ if (i != 0) {
assertTrue(iterator.hasNextTimeValuePair());
TimeValuePair timeValuePair = iterator.nextTimeValuePair();
assertEquals(i, timeValuePair.getTimestamp());
assertEquals(i, timeValuePair.getValue().getLong());
- assertFalse(iterator.hasNextTimeValuePair());
}
+ assertFalse(iterator.hasNextTimeValuePair());
}
Modification[] mods = modFile.getModifications().toArray(new Modification[0]);
@@ -202,8 +200,8 @@ public class LogReplayerTest {
// test insert tablet
for (int i = 0; i < 2; i++) {
- ReadOnlyMemChunk memChunk =
- memTable.query(
+ MeasurementPath fullPath =
+ new MeasurementPath(
"root.sg.device5",
"sensor" + i,
new UnaryMeasurementSchema(
@@ -211,9 +209,8 @@ public class LogReplayerTest {
TSDataType.INT64,
TSEncoding.PLAIN,
CompressionType.UNCOMPRESSED,
- Collections.emptyMap()),
- Long.MIN_VALUE,
- null);
+ Collections.emptyMap()));
+ ReadOnlyMemChunk memChunk = memTable.query(fullPath, Long.MIN_VALUE, null);
// s0 has datatype boolean, but required INT64, will return null
if (i == 0) {
assertNull(memChunk);
[iotdb] 03/03: Merge branch 'new_vector' of
https://github.com/apache/iotdb into new_vector
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 92504cc63e6a813ca135ef400cc9ea4f5d6645ca
Merge: 9f2cf3e f0e4a8e
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Nov 1 14:24:04 2021 +0800
Merge branch 'new_vector' of https://github.com/apache/iotdb into new_vector
.../db/engine/memtable/IWritableMemChunk.java | 2 +-
.../db/engine/memtable/VectorWritableMemChunk.java | 6 +-
.../iotdb/db/engine/memtable/WritableMemChunk.java | 5 +-
.../db/engine/storagegroup/TsFileProcessor.java | 11 +-
.../apache/iotdb/db/rescon/TVListAllocator.java | 2 +-
.../iotdb/db/utils/datastructure/TVList.java | 65 ++--------
.../iotdb/db/utils/datastructure/VectorTVList.java | 136 ++++++++++++++++++---
7 files changed, 143 insertions(+), 84 deletions(-)