You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/11/02 03:46:02 UTC
[iotdb] branch new_vector updated: support query a non-exsit
measurement in vector
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/new_vector by this push:
new 554392c support query a non-exsit measurement in vector
554392c is described below
commit 554392c594cf877e2f26d35ff96749dc13896086
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue Nov 2 11:45:20 2021 +0800
support query a non-exsit measurement in vector
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 2 +-
.../db/engine/memtable/IWritableMemChunk.java | 2 +-
.../db/engine/memtable/VectorWritableMemChunk.java | 10 ++--
.../iotdb/db/engine/memtable/WritableMemChunk.java | 2 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 32 -----------
.../apache/iotdb/db/metadata/path/AlignedPath.java | 10 ++--
.../iotdb/db/utils/datastructure/TVList.java | 2 +-
.../iotdb/db/utils/datastructure/VectorTVList.java | 64 +++++++++++++++++++---
8 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index b719f8c..9a293fa 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -121,7 +121,7 @@ public abstract class AbstractMemTable implements IMemTable {
k -> {
seriesNumber++;
totalPointsNumThreshold +=
- avgSeriesPointNumThreshold + vectorSchema.getSubMeasurementsCount();
+ avgSeriesPointNumThreshold * vectorSchema.getSubMeasurementsCount();
return genVectorMemSeries(vectorSchema);
});
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index 1d27b0d..28b4026 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -104,7 +104,7 @@ public interface IWritableMemChunk {
* @param measurementList the measurementList to be queried
* @return sorted tv list
*/
- TVList getSortedTvListForQuery(List<String> measurementList);
+ TVList getSortedTvListForQuery(List<IMeasurementSchema> schemaList);
/**
* served for flush requests. The logic is just same as getSortedTVListForQuery, but without add
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
index dd16b8f..c60d6f8 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/VectorWritableMemChunk.java
@@ -175,15 +175,17 @@ public class VectorWritableMemChunk implements IWritableMemChunk {
}
@Override
- public TVList getSortedTvListForQuery(List<String> measurementList) {
+ public TVList getSortedTvListForQuery(List<IMeasurementSchema> schemaList) {
sortTVList();
// increase reference count
list.increaseReferenceCount();
List<Integer> columnIndexList = new ArrayList<>();
- for (String measurement : measurementList) {
- columnIndexList.add(vectorIdIndexMap.get(measurement));
+ List<TSDataType> dataTypeList = new ArrayList<>();
+ for (IMeasurementSchema measurementSchema : schemaList) {
+ columnIndexList.add(vectorIdIndexMap.getOrDefault(measurementSchema.getMeasurementId(), -1));
+ dataTypeList.add(measurementSchema.getType());
}
- return list.getTvListByColumnIndex(columnIndexList);
+ return list.getTvListByColumnIndex(columnIndexList, dataTypeList);
}
private void sortTVList() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
index 8dcfc07..3638f58 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java
@@ -200,7 +200,7 @@ public class WritableMemChunk implements IWritableMemChunk {
}
@Override
- public synchronized TVList getSortedTvListForQuery(List<String> measurementList) {
+ public synchronized TVList getSortedTvListForQuery(List<IMeasurementSchema> measurementSchema) {
throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + list.getDataType());
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
index 568fe39..18d26e3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
@@ -206,38 +206,6 @@ public class ReadOnlyMemChunk {
cachedMetaData = vectorChunkMetadata;
}
- // When query one measurement in a Vector, the timeValuePair is not a vector type
- private void updateValueStatisticsForSingleColumn(
- IMeasurementSchema schema, Statistics[] valueStatistics, TimeValuePair timeValuePair)
- throws QueryProcessException {
- switch (schema.getSubMeasurementsTSDataTypeList().get(0)) {
- case BOOLEAN:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getBoolean());
- break;
- case TEXT:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
- break;
- case FLOAT:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getFloat());
- break;
- case INT32:
- valueStatistics[0].update(timeValuePair.getTimestamp(), timeValuePair.getValue().getInt());
- break;
- case INT64:
- valueStatistics[0].update(timeValuePair.getTimestamp(), timeValuePair.getValue().getLong());
- break;
- case DOUBLE:
- valueStatistics[0].update(
- timeValuePair.getTimestamp(), timeValuePair.getValue().getDouble());
- break;
- default:
- throw new QueryProcessException("Unsupported data type:" + dataType);
- }
- }
-
private void updateValueStatistics(
IMeasurementSchema schema, Statistics[] valueStatistics, TimeValuePair timeValuePair)
throws QueryProcessException {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index 877a8fb..a8398b8 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@ -321,22 +321,24 @@ public class AlignedPath extends PartialPath {
public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable(
Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange> deletionList)
throws QueryProcessException, IOException {
+ // check If Memtable Contains this path
if (!memTableMap.containsKey(getDevice())) {
return null;
}
VectorWritableMemChunk vectorMemChunk =
((VectorWritableMemChunk) memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER));
- List<String> validMeasurementList = new ArrayList<>();
+ boolean containsMeasurement = false;
for (String measurement : measurementList) {
if (vectorMemChunk.containsMeasurement(measurement)) {
- validMeasurementList.add(measurement);
+ containsMeasurement = true;
+ break;
}
}
- if (validMeasurementList.isEmpty()) {
+ if (!containsMeasurement) {
return null;
}
// get sorted tv list is synchronized so different query can get right sorted list reference
- TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(validMeasurementList);
+ TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(schemaList);
int curSize = vectorTvListCopy.size();
return new ReadOnlyMemChunk(getMeasurementSchema(), vectorTvListCopy, curSize, deletionList);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 1a46af4..3fe1e3a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -201,7 +201,7 @@ public abstract class TVList {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
- public TVList getTvListByColumnIndex(List<Integer> columnIndexList) {
+ public TVList getTvListByColumnIndex(List<Integer> columnIndexList, List<TSDataType> dataTypes) {
throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
index 24ea2cb..cabec52 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/VectorTVList.java
@@ -207,21 +207,34 @@ public class VectorTVList extends TVList {
}
@Override
- public TVList getTvListByColumnIndex(List<Integer> columns) {
+ public TVList getTvListByColumnIndex(List<Integer> columnIndex, List<TSDataType> dataTypes) {
List<TSDataType> types = new ArrayList<>();
List<List<Object>> values = new ArrayList<>();
List<List<BitMap>> bitMaps = null;
- for (int column : columns) {
- types.add(this.dataTypes.get(column));
- values.add(this.values.get(column));
- if (this.bitMaps != null && this.bitMaps.get(column) != null) {
+ for (int i = 0; i < columnIndex.size(); i++) {
+ // columnIndex == -1 means querying a non-exist column, generate empty column here
+ if (columnIndex.get(i) == -1) {
+ types.add(dataTypes.get(i));
+ // use bitmap to mark as null value
if (bitMaps == null) {
- bitMaps = new ArrayList<>(columns.size());
- for (int i = 0; i < columns.size(); i++) {
+ bitMaps = new ArrayList<>(columnIndex.size());
+ for (int j = 0; j < columnIndex.size(); j++) {
bitMaps.add(null);
}
}
- bitMaps.set(columns.indexOf(column), this.bitMaps.get(column));
+ generateEmptyColumn(dataTypes.get(i), values, bitMaps);
+ } else {
+ types.add(this.dataTypes.get(columnIndex.get(i)));
+ values.add(this.values.get(columnIndex.get(i)));
+ if (this.bitMaps != null && this.bitMaps.get(columnIndex.get(i)) != null) {
+ if (bitMaps == null) {
+ bitMaps = new ArrayList<>(columnIndex.size());
+ for (int j = 0; j < columnIndex.size(); j++) {
+ bitMaps.add(null);
+ }
+ }
+ bitMaps.set(i, this.bitMaps.get(columnIndex.get(i)));
+ }
}
}
VectorTVList vectorTvList = new VectorTVList(types);
@@ -233,6 +246,41 @@ public class VectorTVList extends TVList {
return vectorTvList;
}
+ private void generateEmptyColumn(
+ TSDataType dataType, List<List<Object>> values, List<List<BitMap>> bitMaps) {
+ List<Object> columnValue = new ArrayList<>();
+ List<BitMap> columnBitMaps = new ArrayList<>();
+ for (int i = 0; i < timestamps.size(); i++) {
+ switch (dataType) {
+ case TEXT:
+ columnValue.add(new Binary[ARRAY_SIZE]);
+ break;
+ case FLOAT:
+ columnValue.add(new float[ARRAY_SIZE]);
+ break;
+ case INT32:
+ columnValue.add(new int[ARRAY_SIZE]);
+ break;
+ case INT64:
+ columnValue.add(new long[ARRAY_SIZE]);
+ break;
+ case DOUBLE:
+ columnValue.add(new double[ARRAY_SIZE]);
+ break;
+ case BOOLEAN:
+ columnValue.add(new boolean[ARRAY_SIZE]);
+ break;
+ default:
+ break;
+ }
+ BitMap bitMap = new BitMap(ARRAY_SIZE);
+ bitMap.markAll();
+ columnBitMaps.add(bitMap);
+ }
+ values.add(columnValue);
+ bitMaps.add(columnBitMaps);
+ }
+
/**
* Get the int value at the given position in VectorTvList.
*