You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/11/04 03:42:21 UTC
[iotdb] 01/01: resolve conflicts and raw query without value filter
on disk is ok
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch new_vector
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6fea8ad7b017b1046e01ce6302c79e258928c351
Merge: da316c7 5e1f780
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Thu Nov 4 11:41:46 2021 +0800
resolve conflicts and raw query without value filter on disk is ok
.github/dependabot.yml | 8 +-
.gitignore | 5 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlLexer.g4 | 74 ++---
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 60 ++--
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 88 +++---
.../iotdb/cluster/coordinator/Coordinator.java | 114 +++++---
.../log/manage/MetaSingleSnapshotLogManager.java | 10 +-
.../cluster/log/snapshot/MetaSimpleSnapshot.java | 38 ++-
.../log/snapshot/MetaSimpleSnapshotTest.java | 42 ++-
.../cluster/server/member/MetaGroupMemberTest.java | 25 +-
.../cluster/utils/CreateTemplatePlanUtil.java | 71 +++++
docs/SystemDesign/DataQuery/AlignByDeviceQuery.md | 12 +-
.../Data-Concept/Data-Model-and-Terminology.md | 72 ++---
.../DML-Data-Manipulation-Language.md | 83 ++++--
.../IoTDB-SQL-Language/Syntax-Conventions.md | 128 +++++++++
.../System-Tools/Monitor-and-Log-Tools.md | 2 +-
.../SystemDesign/DataQuery/AlignByDeviceQuery.md | 12 +-
docs/zh/UserGuide/Comparison/TSDB-Comparison.md | 2 +-
.../Data-Concept/Data-Model-and-Terminology.md | 70 +++--
.../DML-Data-Manipulation-Language.md | 84 ++++--
.../IoTDB-SQL-Language/Syntax-Conventions.md | 129 +++++++++
.../System-Tools/Monitor-and-Log-Tools.md | 10 +-
hive-connector/pom.xml | 16 --
.../influxdb/protocol/input/InfluxLineProtocol.g4 | 93 +++++++
.../org/apache/iotdb/influxdb/IoTDBInfluxDB.java | 31 ++-
.../iotdb/influxdb/example/InfluxDBExample.java | 2 +-
.../influxdb/protocol/input/InfluxLineParser.java | 165 +++++++++++
.../influxdb/integration/IoTDBInfluxDBIT.java | 3 +
.../protocol/input/InfluxLineParserTest.java | 55 ++++
pom.xml | 6 +
server/src/assembly/resources/conf/iotdb-env.bat | 20 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 24 +-
.../SizeTieredCompactionRecoverTask.java | 58 +++-
.../inner/utils/InnerSpaceCompactionUtils.java | 2 +-
.../compaction/task/AbstractCompactionTask.java | 7 +-
.../engine/storagegroup/StorageGroupProcessor.java | 30 +-
.../apache/iotdb/db/metadata/path/AlignedPath.java | 5 +
.../iotdb/db/metadata/template/Template.java | 43 +++
.../db/metadata/template/TemplateManager.java | 11 +
.../iotdb/db/metadata/utils/MetaFormatUtils.java | 2 +-
.../apache/iotdb/db/metadata/utils/MetaUtils.java | 25 +-
.../org/apache/iotdb/db/monitor/StatMonitor.java | 2 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 8 -
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 17 ++
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 309 +++++++++++++--------
.../iotdb/db/qp/strategy/LogicalGenerator.java | 19 +-
.../utils/EmptyOutputStream.java} | 33 +--
.../db/query/aggregation/impl/SumAggrResult.java | 70 +++--
.../db/query/dataset/UDTFAlignByTimeDataSet.java | 2 +
.../iotdb/db/query/expression/Expression.java | 48 +++-
.../query/expression/binary/BinaryExpression.java | 28 +-
...TimeSeriesOperand.java => ConstantOperand.java} | 68 ++---
.../query/expression/unary/FunctionExpression.java | 12 +-
.../query/expression/unary/NegationExpression.java | 12 +-
.../query/expression/unary/TimeSeriesOperand.java | 8 +-
.../query/reader/series/AlignedSeriesReader.java | 2 +-
.../iotdb/db/query/reader/series/SeriesReader.java | 3 +-
.../db/query/udf/builtin/BuiltinFunction.java | 1 +
.../iotdb/db/query/udf/builtin/UDTFCast.java | 259 +++++++++++++++++
.../udf/core/layer/ConstantIntermediateLayer.java | 66 +++++
.../layer/MultiInputColumnIntermediateLayer.java | 8 +-
.../query/udf/core/layer/RawQueryInputLayer.java | 5 +
...InputColumnMultiReferenceIntermediateLayer.java | 5 +
.../udf/core/reader/ConstantLayerPointReader.java | 132 +++++++++
.../db/query/udf/core/reader/LayerPointReader.java | 2 +
.../transformer/ArithmeticBinaryTransformer.java | 17 ++
.../transformer/ArithmeticNegationTransformer.java | 5 +
.../udf/core/transformer/UDFQueryTransformer.java | 5 +
.../tv/ElasticSerializableTVList.java | 5 +
.../db/sync/receiver/transfer/SyncServiceImpl.java | 44 +--
.../iotdb/db/sync/sender/transfer/SyncClient.java | 69 +++--
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 12 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 10 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 149 ++++++----
.../integration/IOTDBGroupByInnerIntervalIT.java | 2 +-
.../iotdb/db/integration/IoTDBAlignByDeviceIT.java | 300 --------------------
.../iotdb/db/integration/IoTDBArithmeticIT.java | 7 +-
.../db/integration/IoTDBCreateStorageGroupIT.java | 2 +-
.../iotdb/db/integration/IoTDBDeletionIT.java | 16 +-
.../iotdb/db/integration/IoTDBGroupByMonthIT.java | 2 +-
.../iotdb/db/integration/IoTDBNestedQueryIT.java | 178 ++++++++++++
.../iotdb/db/integration/IoTDBQuotedPathIT.java | 46 +--
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 2 +-
.../db/integration/IoTDBTriggerExecutionIT.java | 160 +++++------
.../db/integration/IoTDBTriggerManagementIT.java | 172 ++++++------
.../db/integration/IoTDBUDFWindowQueryIT.java | 96 +++++++
.../integration/IoTDBUDTFAlignByTimeQueryIT.java | 42 +++
.../db/integration/IoTDBUDTFBuiltinFunctionIT.java | 34 +++
.../aggregation/IoTDBAggregationByLevelIT.java | 8 +-
.../aggregation/IoTDBAggregationSmallDataIT.java | 2 +-
.../db/integration/auth/IoTDBAuthorizationIT.java | 12 +-
.../iotdb/db/metadata/MManagerBasicTest.java | 6 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 30 +-
.../iotdb/db/monitor/IoTDBStatMonitorTest.java | 4 +-
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 4 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 6 +-
.../iotdb/db/query/dataset/ListDataSetTest.java | 2 +-
.../iotdb/db/query/dataset/SingleDataSetTest.java | 2 +-
.../dataset/groupby/GroupByLevelDataSetTest.java | 9 +-
.../dataset/groupby/GroupByTimeDataSetTest.java | 9 +-
.../iotdb/db/query/udf/example/Accumulator.java | 11 +-
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 52 ++++
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 3 +-
.../java/org/apache/iotdb/session/Session.java | 40 +--
.../org/apache/iotdb/session/pool/SessionPool.java | 3 +-
.../iotdb/session/IoTDBSessionComplexIT.java | 14 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 84 +++++-
site/src/main/.vuepress/config.js | 4 +
109 files changed, 3203 insertions(+), 1352 deletions(-)
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/manage/MetaSingleSnapshotLogManager.java
index cb512c0,87789ff..97198b9
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/MetaSingleSnapshotLogManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/MetaSingleSnapshotLogManager.java
@@@ -29,7 -29,9 +29,9 @@@ import org.apache.iotdb.db.auth.authori
import org.apache.iotdb.db.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
-import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+ import org.apache.iotdb.db.metadata.template.Template;
+ import org.apache.iotdb.db.metadata.template.TemplateManager;
import org.apache.iotdb.db.service.IoTDB;
import org.slf4j.Logger;
diff --cc cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
index 725a2b9,8915a8b..629aa70
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
@@@ -31,7 -31,9 +31,9 @@@ import org.apache.iotdb.db.engine.Stora
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
-import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+ import org.apache.iotdb.db.metadata.template.Template;
+ import org.apache.iotdb.db.metadata.template.TemplateManager;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.SerializeUtils;
diff --cc cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshotTest.java
index df74f37,a5c06c8..18c99c4
--- a/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshotTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshotTest.java
@@@ -31,7 -32,11 +32,11 @@@ import org.apache.iotdb.db.auth.entity.
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+ import org.apache.iotdb.db.exception.metadata.UndefinedTemplateException;
-import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+ import org.apache.iotdb.db.metadata.template.Template;
+ import org.apache.iotdb.db.metadata.template.TemplateManager;
+ import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
import org.apache.iotdb.db.service.IoTDB;
import org.junit.After;
diff --cc cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
index 0703a4c,1c0f8c8..bdb8f93
--- a/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/server/member/MetaGroupMemberTest.java
@@@ -87,11 -65,13 +88,14 @@@ import org.apache.iotdb.db.exception.me
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
+import org.apache.iotdb.db.metadata.path.MeasurementPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+ import org.apache.iotdb.db.metadata.template.Template;
+ import org.apache.iotdb.db.metadata.template.TemplateManager;
import org.apache.iotdb.db.qp.executor.PlanExecutor;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+ import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan;
diff --cc server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index cd5497e,cd5497e..94d834a
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@@ -24,7 -24,7 +24,6 @@@ import org.apache.iotdb.db.conf.IoTDBCo
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.TestOnly;
--import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@@ -189,24 -189,24 +188,16 @@@ public class TimeSeriesMetadataCache
* @param key vector's own fullPath, e.g. root.sg1.d1.vector
* @param subSensorList all subSensors of this vector in query, e.g. [vector.s1, vector.s2,
* vector.s3]
-- * @param allSensors all sensors belonging to this device that appear in query. For vector, this
-- * should contain both vector name and subSensors' name, e.g. [vector, vector.s1, vector.s2,
-- * vector.s3]
*/
// Suppress synchronize warning
// Suppress high Cognitive Complexity warning
@SuppressWarnings({"squid:S1860", "squid:S3776"})
public List<TimeseriesMetadata> get(
-- TimeSeriesMetadataCacheKey key,
-- List<String> subSensorList,
-- Set<String> allSensors,
-- boolean debug)
++ TimeSeriesMetadataCacheKey key, List<String> subSensorList, boolean debug)
throws IOException {
-- // put all sub sensors into allSensors
-- for (int i = 0; i < subSensorList.size(); i++) {
-- subSensorList.set(i, key.measurement + TsFileConstant.PATH_SEPARATOR + subSensorList.get(i));
-- }
-- allSensors.addAll(subSensorList);
++ Set<String> allSensors = new HashSet<>(subSensorList);
++ // add the time column
++ allSensors.add("");
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@@ -215,10 -215,10 +206,7 @@@
&& !bloomFilter.contains(key.device + IoTDBConstant.PATH_SEPARATOR + key.measurement)) {
return Collections.emptyList();
}
-- // for the condition that cache is disabled, we only get what we need
-- Set<String> allSensorSet = new HashSet<>(subSensorList);
-- allSensorSet.add(key.measurement);
-- return readTimeseriesMetadataForVector(reader, key, subSensorList, allSensorSet);
++ return readTimeseriesMetadataForVector(reader, key, subSensorList, allSensors);
}
List<TimeseriesMetadata> res = new ArrayList<>();
@@@ -244,7 -244,7 +232,6 @@@
if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
-- allSensors.removeAll(subSensorList);
return Collections.emptyList();
}
res = readTimeseriesMetadataForVector(reader, key, subSensorList, allSensors);
@@@ -280,7 -280,7 +267,6 @@@
}
}
-- allSensors.removeAll(subSensorList);
return res;
}
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
index caafedd,0000000..43397ba
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java
@@@ -1,349 -1,0 +1,354 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.path;
+
+import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
+import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.AlignedSeriesReader;
+import org.apache.iotdb.db.utils.TestOnly;
+import org.apache.iotdb.db.utils.datastructure.TVList;
+import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * VectorPartialPath represents a vector's fullPath. It not only contains the full path of vector's
+ * own name, but also has subSensorsList which contain all the fullPath of vector's sub sensors.
+ * e.g. VectorPartialPath1(root.sg1.d1.vector1, [root.sg1.d1.vector1.s1, root.sg1.d1.vector1.s2])
+ * VectorPartialPath2(root.sg1.d1.vector2, [root.sg1.d1.vector2.s1, root.sg1.d1.vector2.s2])
+ */
+public class AlignedPath extends PartialPath {
+
+ // todo improve vector implementation by remove this placeholder
+ public static final String VECTOR_PLACEHOLDER = "";
+
+ private List<String> measurementList;
+ private List<IMeasurementSchema> schemaList;
+
+ public AlignedPath() {}
+
+ public AlignedPath(String vectorPath, List<String> subSensorsList) throws IllegalPathException {
+ super(vectorPath);
+ this.measurementList = subSensorsList;
+ }
+
+ public AlignedPath(
+ String vectorPath, List<String> measurementList, List<IMeasurementSchema> schemaList)
+ throws IllegalPathException {
+ super(vectorPath);
+ this.measurementList = measurementList;
+ this.schemaList = schemaList;
+ }
+
+ public AlignedPath(String vectorPath, String subSensor) throws IllegalPathException {
+ super(vectorPath);
+ measurementList = new ArrayList<>();
+ measurementList.add(subSensor);
+ }
+
+ public AlignedPath(PartialPath vectorPath, String subSensor) {
+ super(vectorPath.getNodes());
+ measurementList = new ArrayList<>();
+ measurementList.add(subSensor);
+ }
+
+ public AlignedPath(MeasurementPath path) {
+ super(path.getDevicePath().getNodes());
+ measurementList = new ArrayList<>();
+ measurementList.add(path.getMeasurement());
+ schemaList = new ArrayList<>();
+ schemaList.add(path.getMeasurementSchema());
+ }
+
+ @Override
+ public String getDevice() {
+ return getFullPath();
+ }
+
++ @Override
++ public String getMeasurement() {
++ throw new UnsupportedOperationException("AlignedPath doesn't have measurement name!");
++ }
++
+ public List<String> getMeasurementList() {
+ return measurementList;
+ }
+
+ public String getMeasurement(int index) {
+ return measurementList.get(index);
+ }
+
+ public PartialPath getPathWithMeasurement(int index) {
+ return new PartialPath(nodes).concatNode(measurementList.get(index));
+ }
+
+ public void setMeasurementList(List<String> measurementList) {
+ this.measurementList = measurementList;
+ }
+
+ public void addMeasurement(String measurement) {
+ this.measurementList.add(measurement);
+ }
+
+ public void addMeasurement(List<String> measurements) {
+ this.measurementList.addAll(measurements);
+ }
+
+ public void addMeasurement(MeasurementPath measurementPath) {
+ if (measurementList == null) {
+ measurementList = new ArrayList<>();
+ }
+ measurementList.add(measurementPath.getMeasurement());
+
+ if (schemaList == null) {
+ schemaList = new ArrayList<>();
+ }
+ schemaList.add(measurementPath.getMeasurementSchema());
+ }
+
+ public void addMeasurement(List<String> measurementList, List<IMeasurementSchema> schemaList) {
+ this.measurementList.addAll(measurementList);
+ if (this.schemaList == null) {
+ this.schemaList = new ArrayList<>();
+ }
+ this.schemaList.addAll(schemaList);
+ }
+
+ public List<IMeasurementSchema> getSchemaList() {
+ return this.schemaList == null ? Collections.emptyList() : this.schemaList;
+ }
+
+ public VectorMeasurementSchema getMeasurementSchema() {
+ TSDataType[] types = new TSDataType[measurementList.size()];
+ TSEncoding[] encodings = new TSEncoding[measurementList.size()];
+
+ for (int i = 0; i < measurementList.size(); i++) {
+ types[i] = schemaList.get(i).getType();
+ encodings[i] = schemaList.get(i).getEncodingType();
+ }
+ String[] array = new String[measurementList.size()];
+ for (int i = 0; i < array.length; i++) {
+ array[i] = measurementList.get(i);
+ }
+ return new VectorMeasurementSchema(
+ VECTOR_PLACEHOLDER, array, types, encodings, schemaList.get(0).getCompressor());
+ }
+
+ public TSDataType getSeriesType() {
+ return getMeasurementSchema().getType();
+ }
+
+ @Override
+ public PartialPath copy() {
+ AlignedPath result = new AlignedPath();
+ result.nodes = nodes;
+ result.fullPath = fullPath;
+ result.device = device;
+ result.measurementList = new ArrayList<>(measurementList);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ AlignedPath that = (AlignedPath) o;
+ return Objects.equals(measurementList, that.measurementList);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), measurementList);
+ }
+
+ @Override
+ public AlignedSeriesReader createSeriesReader(
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ QueryDataSource dataSource,
+ Filter timeFilter,
+ Filter valueFilter,
+ TsFileFilter fileFilter,
+ boolean ascending) {
+ return new AlignedSeriesReader(
+ this,
+ allSensors,
+ dataType,
+ context,
+ dataSource,
+ timeFilter,
+ valueFilter,
+ fileFilter,
+ ascending);
+ }
+
+ @Override
+ @TestOnly
+ public AlignedSeriesReader createSeriesReader(
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ List<TsFileResource> seqFileResource,
+ List<TsFileResource> unseqFileResource,
+ Filter timeFilter,
+ Filter valueFilter,
+ boolean ascending) {
+ return new AlignedSeriesReader(
+ this,
+ allSensors,
+ dataType,
+ context,
+ seqFileResource,
+ unseqFileResource,
+ timeFilter,
+ valueFilter,
+ ascending);
+ }
+
+ @Override
+ public TsFileResource createTsFileResource(
+ List<ReadOnlyMemChunk> readOnlyMemChunk,
+ List<IChunkMetadata> chunkMetadataList,
+ TsFileResource originTsFileResource)
+ throws IOException {
+ TsFileResource tsFileResource =
+ new TsFileResource(readOnlyMemChunk, chunkMetadataList, originTsFileResource);
+ tsFileResource.setTimeSeriesMetadata(
+ generateTimeSeriesMetadata(readOnlyMemChunk, chunkMetadataList));
+ return tsFileResource;
+ }
+
+ /**
+ * Because the unclosed tsfile don't have TimeSeriesMetadata and memtables in the memory don't
+ * have chunkMetadata, but query will use these, so we need to generate it for them.
+ */
+ private AlignedTimeSeriesMetadata generateTimeSeriesMetadata(
+ List<ReadOnlyMemChunk> readOnlyMemChunk, List<IChunkMetadata> chunkMetadataList)
+ throws IOException {
+ TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata();
+ timeTimeSeriesMetadata.setOffsetOfChunkMetaDataList(-1);
+ timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1);
+ timeTimeSeriesMetadata.setMeasurementId("");
+ timeTimeSeriesMetadata.setTSDataType(TSDataType.VECTOR);
+
+ Statistics<? extends Serializable> timeStatistics =
+ Statistics.getStatsByType(timeTimeSeriesMetadata.getTSDataType());
+
+ // init each value time series meta
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList = new ArrayList<>();
+ for (IMeasurementSchema valueChunkMetadata : schemaList) {
+ TimeseriesMetadata valueMetadata = new TimeseriesMetadata();
+ valueMetadata.setOffsetOfChunkMetaDataList(-1);
+ valueMetadata.setDataSizeOfChunkMetaDataList(-1);
+ valueMetadata.setMeasurementId(valueChunkMetadata.getMeasurementId());
+ valueMetadata.setTSDataType(valueChunkMetadata.getType());
+ valueMetadata.setStatistics(Statistics.getStatsByType(valueChunkMetadata.getType()));
+ valueTimeSeriesMetadataList.add(valueMetadata);
+ }
+
+ for (IChunkMetadata chunkMetadata : chunkMetadataList) {
+ AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata;
+ timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ valueTimeSeriesMetadataList
+ .get(i)
+ .getStatistics()
+ .mergeStatistics(
+ alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics());
+ }
+ }
+
+ for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
+ if (!memChunk.isEmpty()) {
+ AlignedChunkMetadata alignedChunkMetadata =
+ (AlignedChunkMetadata) memChunk.getChunkMetaData();
+ timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ valueTimeSeriesMetadataList
+ .get(i)
+ .getStatistics()
+ .mergeStatistics(
+ alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics());
+ }
+ }
+ }
+ timeTimeSeriesMetadata.setStatistics(timeStatistics);
+
+ return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList);
+ }
+
+ @Override
+ public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable(
+ Map<String, Map<String, IWritableMemChunk>> memTableMap, List<TimeRange> deletionList)
+ throws QueryProcessException, IOException {
+ // check If Memtable Contains this path
+ if (!memTableMap.containsKey(getDevice())) {
+ return null;
+ }
+ AlignedWritableMemChunk vectorMemChunk =
+ ((AlignedWritableMemChunk) memTableMap.get(getDevice()).get(VECTOR_PLACEHOLDER));
+ boolean containsMeasurement = false;
+ for (String measurement : measurementList) {
+ if (vectorMemChunk.containsMeasurement(measurement)) {
+ containsMeasurement = true;
+ break;
+ }
+ }
+ if (!containsMeasurement) {
+ return null;
+ }
+ // get sorted tv list is synchronized so different query can get right sorted list reference
+ TVList vectorTvListCopy = vectorMemChunk.getSortedTvListForQuery(schemaList);
+ int curSize = vectorTvListCopy.size();
+ return new ReadOnlyMemChunk(getMeasurementSchema(), vectorTvListCopy, curSize, deletionList);
+ }
+}
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
index b967abf,addc879..f84e554
--- a/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
@@@ -18,14 -18,25 +18,21 @@@
*/
package org.apache.iotdb.db.metadata.template;
-import org.apache.iotdb.db.metadata.PartialPath;
-import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
-import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
+ import org.apache.iotdb.db.utils.SerializeUtils;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+ import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema;
import org.apache.commons.lang3.builder.HashCodeBuilder;
+ import java.io.ByteArrayOutputStream;
+ import java.io.DataOutputStream;
+ import java.io.IOException;
+ import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
public class Template {
private String name;
@@@ -79,10 -128,92 +88,44 @@@
return schemaMap.containsKey(measurementId);
}
- public List<IMeasurementMNode> getMeasurementMNode() {
- Set<IMeasurementSchema> deduplicateSchema = new HashSet<>();
- List<IMeasurementMNode> res = new ArrayList<>();
-
- for (IMeasurementSchema measurementSchema : schemaMap.values()) {
- if (deduplicateSchema.add(measurementSchema)) {
- IMeasurementMNode measurementMNode = null;
- if (measurementSchema instanceof UnaryMeasurementSchema) {
- measurementMNode =
- MeasurementMNode.getMeasurementMNode(
- null, measurementSchema.getMeasurementId(), measurementSchema, null);
-
- } else if (measurementSchema instanceof VectorMeasurementSchema) {
- measurementMNode =
- MeasurementMNode.getMeasurementMNode(
- null,
- getMeasurementNodeName(measurementSchema.getMeasurementId()),
- measurementSchema,
- null);
- }
-
- res.add(measurementMNode);
- }
- }
-
- return res;
- }
-
- public String getMeasurementNodeName(String measurementName) {
- return schemaMap.get(measurementName).getMeasurementId();
- }
-
- /**
- * get all path in this template (to support aligned by device query)
- *
- * @return a hash map looks like below {vector -> [s1, s2, s3] normal_timeseries -> []}
- */
- public HashMap<String, List<String>> getAllPath() {
- HashMap<String, List<String>> res = new HashMap<>();
- for (Map.Entry<String, IMeasurementSchema> schemaEntry : schemaMap.entrySet()) {
- if (schemaEntry.getValue() instanceof VectorMeasurementSchema) {
- VectorMeasurementSchema vectorMeasurementSchema =
- (VectorMeasurementSchema) schemaEntry.getValue();
- res.put(schemaEntry.getKey(), vectorMeasurementSchema.getSubMeasurementsList());
- } else {
- res.put(schemaEntry.getKey(), new ArrayList<>());
- }
- }
-
- return res;
+ public IMeasurementSchema getSchema(String measurementId) {
+ return schemaMap.get(measurementId);
}
+ public ByteBuffer serialize() {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+
+ SerializeUtils.serialize(name, dataOutputStream);
+ try {
+ dataOutputStream.writeInt(schemaMap.size());
+ for (Map.Entry<String, IMeasurementSchema> entry : schemaMap.entrySet()) {
+ SerializeUtils.serialize(entry.getKey(), dataOutputStream);
+ entry.getValue().partialSerializeTo(dataOutputStream);
+ }
+ } catch (IOException e) {
+ // unreachable
+ }
+ return ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+ }
+
+ public void deserialize(ByteBuffer buffer) {
+ name = SerializeUtils.deserializeString(buffer);
+ int schemaSize = buffer.getInt();
+ schemaMap = new HashMap<>(schemaSize);
+ for (int i = 0; i < schemaSize; i++) {
+ String schemaName = SerializeUtils.deserializeString(buffer);
+ byte flag = ReadWriteIOUtils.readByte(buffer);
+ IMeasurementSchema measurementSchema = null;
+ if (flag == (byte) 0) {
+ measurementSchema = UnaryMeasurementSchema.partialDeserializeFrom(buffer);
+ } else if (flag == (byte) 1) {
+ measurementSchema = VectorMeasurementSchema.partialDeserializeFrom(buffer);
+ }
+ schemaMap.put(schemaName, measurementSchema);
+ }
+ }
+
@Override
public boolean equals(Object t) {
if (this == t) {
diff --cc server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
index d2af06c,8fe80cb..01d1234
--- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
@@@ -21,9 -21,10 +21,10 @@@ package org.apache.iotdb.db.qp.sql
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.trigger.executor.TriggerEvent;
+ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.runtime.SQLParserException;
import org.apache.iotdb.db.index.common.IndexType;
-import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.constant.FilterConstant;
import org.apache.iotdb.db.qp.constant.FilterConstant.FilterType;
import org.apache.iotdb.db.qp.constant.SQLConstant;
diff --cc server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
index 242809c,21f4dc4..5558b50
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/Expression.java
@@@ -21,9 -21,10 +21,10 @@@ package org.apache.iotdb.db.query.expre
import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.UDTFPlan;
import org.apache.iotdb.db.qp.utils.WildcardsRemover;
+ import org.apache.iotdb.db.query.expression.unary.ConstantOperand;
import org.apache.iotdb.db.query.udf.core.executor.UDTFExecutor;
import org.apache.iotdb.db.query.udf.core.layer.IntermediateLayer;
import org.apache.iotdb.db.query.udf.core.layer.LayerMemoryAssigner;
diff --cc server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
index eb08fd4,eb5dd59..7f4bb48
--- a/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/expression/unary/TimeSeriesOperand.java
@@@ -114,8 -119,8 +119,7 @@@ public class TimeSeriesOperand extends
return expressionIntermediateLayerMap.get(this);
}
-- @Override
- public String toString() {
+ public String getExpressionStringInternal() {
- return path.isMeasurementAliasExists() ? path.getFullPathWithAlias() : path.getExactFullPath();
+ return path.isMeasurementAliasExists() ? path.getFullPathWithAlias() : path.getFullPath();
}
}
diff --cc server/src/main/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesReader.java
index 4b91a0f,0000000..cebc792
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/AlignedSeriesReader.java
@@@ -1,95 -1,0 +1,95 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.query.reader.series;
+
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.metadata.path.AlignedPath;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.db.utils.TestOnly;
+import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+public class AlignedSeriesReader extends SeriesReader {
+
+ public AlignedSeriesReader(
+ PartialPath seriesPath,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ QueryDataSource dataSource,
+ Filter timeFilter,
+ Filter valueFilter,
+ TsFileFilter fileFilter,
+ boolean ascending) {
+ super(
+ seriesPath,
+ allSensors,
+ dataType,
+ context,
+ dataSource,
+ timeFilter,
+ valueFilter,
+ fileFilter,
+ ascending);
+ }
+
+ @TestOnly
+ public AlignedSeriesReader(
+ PartialPath seriesPath,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ List<TsFileResource> seqFileResource,
+ List<TsFileResource> unseqFileResource,
+ Filter timeFilter,
+ Filter valueFilter,
+ boolean ascending) {
+ super(
+ seriesPath,
+ allSensors,
+ dataType,
+ context,
+ seqFileResource,
+ unseqFileResource,
+ timeFilter,
+ valueFilter,
+ ascending);
+ }
+
+ @Override
+ protected ITimeSeriesMetadata loadTimeSeriesMetadata(
+ TsFileResource resource,
+ PartialPath seriesPath,
+ QueryContext context,
+ Filter filter,
+ Set<String> allSensors)
+ throws IOException {
+ return FileLoaderUtils.loadTimeSeriesMetadata(
- resource, (AlignedPath) seriesPath, context, filter, allSensors);
++ resource, (AlignedPath) seriesPath, context, filter);
+ }
+}
diff --cc server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index c27a2d3,c2af73e..05e6e8b
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@@ -150,7 -153,7 +150,6 @@@ public class SeriesReader
boolean ascending) {
this.seriesPath = seriesPath;
this.allSensors = allSensors;
-- this.allSensors.add(seriesPath.getMeasurement());
this.dataType = dataType;
this.context = context;
QueryUtils.filterQueryDataSource(dataSource, fileFilter);
@@@ -1011,7 -1014,7 +1010,7 @@@
protected void unpackSeqTsFileResource() throws IOException {
ITimeSeriesMetadata timeseriesMetadata =
-- FileLoaderUtils.loadTimeSeriesMetadata(
++ loadTimeSeriesMetadata(
orderUtils.getNextSeqFileResource(seqFileResource, true),
seriesPath,
context,
diff --cc server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 854645b,3d2931b..e1e4594
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@@ -152,16 -157,16 +152,11 @@@ public class FileLoaderUtils
* @param resource corresponding TsFileResource
* @param vectorPath instance of VectorPartialPath, vector's full path, e.g. (root.sg1.d1.vector,
* [root.sg1.d1.vector.s1, root.sg1.d1.vector.s2])
-- * @param allSensors all sensors belonging to this device that appear in query
*/
- private static VectorTimeSeriesMetadata loadVectorTimeSeriesMetadata(
- TsFileResource resource,
- VectorPartialPath vectorPath,
- QueryContext context,
- Filter filter,
- Set<String> allSensors)
+ public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
- TsFileResource resource,
- AlignedPath vectorPath,
- QueryContext context,
- Filter filter,
- Set<String> allSensors)
++ TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter)
throws IOException {
- VectorTimeSeriesMetadata vectorTimeSeriesMetadata = null;
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
// If the tsfile is closed, we need to load from tsfile
if (resource.isClosed()) {
if (!resource.getTsFile().exists()) {
@@@ -174,11 -179,11 +169,8 @@@
TimeSeriesMetadataCache.getInstance()
.get(
new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
-- resource.getTsFilePath(),
-- vectorPath.getDevice(),
-- vectorPath.getMeasurement()),
- new ArrayList<>(vectorPath.getSubSensorsList()),
- allSensors,
++ resource.getTsFilePath(), vectorPath.getDevice(), ""),
+ new ArrayList<>(vectorPath.getMeasurementList()),
- allSensors,
context.isDebug());
// assemble VectorTimeSeriesMetadata