You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/05/19 01:58:00 UTC
[iotdb] branch rel/0.12 updated: Filter RowRecord automatically if
any column in it is null or all columns are null (#3217)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new d820166 Filter RowRecord automatically if any column in it is null or all columns are null (#3217)
d820166 is described below
commit d820166108f1a4edeb7a2dfd5af02baefe22550e
Author: Jackie Tien <Ja...@foxmail.com>
AuthorDate: Wed May 19 09:57:24 2021 +0800
Filter RowRecord automatically if any column in it is null or all columns are null (#3217)
---
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 22 +-
.../DML-Data-Manipulation-Language.md | 14 ++
.../DML-Data-Manipulation-Language.md | 15 ++
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +
.../iotdb/db/qp/logical/crud/QueryOperator.java | 22 ++
.../iotdb/db/qp/physical/crud/QueryPlan.java | 22 ++
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 63 +++---
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 5 +-
.../dataset/RawQueryDataSetWithoutValueFilter.java | 68 +++++--
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 20 +-
.../db/integration/IoTDBWithoutAllNullIT.java | 192 ++++++++++++++++++
.../db/integration/IoTDBWithoutAnyNullIT.java | 221 +++++++++++++++++++++
.../apache/iotdb/tsfile/read/common/RowRecord.java | 42 +++-
.../tsfile/read/query/dataset/QueryDataSet.java | 29 ++-
14 files changed, 661 insertions(+), 76 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
index 6a027cd..54d9aab 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
@@ -251,15 +251,19 @@ specialClause
| orderByTimeClause specialLimit? #orderByTimeStatement
| groupByTimeClause orderByTimeClause? specialLimit? #groupByTimeStatement
| groupByFillClause orderByTimeClause? specialLimit? #groupByFillStatement
- | fillClause slimitClause? alignByDeviceClauseOrDisableAlign? #fillStatement
- | alignByDeviceClauseOrDisableAlign #alignByDeviceStatementOrDisableAlignInSpecialClause
| groupByLevelClause orderByTimeClause? specialLimit? #groupByLevelStatement
+ | fillClause slimitClause? alignByDeviceClauseOrDisableAlign? #fillStatement
;
specialLimit
: limitClause slimitClause? alignByDeviceClauseOrDisableAlign? #limitStatement
| slimitClause limitClause? alignByDeviceClauseOrDisableAlign? #slimitStatement
- | alignByDeviceClauseOrDisableAlign #alignByDeviceClauseOrDisableAlignInSpecialLimit
+ | withoutNullClause limitClause? slimitClause? alignByDeviceClauseOrDisableAlign? #withoutNullStatement
+ | alignByDeviceClauseOrDisableAlign #alignByDeviceClauseOrDisableAlignStatement
+ ;
+
+withoutNullClause
+ : WITHOUT NULL (ALL | ANY)
;
orderByTimeClause
@@ -1294,6 +1298,18 @@ DEBUG
: D E B U G
;
+NULL
+ : N U L L
+ ;
+
+WITHOUT
+ : W I T H O U T
+ ;
+
+ANY
+ : A N Y
+ ;
+
//============================
// End of the keywords list
//============================
diff --git a/docs/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md b/docs/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
index c7540d2..93b38cc 100644
--- a/docs/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
+++ b/docs/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
@@ -1293,6 +1293,20 @@ Total line number = 10
It costs 0.009s
```
+#### Null Value Control over Query Results
+
+* IoTDB will join all the sensor value by its time, and if some sensors don't have values in that timestamp, we will fill it with null. In some analysis scenarios, we only need the row if all the columns of it have value.
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ANY
+```
+
+* In group by query, we will fill null for any group by interval if the columns don't have values in that group by interval. However, if all columns in that group by interval are null, maybe users don't need that RowRecord, so we can use `WITHOUT NULL ALL` to filter that row.
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ALL
+```
+
### Use Alias
Since the unique data model of IoTDB, lots of additional information like device will be carried before each sensor. Sometimes, we want to query just one specific device, then these prefix information show frequently will be redundant in this situation, influencing the analysis of result set. At this time, we can use `AS` function provided by IoTDB, assign an alias to time series selected in query.
diff --git a/docs/zh/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md b/docs/zh/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
index d4a4e74..a568a71 100644
--- a/docs/zh/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
+++ b/docs/zh/UserGuide/IoTDB-SQL-Language/DML-Data-Manipulation-Language.md
@@ -282,6 +282,21 @@ It costs 0.018s
更多语法请参照[SQL REFERENCE](../Appendix/SQL-Reference.md)。
+#### 结果空值过滤
+
+* 如果结果集中，任意一列为null，则过滤掉该行；即获得的结果集不包含任何空值
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ANY
+```
+
+* 在降采样查询中，如果结果集的某一行所有列都为null，则过滤掉该行；即获得的结果集不包含所有值都为null的行
+
+```
+select * from root.ln.* where time <= 2017-11-01T00:01:00 WITHOUT NULL ALL
+```
+
+
#### 聚合函数
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index c3156de..ff2e37b 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -508,6 +508,8 @@ public class PlanExecutor implements IPlanExecutor {
}
queryDataSet.setRowLimit(queryPlan.getRowLimit());
queryDataSet.setRowOffset(queryPlan.getRowOffset());
+ queryDataSet.setWithoutAllNull(queryPlan.isWithoutAllNull());
+ queryDataSet.setWithoutAnyNull(queryPlan.isWithoutAnyNull());
return queryDataSet;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java
index 87aa1d9..c640470 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/QueryOperator.java
@@ -62,6 +62,12 @@ public class QueryOperator extends SFWOperator {
private IndexType indexType;
+ // if true, we don't need the row whose any column is null
+ private boolean withoutAnyNull;
+
+ // if true, we don't need the row whose all columns are null
+ private boolean withoutAllNull;
+
public QueryOperator(int tokenIntType) {
super(tokenIntType);
operatorType = Operator.OperatorType.QUERY;
@@ -250,4 +256,20 @@ public class QueryOperator extends SFWOperator {
public void setAscending(boolean ascending) {
this.ascending = ascending;
}
+
+ public boolean isWithoutAnyNull() {
+ return withoutAnyNull;
+ }
+
+ public void setWithoutAnyNull(boolean withoutAnyNull) {
+ this.withoutAnyNull = withoutAnyNull;
+ }
+
+ public boolean isWithoutAllNull() {
+ return withoutAllNull;
+ }
+
+ public void setWithoutAllNull(boolean withoutAllNull) {
+ this.withoutAllNull = withoutAllNull;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java
index a188db5..a602c16 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/QueryPlan.java
@@ -44,6 +44,12 @@ public abstract class QueryPlan extends PhysicalPlan {
private boolean enableRedirect = false;
+ // if true, we don't need the row whose any column is null
+ private boolean withoutAnyNull;
+
+ // if true, we don't need the row whose all columns are null
+ private boolean withoutAllNull;
+
public QueryPlan() {
super(true);
setOperatorType(Operator.OperatorType.QUERY);
@@ -136,4 +142,20 @@ public abstract class QueryPlan extends PhysicalPlan {
public void setEnableRedirect(boolean enableRedirect) {
this.enableRedirect = enableRedirect;
}
+
+ public boolean isWithoutAnyNull() {
+ return withoutAnyNull;
+ }
+
+ public void setWithoutAnyNull(boolean withoutAnyNull) {
+ this.withoutAnyNull = withoutAnyNull;
+ }
+
+ public boolean isWithoutAllNull() {
+ return withoutAllNull;
+ }
+
+ public void setWithoutAllNull(boolean withoutAllNull) {
+ this.withoutAllNull = withoutAllNull;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
index acb84fb..ad681e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
@@ -80,8 +80,8 @@ import org.apache.iotdb.db.qp.logical.sys.TracingOperator;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AggregationCallContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AggregationElementContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AliasClauseContext;
-import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignInSpecialLimitContext;
-import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceStatementOrDisableAlignInSpecialClauseContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlignByDeviceClauseOrDisableAlignStatementContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterClauseContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterTimeseriesContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.AlterUserContext;
@@ -212,6 +212,7 @@ import org.apache.iotdb.db.qp.sql.SqlBaseParser.UdfAttributeContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.UdfCallContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.UnsetTTLStatementContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.WhereClauseContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.WithoutNullStatementContext;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.query.executor.fill.IFill;
import org.apache.iotdb.db.query.executor.fill.LinearFill;
@@ -1133,11 +1134,7 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
parseSlimitClause(ctx.slimitClause(), queryOp);
}
if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
- if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
- parseAlignByDeviceClause(queryOp);
- } else {
- parseDisableAlign(queryOp);
- }
+ parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
}
return queryOp;
}
@@ -1149,23 +1146,42 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
parseLimitClause(ctx.limitClause(), queryOp);
}
if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
- if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
- parseAlignByDeviceClause(queryOp);
- } else {
- parseDisableAlign(queryOp);
- }
+ parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
}
return queryOp;
}
@Override
- public Operator visitAlignByDeviceClauseOrDisableAlignInSpecialLimit(
- AlignByDeviceClauseOrDisableAlignInSpecialLimitContext ctx) {
- if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
+ public Operator visitAlignByDeviceClauseOrDisableAlignStatement(
+ AlignByDeviceClauseOrDisableAlignStatementContext ctx) {
+ parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
+ return queryOp;
+ }
+
+ private void parseAlignByDeviceClauseOrDisableAlign(
+ AlignByDeviceClauseOrDisableAlignContext ctx) {
+ if (ctx.alignByDeviceClause() != null) {
parseAlignByDeviceClause(queryOp);
} else {
parseDisableAlign(queryOp);
}
+ }
+
+ @Override
+ public Operator visitWithoutNullStatement(WithoutNullStatementContext ctx) {
+ if (ctx.withoutNullClause().WITHOUT() != null) {
+ queryOp.setWithoutAllNull(ctx.withoutNullClause().ALL() != null);
+ queryOp.setWithoutAnyNull(ctx.withoutNullClause().ANY() != null);
+ }
+ if (ctx.limitClause() != null) {
+ parseLimitClause(ctx.limitClause(), queryOp);
+ }
+ if (ctx.slimitClause() != null) {
+ parseSlimitClause(ctx.slimitClause(), queryOp);
+ }
+ if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
+ parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
+ }
return queryOp;
}
@@ -1209,22 +1225,7 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
parseSlimitClause(ctx.slimitClause(), queryOp);
}
if (ctx.alignByDeviceClauseOrDisableAlign() != null) {
- if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
- parseAlignByDeviceClause(queryOp);
- } else {
- parseDisableAlign(queryOp);
- }
- }
- return queryOp;
- }
-
- @Override
- public Operator visitAlignByDeviceStatementOrDisableAlignInSpecialClause(
- AlignByDeviceStatementOrDisableAlignInSpecialClauseContext ctx) {
- if (ctx.alignByDeviceClauseOrDisableAlign().alignByDeviceClause() != null) {
- parseAlignByDeviceClause(queryOp);
- } else {
- parseDisableAlign(queryOp);
+ parseAlignByDeviceClauseOrDisableAlign(ctx.alignByDeviceClauseOrDisableAlign());
}
return queryOp;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
index dd514d3..ef3ca16 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/PhysicalGenerator.java
@@ -519,7 +519,7 @@ public class PhysicalGenerator {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private PhysicalPlan transformQuery(QueryOperator queryOperator, int fetchSize)
throws QueryProcessException {
- QueryPlan queryPlan = null;
+ QueryPlan queryPlan;
if (queryOperator.hasAggregation()) {
queryPlan = new AggPhysicalPlanRule().transform(queryOperator, fetchSize);
@@ -566,6 +566,9 @@ public class PhysicalGenerator {
}
}
+ queryPlan.setWithoutAllNull(queryOperator.isWithoutAllNull());
+ queryPlan.setWithoutAnyNull(queryOperator.isWithoutAnyNull());
+
if (queryOperator.getIndexType() != null) {
if (queryPlan instanceof QueryIndexPlan) {
((QueryIndexPlan) queryPlan).setIndexType(queryOperator.getIndexType());
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithoutValueFilter.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithoutValueFilter.java
index c46fe1d..d135519 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithoutValueFilter.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/RawQueryDataSetWithoutValueFilter.java
@@ -55,7 +55,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
private final ManagedSeriesReader reader;
private final String pathName;
- private BlockingQueue<BatchData> blockingQueue;
+ private final BlockingQueue<BatchData> blockingQueue;
public ReadTask(
ManagedSeriesReader reader, BlockingQueue<BatchData> blockingQueue, String pathName) {
@@ -243,6 +243,10 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
long minTime = timeHeap.pollFirst();
+ if (withoutAnyNull && filterRowRecord(seriesNum, minTime)) {
+ continue;
+ }
+
if (rowOffset == 0) {
timeBAOS.write(BytesUtils.longToBytes(minTime));
}
@@ -304,22 +308,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
}
}
- // move next
- cachedBatchDataArray[seriesIndex].next();
-
- // check the interrupted status of query before taking next batch
- QueryTimeManager.checkQueryAlive(queryId);
-
- // get next batch if current batch is empty and still have remaining batch data in queue
- if (!cachedBatchDataArray[seriesIndex].hasCurrent()
- && !noMoreDataInQueueArray[seriesIndex]) {
- fillCache(seriesIndex);
- }
-
- // try to put the next timestamp into the heap
- if (cachedBatchDataArray[seriesIndex].hasCurrent()) {
- timeHeap.add(cachedBatchDataArray[seriesIndex].currentTime());
- }
+ prepareForNext(seriesIndex);
}
}
@@ -381,6 +370,49 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
return tsQueryDataSet;
}
+ /** if any column in the row record is null, we filter it. */
+ private boolean filterRowRecord(int seriesNum, long minTime)
+ throws IOException, InterruptedException {
+ boolean hasNull = false;
+ for (int seriesIndex = 0; seriesIndex < seriesNum; seriesIndex++) {
+ if (cachedBatchDataArray[seriesIndex] == null
+ || !cachedBatchDataArray[seriesIndex].hasCurrent()
+ || cachedBatchDataArray[seriesIndex].currentTime() != minTime) {
+ hasNull = true;
+ break;
+ }
+ }
+ if (hasNull) {
+ for (int seriesIndex = 0; seriesIndex < seriesNum; seriesIndex++) {
+ if (cachedBatchDataArray[seriesIndex] != null
+ && cachedBatchDataArray[seriesIndex].hasCurrent()
+ && cachedBatchDataArray[seriesIndex].currentTime() == minTime) {
+ prepareForNext(seriesIndex);
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private void prepareForNext(int seriesIndex) throws IOException, InterruptedException {
+ // move next
+ cachedBatchDataArray[seriesIndex].next();
+
+ // check the interrupted status of query before taking next batch
+ QueryTimeManager.checkQueryAlive(queryId);
+
+ // get next batch if current batch is empty and still have remaining batch data in queue
+ if (!cachedBatchDataArray[seriesIndex].hasCurrent() && !noMoreDataInQueueArray[seriesIndex]) {
+ fillCache(seriesIndex);
+ }
+
+ // try to put the next timestamp into the heap
+ if (cachedBatchDataArray[seriesIndex].hasCurrent()) {
+ timeHeap.add(cachedBatchDataArray[seriesIndex].currentTime());
+ }
+ }
+
protected void fillCache(int seriesIndex) throws IOException, InterruptedException {
BatchData batchData = blockingQueueArray[seriesIndex].take();
// no more batch data in this time series queue
@@ -462,7 +494,7 @@ public class RawQueryDataSetWithoutValueFilter extends QueryDataSet
public Object[] nextRowInObjects() throws IOException {
int seriesNumber = seriesReaderList.size();
- Long minTime = timeHeap.pollFirst();
+ long minTime = timeHeap.pollFirst();
Object[] rowInObjects = new Object[seriesNumber + 1];
rowInObjects[seriesNumber] = minTime;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
index 705c0cf..a88b074 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
@@ -42,18 +42,6 @@ public class QueryDataSetUtils {
private QueryDataSetUtils() {}
- /**
- * convert query data set by fetch size.
- *
- * @param queryDataSet -query dataset
- * @param fetchSize -fetch size
- * @return -convert query dataset
- */
- public static TSQueryDataSet convertQueryDataSetByFetchSize(
- QueryDataSet queryDataSet, int fetchSize) throws IOException {
- return convertQueryDataSetByFetchSize(queryDataSet, fetchSize, null);
- }
-
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
public static TSQueryDataSet convertQueryDataSetByFetchSize(
QueryDataSet queryDataSet, int fetchSize, WatermarkEncoder watermarkEncoder)
@@ -78,6 +66,14 @@ public class QueryDataSetUtils {
for (int i = 0; i < fetchSize; i++) {
if (queryDataSet.hasNext()) {
RowRecord rowRecord = queryDataSet.next();
+
+ // filter rows whose columns are null according to the rule
+ if ((queryDataSet.isWithoutAllNull() && rowRecord.isAllNull())
+ || (queryDataSet.isWithoutAnyNull() && rowRecord.hasNullField())) {
+ i--;
+ continue;
+ }
+
if (watermarkEncoder != null) {
rowRecord = watermarkEncoder.encodeRecord(rowRecord);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAllNullIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAllNullIT.java
new file mode 100644
index 0000000..95cce53
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAllNullIT.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IoTDBWithoutAllNullIT {
+
+ private static final String[] dataSet =
+ new String[] {
+ "SET STORAGE GROUP TO root.testWithoutAllNull",
+ "CREATE TIMESERIES root.testWithoutAllNull.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.testWithoutAllNull.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.testWithoutAllNull.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1) " + "values(6, 26)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s2) " + "values(7, false)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(9, 29, true)",
+ "flush",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(10, 20, true)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2,s3) "
+ + "values(11, 21, false, 11.1)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2) " + "values(12, 22, true)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s2,s3) "
+ + "values(13, 23, false, 33.3)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s1,s3) " + "values(14, 24, 44.4)",
+ "INSERT INTO root.testWithoutAllNull.d1(timestamp,s2,s3) " + "values(15, true, 55.5)",
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(1000);
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ prepareData();
+ }
+
+ private static void prepareData() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : dataSet) {
+ statement.execute(sql);
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void withoutAllNullTest1() {
+ String[] retArray1 = new String[] {"6,20,true,null", "11,24,true,55.5"};
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ALL");
+
+ assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+ assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void withoutAllNullTest2() {
+ String[] retArray1 = new String[] {"11,24,true,55.5"};
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ALL limit 1 offset 1");
+
+ assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+ assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void withoutAllNullTest3() {
+ String[] retArray1 = new String[] {"11,24,true,55.5"};
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "select last_value(*) from root.testWithoutAllNull.d1 GROUP BY([1, 21), 5ms) WITHOUT NULL ANY");
+
+ assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s1)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s2)")
+ + ","
+ + resultSet.getString("last_value(root.testWithoutAllNull.d1.s3)");
+ assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAnyNullIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAnyNullIT.java
new file mode 100644
index 0000000..e7a62af
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBWithoutAnyNullIT.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.integration;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.constant.TestConstant.TIMESTAMP_STR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class IoTDBWithoutAnyNullIT {
+
+ private static final String[] dataSet =
+ new String[] {
+ "SET STORAGE GROUP TO root.testWithoutAnyNull",
+ "CREATE TIMESERIES root.testWithoutAnyNull.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.testWithoutAnyNull.d1.s2 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.testWithoutAnyNull.d1.s3 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+ + "values(1, 21, false, 11.1)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2) " + "values(2, 22, true)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+ + "values(3, 23, false, 33.3)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s3) " + "values(4, 24, 44.4)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2,s3) " + "values(5, true, 55.5)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1) " + "values(6, 26)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2) " + "values(7, false)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s3) " + "values(8, 88.8)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) " + "values(9, 29, true, 99.9)",
+ "flush",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+ + "values(10, 20, true, 10.0)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+ + "values(11, 21, false, 11.1)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2) " + "values(12, 22, true)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) "
+ + "values(13, 23, false, 33.3)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s3) " + "values(14, 24, 44.4)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2,s3) " + "values(15, true, 55.5)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1) " + "values(16, 26)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s2) " + "values(17, false)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s3) " + "values(18, 88.8)",
+ "INSERT INTO root.testWithoutAnyNull.d1(timestamp,s1,s2,s3) " + "values(19, 29, true, 99.9)"
+ };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(1000);
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ prepareData();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ IoTDBDescriptor.getInstance().getConfig().setPartitionInterval(86400);
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void withoutAnyNullTest1() {
+ String[] retArray1 =
+ new String[] {
+ "1,21,false,11.1",
+ "3,23,false,33.3",
+ "9,29,true,99.9",
+ "10,20,true,10.0",
+ "11,21,false,11.1",
+ "13,23,false,33.3",
+ "19,29,true,99.9"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute("select * from root.testWithoutAnyNull.d1 WITHOUT NULL ANY");
+
+ assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+ assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void withoutAnyNullTest2() {
+ String[] retArray =
+ new String[] {"10,20,true,10.0", "11,21,false,11.1", "13,23,false,33.3", "19,29,true,99.9"};
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "select * from root.testWithoutAnyNull.d1 WHERE time >= 10 WITHOUT NULL ANY");
+
+ int cnt;
+ assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+ assertEquals(retArray[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void withoutAnyNullTest3() {
+ String[] retArray1 =
+ new String[] {
+ "3,23,false,33.3",
+ "9,29,true,99.9",
+ "10,20,true,10.0",
+ "11,21,false,11.1",
+ "13,23,false,33.3"
+ };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "select * from root.testWithoutAnyNull.d1 WITHOUT NULL ANY limit 5 offset 1");
+
+ assertTrue(hasResultSet);
+ int cnt;
+ try (ResultSet resultSet = statement.getResultSet()) {
+ cnt = 0;
+ while (resultSet.next()) {
+ String ans =
+ resultSet.getString(TIMESTAMP_STR)
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s1")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s2")
+ + ","
+ + resultSet.getString("root.testWithoutAnyNull.d1.s3");
+ assertEquals(retArray1[cnt], ans);
+ cnt++;
+ }
+ assertEquals(retArray1.length, cnt);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : dataSet) {
+ statement.execute(sql);
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/RowRecord.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/RowRecord.java
index a413f56..ba41702 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/RowRecord.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/RowRecord.java
@@ -25,8 +25,13 @@ import java.util.List;
public class RowRecord {
- private long timestamp;
- private List<Field> fields;
+ private final long timestamp;
+ private final List<Field> fields;
+ /** if any column is null, this field should be set to true; otherwise false */
+ private boolean hasNullField = false;
+
+ /** if any column is not null, this field should be set to false; otherwise true */
+ private boolean allNull = true;
public RowRecord(long timestamp) {
this.timestamp = timestamp;
@@ -36,14 +41,31 @@ public class RowRecord {
public RowRecord(long timestamp, List<Field> fields) {
this.timestamp = timestamp;
this.fields = fields;
+ for (Field field : fields) {
+ if (field == null || field.getDataType() == null) {
+ hasNullField = true;
+ } else {
+ allNull = false;
+ }
+ }
}
public void addField(Field f) {
this.fields.add(f);
+ if (f == null || f.getDataType() == null) {
+ hasNullField = true;
+ } else {
+ allNull = false;
+ }
}
public void addField(Object value, TSDataType dataType) {
this.fields.add(Field.getField(value, dataType));
+ if (value == null || dataType == null) {
+ hasNullField = true;
+ } else {
+ allNull = false;
+ }
}
@Override
@@ -61,19 +83,19 @@ public class RowRecord {
return timestamp;
}
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
public List<Field> getFields() {
return fields;
}
- public void setFields(List<Field> fields) {
- this.fields = fields;
- }
-
public void setField(int index, Field field) {
this.fields.set(index, field);
}
+
+ public boolean hasNullField() {
+ return hasNullField;
+ }
+
+ public boolean isAllNull() {
+ return allNull;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/QueryDataSet.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/QueryDataSet.java
index 453403e..c782743 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/QueryDataSet.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/dataset/QueryDataSet.java
@@ -42,6 +42,12 @@ public abstract class QueryDataSet {
*/
protected EndPoint endPoint = null;
+ /** if any column is null, we don't need that row */
+ protected boolean withoutAnyNull;
+
+ /** Only if all columns are null, we don't need that row */
+ protected boolean withoutAllNull;
+
/** For redirect query. Need keep consistent with EndPoint in rpc.thrift. */
public static class EndPoint {
private String ip = null;
@@ -97,7 +103,12 @@ public abstract class QueryDataSet {
// proceed to the OFFSET row by skipping rows
while (rowOffset > 0) {
if (hasNextWithoutConstraint()) {
- nextWithoutConstraint(); // DO NOT use next()
+ RowRecord rowRecord = nextWithoutConstraint(); // DO NOT use next()
+ // filter rows whose columns are null according to the rule
+ if ((withoutAllNull && rowRecord.isAllNull())
+ || (withoutAnyNull && rowRecord.hasNullField())) {
+ continue;
+ }
rowOffset--;
} else {
return false;
@@ -167,4 +178,20 @@ public abstract class QueryDataSet {
public void setEndPoint(EndPoint endPoint) {
this.endPoint = endPoint;
}
+
+ public boolean isWithoutAnyNull() {
+ return withoutAnyNull;
+ }
+
+ public void setWithoutAnyNull(boolean withoutAnyNull) {
+ this.withoutAnyNull = withoutAnyNull;
+ }
+
+ public boolean isWithoutAllNull() {
+ return withoutAllNull;
+ }
+
+ public void setWithoutAllNull(boolean withoutAllNull) {
+ this.withoutAllNull = withoutAllNull;
+ }
}